/// <summary>
/// Downloads the resource with the specified URI to a local file.
/// </summary>
public void DownloadFile(URL address, string fileName)
{
URLConnection connection = null;
InputStream inputStream = null;
OutputStream outputStream = new FileOutputStream(fileName);
try
{
connection = OpenConnection(address);
inputStream = connection.GetInputStream();
var buffer = new byte[BUFFER_SIZE];
int len;
while ((len = inputStream.Read(buffer, 0, BUFFER_SIZE)) > 0)
{
outputStream.Write(buffer, 0, len);
}
outputStream.Flush();
}
finally
{
if (inputStream != null)
{
inputStream.Close();
}
var httpConnection = connection as HttpURLConnection;
if (httpConnection != null)
{
httpConnection.Disconnect();
}
outputStream.Close();
}
}
/// <summary>
/// Download a resource from the specified URI.
/// </summary>
public byte[] DownloadData(URL address)
{
URLConnection connection = null;
InputStream inputStream = null;
try
{
connection = OpenConnection(address);
inputStream = connection.GetInputStream();
var memStream = new ByteArrayOutputStream();
var buffer = new byte[BUFFER_SIZE];
int len;
while ((len = inputStream.Read(buffer, 0, BUFFER_SIZE)) > 0)
{
memStream.Write(buffer, 0, len);
}
return(memStream.ToByteArray());
}
finally
{
if (inputStream != null)
{
inputStream.Close();
}
var httpConnection = connection as HttpURLConnection;
if (httpConnection != null)
{
httpConnection.Disconnect();
}
}
}
/// <summary>access a url, ignoring some IOException such as the page does not exist</summary>
/// <exception cref="System.IO.IOException"/>
internal static void Access(string urlstring)
{
Log.Warn("access " + urlstring);
Uri url = new Uri(urlstring);
URLConnection connection = url.OpenConnection();
connection.Connect();
try
{
BufferedReader @in = new BufferedReader(new InputStreamReader(connection.GetInputStream
()));
try
{
for (; @in.ReadLine() != null;)
{
}
}
finally
{
@in.Close();
}
}
catch (IOException ioe)
{
Log.Warn("urlstring=" + urlstring, ioe);
}
}
/// <summary>
/// Download a string from the specified URI.
/// </summary>
public string DownloadString(URL address)
{
URLConnection connection = null;
InputStream inputStream = null;
try
{
connection = OpenConnection(address);
inputStream = connection.GetInputStream();
var reader = new InputStreamReader(inputStream);
var buffer = new char[BUFFER_SIZE];
var builder = new StringBuilder();
int len;
while ((len = reader.Read(buffer, 0, BUFFER_SIZE)) > 0)
{
builder.Append(buffer, 0, len);
}
return(builder.ToString());
}
finally
{
if (inputStream != null)
{
inputStream.Close();
}
var httpConnection = connection as HttpURLConnection;
if (httpConnection != null)
{
httpConnection.Disconnect();
}
}
}
protected internal static Edu.Stanford.Nlp.Parser.Lexparser.ChineseLexiconAndWordSegmenter GetSegmenterDataFromSerializedFile(string serializedFileOrUrl)
{
Edu.Stanford.Nlp.Parser.Lexparser.ChineseLexiconAndWordSegmenter cs = null;
try
{
log.Info("Loading segmenter from serialized file " + serializedFileOrUrl + " ...");
ObjectInputStream @in;
InputStream @is;
if (serializedFileOrUrl.StartsWith("http://"))
{
URL u = new URL(serializedFileOrUrl);
URLConnection uc = u.OpenConnection();
@is = uc.GetInputStream();
}
else
{
@is = new FileInputStream(serializedFileOrUrl);
}
if (serializedFileOrUrl.EndsWith(".gz"))
{
// it's faster to do the buffering _outside_ the gzipping as here
@in = new ObjectInputStream(new BufferedInputStream(new GZIPInputStream(@is)));
}
else
{
@in = new ObjectInputStream(new BufferedInputStream(@is));
}
cs = (Edu.Stanford.Nlp.Parser.Lexparser.ChineseLexiconAndWordSegmenter)@in.ReadObject();
@in.Close();
log.Info(" done.");
return(cs);
}
catch (InvalidClassException ice)
{
// For this, it's not a good idea to continue and try it as a text file!
log.Info();
// as in middle of line from above message
throw new Exception(ice);
}
catch (FileNotFoundException fnfe)
{
// For this, it's not a good idea to continue and try it as a text file!
log.Info();
// as in middle of line from above message
throw new Exception(fnfe);
}
catch (StreamCorruptedException)
{
}
catch (Exception e)
{
// suppress error message, on the assumption that we've really got
// a text grammar, and that'll be tried next
log.Info();
// as in middle of line from above message
Sharpen.Runtime.PrintStackTrace(e);
}
return(null);
}
/// <exception cref="System.IO.IOException"/>
private static void ProcessUrl(Uri url)
{
URLConnection con = url.OpenConnection();
// con.setConnectTimeout(connectTimeout);
// con.setReadTimeout(readTimeout);
InputStream @in = con.GetInputStream();
// Read metadata
Com.Drew.Metadata.Metadata metadata;
try
{
metadata = ImageMetadataReader.ReadMetadata(@in);
}
catch (ImageProcessingException e)
{
// this is an error in the Jpeg segment structure. we're looking for bad handling of
// metadata segments. in this case, we didn't even get a segment.
System.Console.Error.Printf("%s: %s [Error Extracting Metadata]\n\t%s%n", e.GetType().FullName, url, e.Message);
return;
}
catch (Exception t)
{
// general, uncaught exception during processing of jpeg segments
System.Console.Error.Printf("%s: %s [Error Extracting Metadata]%n", t.GetType().FullName, url);
Sharpen.Runtime.PrintStackTrace(t, System.Console.Error);
return;
}
if (metadata.HasErrors())
{
System.Console.Error.Println(url);
foreach (Com.Drew.Metadata.Directory directory in metadata.GetDirectories())
{
if (!directory.HasErrors())
{
continue;
}
foreach (string error in directory.GetErrors())
{
System.Console.Error.Printf("\t[%s] %s%n", directory.GetName(), error);
}
}
}
// Iterate through all values
foreach (Com.Drew.Metadata.Directory directory_1 in metadata.GetDirectories())
{
foreach (Tag tag in directory_1.GetTags())
{
string tagName = tag.GetTagName();
string directoryName = directory_1.GetName();
string description = tag.GetDescription();
// truncate the description if it's too long
if (description != null && description.Length > 1024)
{
description = Sharpen.Runtime.Substring(description, 0, 1024) + "...";
}
System.Console.Out.Printf("[%s] %s = %s%n", directoryName, tagName, description);
}
}
}
/// <exception cref="System.IO.IOException"></exception>
internal override WalkRemoteObjectDatabase.FileStream Open(string path)
{
URLConnection c = this._enclosing.s3.Get(this._enclosing.bucket, this.ResolveKey(
path));
InputStream raw = c.GetInputStream();
InputStream @in = this._enclosing.s3.Decrypt(c);
int len = c.GetContentLength();
return(new WalkRemoteObjectDatabase.FileStream(@in, raw == @in ? len : -1));
}
// cluster
/// <summary>Read in the content from a URL</summary>
/// <param name="url">URL To read</param>
/// <returns>the text from the output</returns>
/// <exception cref="System.IO.IOException">if something went wrong</exception>
private static string ReadOutput(Uri url)
{
StringBuilder @out = new StringBuilder();
URLConnection connection = url.OpenConnection();
BufferedReader @in = new BufferedReader(new InputStreamReader(connection.GetInputStream
(), Charsets.Utf8));
string inputLine;
while ((inputLine = @in.ReadLine()) != null)
{
@out.Append(inputLine);
}
@in.Close();
return(@out.ToString());
}
/// <exception cref="System.Exception"/>
public virtual void TestDynamicLogLevel()
{
string logName = typeof(TestLogLevel).FullName;
Org.Apache.Commons.Logging.Log testlog = LogFactory.GetLog(logName);
//only test Log4JLogger
if (testlog is Log4JLogger)
{
Logger log = ((Log4JLogger)testlog).GetLogger();
log.Debug("log.debug1");
log.Info("log.info1");
log.Error("log.error1");
Assert.True(!Level.Error.Equals(log.GetEffectiveLevel()));
HttpServer2 server = new HttpServer2.Builder().SetName("..").AddEndpoint(new URI(
"http://*****:*****@out.WriteLine("*** Connecting to " + url);
URLConnection connection = url.OpenConnection();
connection.Connect();
BufferedReader @in = new BufferedReader(new InputStreamReader(connection.GetInputStream
()));
for (string line; (line = @in.ReadLine()) != null; @out.WriteLine(line))
{
}
@in.Close();
log.Debug("log.debug2");
log.Info("log.info2");
log.Error("log.error2");
Assert.True(Level.Error.Equals(log.GetEffectiveLevel()));
//command line
string[] args = new string[] { "-setlevel", authority, logName, Level.Debug.ToString
() };
LogLevel.Main(args);
log.Debug("log.debug3");
log.Info("log.info3");
log.Error("log.error3");
Assert.True(Level.Debug.Equals(log.GetEffectiveLevel()));
}
else
{
@out.WriteLine(testlog.GetType() + " not tested.");
}
}
/// <exception cref="System.IO.IOException"></exception>
public virtual void TestDocWithAttachment()
{
string inlineTextString = "Inline text string created by cblite functional test";
Send("PUT", "/db", Status.Created, null);
IDictionary <string, object> attachment = new Dictionary <string, object>();
attachment["content_type"] = "text/plain";
attachment["data"] = "SW5saW5lIHRleHQgc3RyaW5nIGNyZWF0ZWQgYnkgY2JsaXRlIGZ1bmN0aW9uYWwgdGVzdA=="
;
IDictionary <string, object> attachments = new Dictionary <string, object>();
attachments["inline.txt"] = attachment;
IDictionary <string, object> docWithAttachment = new Dictionary <string, object>();
docWithAttachment["_id"] = "docWithAttachment";
docWithAttachment["text"] = inlineTextString;
docWithAttachment["_attachments"] = attachments;
IDictionary <string, object> result = (IDictionary <string, object>)SendBody("PUT",
"/db/docWithAttachment", docWithAttachment, Status.Created, null);
result = (IDictionary <string, object>)Send("GET", "/db/docWithAttachment", Status
.Ok, null);
IDictionary <string, object> attachmentsResult = (IDictionary <string, object>)result
["_attachments"];
IDictionary <string, object> attachmentResult = (IDictionary <string, object>)attachmentsResult
.Get("inline.txt");
// there should be either a content_type or content-type field.
//https://github.com/couchbase/couchbase-lite-android-core/issues/12
//content_type becomes null for attachments in responses, should be as set in Content-Type
string contentTypeField = (string)attachmentResult["content_type"];
NUnit.Framework.Assert.IsTrue(attachmentResult.ContainsKey("content_type"));
NUnit.Framework.Assert.IsNotNull(contentTypeField);
URLConnection conn = SendRequest("GET", "/db/docWithAttachment/inline.txt", null,
null);
string contentType = conn.GetHeaderField("Content-Type");
NUnit.Framework.Assert.IsNotNull(contentType);
NUnit.Framework.Assert.IsTrue(contentType.Contains("text/plain"));
StringWriter writer = new StringWriter();
IOUtils.Copy(conn.GetInputStream(), writer, "UTF-8");
string responseString = writer.ToString();
NUnit.Framework.Assert.IsTrue(responseString.Contains(inlineTextString));
}
private static void Process(string urlstring)
{
try
{
Uri url = new Uri(urlstring);
System.Console.Out.WriteLine("Connecting to " + url);
URLConnection connection = url.OpenConnection();
connection.Connect();
BufferedReader @in = new BufferedReader(new InputStreamReader(connection.GetInputStream
(), Charsets.Utf8));
for (string line; (line = @in.ReadLine()) != null;)
{
if (line.StartsWith(Marker))
{
System.Console.Out.WriteLine(Tag.Matcher(line).ReplaceAll(string.Empty));
}
}
@in.Close();
}
catch (IOException ioe)
{
System.Console.Error.WriteLine(string.Empty + ioe);
}
}
/// <exception cref="System.IO.IOException"></exception>
public virtual void TestDocWithAttachment()
{
string inlineTextString = "Inline text string created by cblite functional test";
Send("PUT", "/db", Status.Created, null);
IDictionary <string, object> attachment = new Dictionary <string, object>();
attachment.Put("content_type", "text/plain");
attachment.Put("data", "SW5saW5lIHRleHQgc3RyaW5nIGNyZWF0ZWQgYnkgY2JsaXRlIGZ1bmN0aW9uYWwgdGVzdA=="
);
IDictionary <string, object> attachments = new Dictionary <string, object>();
attachments.Put("inline.txt", attachment);
IDictionary <string, object> docWithAttachment = new Dictionary <string, object>();
docWithAttachment.Put("_id", "docWithAttachment");
docWithAttachment.Put("text", inlineTextString);
docWithAttachment.Put("_attachments", attachments);
IDictionary <string, object> result = (IDictionary <string, object>)SendBody("PUT",
"/db/docWithAttachment", docWithAttachment, Status.Created, null);
IDictionary expChanges = new Dictionary <string, IDictionary <string, object> >();
IList changesResults = new ArrayList();
IDictionary docChanges = new Dictionary <string, object>();
docChanges.Put("id", "docWithAttachment");
docChanges.Put("seq", 1);
IList lChanges = new AList <IDictionary <string, object> >();
Hashtable mChanges = new Dictionary <string, object>();
mChanges.Put("rev", result.Get("rev"));
lChanges.AddItem(mChanges);
docChanges.Put("changes", lChanges);
changesResults.AddItem(docChanges);
expChanges.Put("results", changesResults);
expChanges.Put("last_seq", 1);
Send("GET", "/db/_changes?feed=normal&heartbeat=300000&style=all_docs", Status.Ok
, expChanges);
result = (IDictionary <string, object>)Send("GET", "/db/docWithAttachment", Status
.Ok, null);
IDictionary <string, object> attachmentsResult = (IDictionary <string, object>)result
.Get("_attachments");
IDictionary <string, object> attachmentResult = (IDictionary <string, object>)attachmentsResult
.Get("inline.txt");
// there should be either a content_type or content-type field.
//https://github.com/couchbase/couchbase-lite-android-core/issues/12
//content_type becomes null for attachments in responses, should be as set in Content-Type
string contentTypeField = (string)attachmentResult.Get("content_type");
NUnit.Framework.Assert.IsTrue(attachmentResult.ContainsKey("content_type"));
NUnit.Framework.Assert.IsNotNull(contentTypeField);
URLConnection conn = SendRequest("GET", "/db/docWithAttachment/inline.txt", null,
null);
string contentType = conn.GetHeaderField("Content-Type");
NUnit.Framework.Assert.IsNotNull(contentType);
NUnit.Framework.Assert.IsTrue(contentType.Contains("text/plain"));
StringWriter writer = new StringWriter();
IOUtils.Copy(conn.GetInputStream(), writer, "UTF-8");
string responseString = writer.ToString();
NUnit.Framework.Assert.IsTrue(responseString.Contains(inlineTextString));
}
// 1. Create the input
// 1.1 Create a protocol buffer
// 1.2 Create the query params
// 2. Create a connection
// 3. Do the annotation
// This method has two contracts:
// 1. It should call the two relevant callbacks
// 2. It must not throw an exception
/// <summary>Actually try to perform the annotation on the server side.</summary>
/// <remarks>
/// Actually try to perform the annotation on the server side.
/// This is factored out so that we can retry up to 3 times.
/// </remarks>
/// <param name="annotation">The annotation we need to fill.</param>
/// <param name="backend">The backend we are querying against.</param>
/// <param name="serverURL">The URL of the server we are hitting.</param>
/// <param name="message">The message we are sending the server (don't need to recompute each retry).</param>
/// <param name="tries">The number of times we've tried already.</param>
private void DoAnnotation(Annotation annotation, StanfordCoreNLPClient.Backend backend, URL serverURL, byte[] message, int tries)
{
try
{
// 1. Set up the connection
URLConnection connection = serverURL.OpenConnection();
// 1.1 Set authentication
if (apiKey != null && apiSecret != null)
{
string userpass = apiKey + ":" + apiSecret;
string basicAuth = "Basic " + Sharpen.Runtime.GetStringForBytes(Base64.GetEncoder().Encode(Sharpen.Runtime.GetBytesForString(userpass)));
connection.SetRequestProperty("Authorization", basicAuth);
}
// 1.2 Set some protocol-independent properties
connection.SetDoOutput(true);
connection.SetRequestProperty("Content-Type", "application/x-protobuf");
connection.SetRequestProperty("Content-Length", int.ToString(message.Length));
connection.SetRequestProperty("Accept-Charset", "utf-8");
connection.SetRequestProperty("User-Agent", typeof(StanfordCoreNLPClient).FullName);
switch (backend.protocol)
{
case "https":
case "http":
{
// 1.3 Set some protocol-dependent properties
((HttpURLConnection)connection).SetRequestMethod("POST");
break;
}
default:
{
throw new InvalidOperationException("Haven't implemented protocol: " + backend.protocol);
}
}
// 2. Annotate
// 2.1. Fire off the request
connection.Connect();
connection.GetOutputStream().Write(message);
connection.GetOutputStream().Flush();
// 2.2 Await a response
// -- It might be possible to send more than one message, but we are not going to do that.
Annotation response = serializer.Read(connection.GetInputStream()).first;
// 2.3. Copy response over to original annotation
foreach (Type key in response.KeySet())
{
annotation.Set(key, response.Get(key));
}
}
catch (Exception t)
{
// 3. We encountered an error -- retry
if (tries < 3)
{
log.Warn(t);
DoAnnotation(annotation, backend, serverURL, message, tries + 1);
}
else
{
throw new Exception(t);
}
}
}
/// <summary>
/// Decrypt an input stream from
/// <see cref="Get(string, string)">Get(string, string)</see>
/// .
/// </summary>
/// <param name="u">
/// connection previously created by
/// <see cref="Get(string, string)">Get(string, string)</see>
/// }.
/// </param>
/// <returns>stream to read plain text from.</returns>
/// <exception cref="System.IO.IOException">decryption could not be configured.</exception>
public virtual InputStream Decrypt(URLConnection u)
{
return(encryption.Decrypt(u.GetInputStream()));
}