/// <summary>
/// Get a string representation of the web client.
/// </summary>
public override string ToString()
{
if (Connections.Count == 0)
{
return("[WebClient No Connections]");
}
var builder = StringBuilderCache.Get();
builder.Append("[WebClient ");
bool first = true;
foreach (var connection in Connections)
{
if (first)
{
first = false;
}
else
{
builder.Append(Chars.Comma);
}
builder.Append(connection.RemoteEndpoint);
}
builder.Append(" ]");
return(StringBuilderCache.SetAndGet(builder));
}
/// <summary>
/// Get the equivalent Cassandra data type for the specified net type.
/// </summary>
public static string GetDataTypeString(Type type)
{
// get the data type
DataType dataType;
Type baseType;
if (type.IsGenericType && !type.IsGenericTypeDefinition)
{
baseType = type.GetGenericTypeDefinition();
}
else
{
baseType = type;
}
if (TypeMap.TryGetValue(baseType, out dataType))
{
switch (dataType)
{
case DataType.List:
return("LIST<" + GetDataTypeString(type.GetGenericArguments()[0]) + Chars.GreaterThan);
case DataType.Set:
return("SET<" + GetDataTypeString(type.GetGenericArguments()[0]) + Chars.GreaterThan);
case DataType.Map:
var args = type.GetGenericArguments();
if (args.Length != 2)
{
throw new ArgumentException("Invalid number of arguments for MAP type.");
}
return("MAP<" + GetDataTypeString(args[0]) + Chars.Comma + GetDataTypeString(args[1]) + Chars.GreaterThan);
case DataType.Tuple:
StringBuilder sb = StringBuilderCache.Get();
sb.Append("TUPLE<");
bool first = true;
foreach (Type genType in type.GetGenericArguments())
{
if (first)
{
first = false;
}
else
{
sb.Append(Chars.Comma);
}
sb.Append(GetDataTypeString(genType));
}
sb.Append(Chars.GreaterThan);
return(StringBuilderCache.SetAndGet(sb));
default:
return(dataType.ToString().ToLowercase());
}
}
// it's a user defined type or tuple
return("blob");
}
/// <summary>
/// Get a json representation of a node.
/// </summary>
public static string SerializeJson(Node node)
{
var writer = StringBuilderCache.Get();
bool success = SerializeJson(node, writer, string.Empty);
return(StringBuilderCache.SetAndGet(writer));
}
/// <summary>
/// Get the complete string representation and objects to bind to the query.
/// </summary>
public void Build()
{
_build = false;
// initialize the string builder
_builder = StringBuilderCache.Get();
if (_query.Keywords.Count == 0)
{
Log.Warning("Query contains zero commands.");
}
else
{
switch (_query.Keywords[_keywordIndex])
{
case Cql.Select:
++_keywordIndex;
ConstructSelect();
break;
case Cql.Update:
++_keywordIndex;
ConstructUpdate();
break;
case Cql.Insert:
++_keywordIndex;
ConstructInsert();
break;
case Cql.Delete:
++_keywordIndex;
ConstructDelete();
break;
case Cql.Create:
++_keywordIndex;
switch (_query.Keywords[_keywordIndex])
{
case Cql.Table:
++_keywordIndex;
ConstructCreateTable();
break;
default:
Log.Warning("Unexpected initial command type '" + _query.Keywords[0] + "'.");
break;
}
break;
default:
Log.Warning("Unexpected initial command type '" + _query.Keywords[0] + "'.");
break;
}
}
_queryString = StringBuilderCache.SetAndGet(_builder);
_builder = null;
}
/// <summary>
/// Initialize default identities.
/// </summary>
protected static void Initialize()
{
// add default identities
Identity identity = new Identity(true);
identity.Add("Accept", "image/jpeg, application/x-ms-application, image/gif, application/xaml+xml, image/pjpeg, application/x-ms-xbap, application/x-shockwave-flash, application/msword, */*");
identity.Add("User-Agent", "Mozilla/5.0 (compatible; Konqueror/3.5; Linux; X11; i686; en_US) KHTML/3.5.3 (like Gecko)");
identity.Add("Accept-Language", "en-US,en;q=0.5");
identity.Add("If-None-Match", "mJ4imx1p");
identity = new Identity(true);
identity.Add("Accept", "text/html, application/xml;q=0.9, application/xhtml xml, image/png, image/jpeg, image/gif, image/x-xbitmap, */*;q=0.1");
identity.Add("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13");
identity.Add("Accept-Language", "en-US,en;q=0.5");
identity.Add("If-None-Match", "g2ed3k5u");
identity = new Identity(true);
identity.Add("Accept", "text/html, image/png, image/jpeg, image/gif, image/x-xbitmap, */*;q=0.1");
identity.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1");
identity.Add("Accept-Language", "en-US,en;q=0.5");
identity.Add("If-None-Match", "34vn9p8m");
identity = new Identity(true);
identity.Add("Accept", "application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5");
identity.Add("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.11) Gecko/20101023 Firefox/3.6.11 (Palemoon/3.6.11) ( .NET CLR 3.5.30729; .NET4.0E)");
identity.Add("Accept-Language", "en-US,en;q=0.5");
identity.Add("If-None-Match", "iu753id7");
// set common headers
for (int i = _identities.Count - 1; i >= 0; --i)
{
identity = _identities[i];
// spoof the 'Via' heading
var builder = StringBuilderCache.Get(64);
builder.Append("1.1 ");
builder.Append(Randomize.Byte);
builder.Append(Chars.Stop);
builder.Append(Randomize.Byte);
builder.Append(Chars.Stop);
builder.Append(Randomize.Byte);
builder.Append(Chars.Stop);
builder.Append(Randomize.Byte);
identity.Add("Via", builder.ToString());
// spoof the 'X-Forwarded-For' heading
builder.Length = 0;
builder.Append(Randomize.Byte);
builder.Append(Chars.Stop);
builder.Append(Randomize.Byte);
builder.Append(Chars.Stop);
builder.Append(Randomize.Byte);
builder.Append(Chars.Stop);
builder.Append(Randomize.Byte);
identity.Add("X-Forwarded-For", StringBuilderCache.SetAndGet(builder));
identity.Add("Accept-Encoding", "gzip, deflate");
identity.KeepAlive = true;
}
}
/// <summary>
/// Resolve the specified path with the specified callback. If the path has a defined
/// callback, it will be trimmed to a sub-path.
/// </summary>
public void Resolve(string path, IAction <string> onResolved)
{
// split the path into sections
string[] sections = path.Split(Chars.ForwardSlash);
int index = 0;
if (sections[index] == string.Empty)
{
++index;
}
int subPathIndex = 0;
bool first = true;
var builder = StringBuilderCache.Get();
// iterate
while (index < sections.Length)
{
if (first)
{
first = false;
}
else
{
builder.Append(Chars.ForwardSlash);
}
builder.Append(sections[index]);
++index;
// does the path contain a callback? yes, update the sub path
if (_routes.ContainsKey(builder.ToString()))
{
subPathIndex = index;
}
}
// get the sub path
builder.Length = 0;
first = true;
while (subPathIndex < sections.Length)
{
if (first)
{
first = false;
}
else
{
builder.Append(Chars.ForwardSlash);
}
builder.Append(sections[subPathIndex]);
++subPathIndex;
}
onResolved.ArgA = StringBuilderCache.SetAndGet(builder);
onResolved.Run();
}
/// <summary>
/// Get the style as a CSS class.
/// </summary>
public string ToCss(StringBuilder builder)
{
builder.Append(Class);
builder.Append(Chars.BraceOpen);
bool first = true;
if (_params != null)
{
foreach (var param in _params)
{
if (first)
{
first = false;
builder.Append(param.Key);
builder.Append(Chars.Colon);
builder.Append(param.Value);
}
else
{
builder.Append(Chars.SemiColon);
builder.Append(param.Key);
builder.Append(Chars.Colon);
builder.Append(param.Value);
}
}
}
foreach (var param in _properties)
{
if (first)
{
first = false;
builder.Append(StyleKeys.Map[param.Key]);
builder.Append(Chars.Colon);
builder.Append(param.Value);
}
else
{
builder.Append(Chars.SemiColon);
builder.Append(StyleKeys.Map[param.Key]);
builder.Append(Chars.Colon);
builder.Append(param.Value);
}
}
builder.Append(Chars.SemiColon);
builder.Append(Chars.BraceClose);
if (_children != null)
{
foreach (var style in _children)
{
builder.Append(Chars.NewLine);
builder.Append(Class);
style.ToCss(builder);
}
}
return(StringBuilderCache.SetAndGet(builder));
}
/// <summary>
/// Get a hex string representation of this array of bytes.
/// </summary>
public static string ToString(this byte[] bytes)
{
var builder = StringBuilderCache.Get(bytes.Length * 2);
if (bytes != null)
{
foreach (byte bit in bytes)
{
builder.Append(HexStringTable[bit]);
}
}
return(StringBuilderCache.SetAndGet(builder));
}
/// <summary>
/// Build the javascript.
/// </summary>
public string Build()
{
String = StringBuilderCache.Get();
foreach (var component in Components)
{
component.Build(this);
}
var componentsStr = String.ToString();
String.Clear();
foreach (var prototype in Prototypes)
{
prototype.Build(this);
}
return(StringBuilderCache.SetAndGet(String) + componentsStr);
}
/// <summary>
/// Get a string representation of the post parameters.
/// </summary>
public override string ToString()
{
var builder = StringBuilderCache.Get();
builder.Append("Post parameters [");
// iterate the string parameters
foreach (var parameter in ParamsStrings)
{
builder.Append(Chars.NewLine);
builder.Append(parameter.Key);
builder.Append(Chars.Equal);
builder.Append(parameter.Value.Value);
builder.Append(Chars.NewLine);
foreach (var attribute in parameter.Value.Params)
{
builder.Append(attribute.Key);
builder.Append(Chars.Colon);
builder.Append(attribute.Value);
builder.Append(Chars.SemiColon);
builder.Append(Chars.Space);
}
}
// iterate the byte parameters
foreach (var parameter in ParamsBytes)
{
builder.Append(Chars.NewLine);
builder.Append(parameter.Key);
builder.Append(Chars.Equal);
builder.Append(parameter.Value.Value.Count);
builder.Append("bytes");
builder.Append(Chars.NewLine);
foreach (var attribute in parameter.Value.Params)
{
builder.Append(attribute.Key);
builder.Append(Chars.Colon);
builder.Append(attribute.Value);
builder.Append(Chars.SemiColon);
builder.Append(Chars.Space);
}
}
builder.Append(Chars.NewLine);
builder.Append(Chars.BracketSqClose);
return(StringBuilderCache.SetAndGet(builder));
}
/// <summary>
/// Resolve the specified path with the specified callback. If the path has a defined
/// callback, it will be trimmed to a sub-path.
/// </summary>
public void Resolve(string path, HttpMethod method, IAction <string> onResolved)
{
path = path.ToLowercase();
// split the path into sections
string[] sections = path.Split(Chars.ForwardSlash);
int index = 0;
if (sections[index] == string.Empty)
{
++index;
}
int subPathIndex = 0;
bool first = true;
var builder = StringBuilderCache.Get();
// iterate
while (index < sections.Length)
{
if (first)
{
first = false;
}
else
{
builder.Append(Chars.ForwardSlash);
}
builder.Append(sections[index]);
++index;
// does the path contain a callback? yes, update the sub path
ArrayRig <Route> routes;
if (_routes.TryGetValue(builder.ToString(), out routes))
{
foreach (var route in routes)
{
if (route.Methods.Is(method))
{
subPathIndex = index;
}
}
}
}
// get the sub path
builder.Length = 0;
first = true;
while (subPathIndex < sections.Length)
{
if (first)
{
first = false;
}
else
{
builder.Append(Chars.ForwardSlash);
}
builder.Append(sections[subPathIndex]);
++subPathIndex;
}
onResolved.ArgA = StringBuilderCache.SetAndGet(builder);
onResolved.Run();
}
public void Show()
{
StringBuilder builder = StringBuilderCache.Get();
builder.Append("------------- Crawling Stats ---------------\n");
if (ReadCount != 0)
{
builder.Append("Avg Read Time : ");
builder.Append(ReadTime / ReadCount);
builder.Append("ms");
builder.AppendLine();
builder.Append("Total Reads : ");
builder.Append(ReadCount);
}
if (ConnectCount != 0)
{
builder.AppendLine();
builder.Append("Avg Connection Time : ");
builder.Append(ConnectTime / ConnectCount);
builder.Append("ms");
builder.AppendLine();
builder.Append("Total Connections : ");
builder.Append(ConnectCount);
}
builder.AppendLine();
builder.Append("Total Attempts : ");
builder.Append(AttemptCount);
if (ProcessCount != 0)
{
builder.AppendLine();
builder.Append("Avg Processing Time : ");
builder.Append(ProcessTime / ProcessCount);
builder.Append("ms");
builder.AppendLine();
builder.Append("Total Processed : ");
builder.Append(ProcessCount);
}
if (AssetCount != 0)
{
builder.AppendLine();
builder.Append("Avg Time Per Asset : ");
builder.Append((_time.Milliseconds / AssetCount));
builder.Append("ms");
builder.AppendLine();
builder.Append("Total Assets : ");
builder.Append(AssetCount);
builder.AppendLine();
builder.Append("Assets Found");
foreach (KeyValuePair <string, Teple <long, long> > entry in Assets)
{
builder.AppendLine();
builder.Append(entry.Key);
builder.AppendLine();
builder.Append(" Avg Time Per Asset : ");
builder.Append(entry.Value.ArgB / entry.Value.ArgA);
builder.Append("ms");
builder.AppendLine();
builder.Append(" Total Assets : ");
builder.Append(entry.Value.ArgA);
}
}
builder.Append("\n------------- -------------- ---------------\n");
Log.Stat(StringBuilderCache.SetAndGet(builder));
// is the log timer set? yes, restart
if (_logTime > 0)
{
_logTimer.Reset(_logTime);
}
}
/// <summary>
/// Resolve the specified path.
/// </summary>
public void Resolve(string path, T value)
{
// get the index of the start of the params
int subPathIndex = path.IndexOf(Chars.Question);
string paramsSegment = null;
string[] segments;
// was the params index found?
if (subPathIndex == -1)
{
// split the path into segments
segments = path.Split(Chars.ForwardSlash);
}
else
{
// split the path into path and params
Struct <string, string> sections = path.Split(subPathIndex, 0, -1);
// split the path into segments
segments = sections.ArgA.Split(Chars.ForwardSlash);
// persist the params segment
paramsSegment = sections.ArgB;
}
int index = 0;
if (segments[index] == string.Empty)
{
++index;
}
IAction <string, T> callback = null;
var builder = StringBuilderCache.Get();
bool first = true;
subPathIndex = 0;
// iterate
while (index < segments.Length)
{
if (first)
{
first = false;
}
else
{
builder.Append(Chars.ForwardSlash);
}
builder.Append(segments[index]);
++index;
// does the path have a callback?
IAction <string, T> callbackAction;
if (_routes.TryGetValue(builder.ToString(), out callbackAction))
{
callback = callbackAction;
subPathIndex = index;
}
}
// get the sub path
builder.Length = 0;
first = true;
while (subPathIndex < segments.Length)
{
if (first)
{
first = false;
}
else
{
builder.Append(Chars.ForwardSlash);
}
builder.Append(segments[subPathIndex]);
++subPathIndex;
}
// append the params segment
builder.Append(paramsSegment);
// has the callback been set?
if (callback == null)
{
// no, has the default callback been set?
if (DefaultCallback == null)
{
// no, log the error
Log.Error("No default callback set. Request for '" + path + "' went unresolved.");
return;
}
// set the default callback
callback = DefaultCallback;
}
_lock.Take();
callback.ArgA = StringBuilderCache.SetAndGet(builder);
callback.ArgB = value;
callback.Run();
_lock.Release();
}