//---------------------------------------------------------------------
// Static members
/// <summary>
/// Reads the GeoTracker server settings from the application's configuration
/// using the specified key prefix.
/// </summary>
/// <param name="keyPrefix">The application configuration key prefix.</param>
/// <returns>The server settings.</returns>
/// <remarks>
/// <para>
/// The GeoTracker server settings are loaded from the application
/// configuration, using the specified key prefix. The following
/// settings are recognized by the class:
/// </para>
/// <div class="tablediv">
/// <table class="dtTABLE" cellspacing="0" ID="Table1">
/// <tr valign="top">
/// <th width="1">Setting</th>
/// <th width="1">Default</th>
/// <th width="90%">Description</th>
/// </tr>
/// <tr valign="top">
/// <td>ServerEP</td>
/// <td><b>logical://LillTek/GeoTracker/Server</b></td>
/// <td>
/// The external LillTek Messaging endpoint for the GeoTracker server cluster.
/// This is the endpoint that <see cref="GeoTrackerClient" /> instances will
/// use to communicate with the cluster.
/// </td>
/// </tr>
/// <tr valign="top">
/// <td>ClusterEP</td>
/// <td><b>logical://LillTek/GeoTracker/Cluster</b></td>
/// <td>
/// The internal LillTek Messaging endpoint for the root of the GeoTracker server cluster.
/// This is the endpoint that cluster servers will use to communicate with each other.
/// </td>
/// </tr>
/// <tr valign="top">
/// <td>ClusterTopology</td>
/// <td><see cref="DynamicHashedTopology"/></td>
/// <td>
/// Describes the topology provider to be used to distribute traffic to
/// GeoTracker server instances within the cluster. This provide must
/// implement <see cref="TopologyCapability" />.<see cref="TopologyCapability.Locality" />.
/// </td>
/// </tr>
/// <tr valign="top">
/// <td>ClusterArgs</td>
/// <td>None</td>
/// <td>
/// Cluster topology specific parameters formatted as <b>name=value</b> pairs separated
/// by LF ('\n') characters. Use the <see cref="Config" /> "{{" ... ""}}" setting syntax
/// and place each argument on a separate line in the config file.
/// </td>
/// </tr>
/// <tr valign="top">
/// <td>GeoFixArchiver</td>
/// <td><see cref="NullGeoFixArchiver"/></td>
/// <td>
/// The pluggable <see cref="IGeoFixArchiver" /> type to be used to archive
/// <see cref="GeoFix" />es received by the server.
/// </td>
/// </tr>
/// <tr valign="top">
/// <td>GeoFixArchiverArgs</td>
/// <td>Empty</td>
/// <td>
/// Archiver specific parameters formatted as <b>name=value</b> pairs separated
/// by LF ('\n') characters. Use the <see cref="Config" /> "{{" ... ""}}" setting syntax
/// and place each argument on a separate line in the config file.
/// </td>
/// </tr>
/// <tr valign="top">
/// <td>GeoFixRetentionInterval</td>
/// <td>1h</td>
/// <td>
/// The length of time entity <see cref="GeoFix" />es will be retained by GeoTracker.
/// </td>
/// </tr>
/// <tr valign="top">
/// <td>GeoFixPurgeInterval</td>
/// <td>1m</td>
/// <td>
/// The interval at which old cached entity <see cref="GeoFix" />es will be purged.
/// </td>
/// </tr>
/// <tr valign="top">
/// <td>MaxEntityGeoFixes</td>
/// <td>50</td>
/// <td>
/// The maximum number of <see cref="GeoFix" />es to be cached in memory for an entity.
/// </td>
/// </tr>
/// <tr valign="top">
/// <td>IndexHighWatermarkLimit</td>
/// <td>1000</td>
/// <td>
/// <para>
/// Used to decide when to attempt split an index block into sub-blocks when the number
/// of entities in a block is greater than or equal to this value.
/// </para>
/// <note>
/// <b>IndexHighWatermarkLimit</b> must be greater than or equal to <b>IndexLowWatermarkLimit</b>.
/// </note>
/// </td>
/// </tr>
/// <tr valign="top">
/// <td>IndexLowWatermarkLimit</td>
/// <td>750</td>
/// <td>
/// <para>
/// Used to decide when to attempt coalesce sub-blocks when the number
/// of entities in a block is greater than or equal to this value.
/// </para>
/// <note>
/// <b>IndexHighWatermarkLimit</b> must be greater than or equal to <b>IndexLowWatermarkLimit</b>.
/// </note>
/// </td>
/// </tr>
/// <tr valign="top">
/// <td>IndexMaxGroupTableLevel</td>
/// <td>2</td>
/// <td>
/// Controls how deep into the index heirarchy group hash tables will be maintained by
/// the GeoTracker server. Enabling these tables deeper in the heirarchy may result in better
/// performance for queries with group constraints but potentially at the cost of
/// severe memory utilizaton.
/// </td>
/// </tr>
/// <tr>
/// <td>IndexBalancingInterval</td>
/// <td>5m</td>
/// <td>
/// The interval at which the server will attempt to rebalance the location index.
/// </td>
/// </tr>
/// <tr valign="top">
/// <td>IPGeocodeEnabled</td>
/// <td><c>true</c></td>
/// <td>
/// Controls whether IP geocoding services are to be made available by the GeoTracker server.
/// </td>
/// </tr>
/// <tr valign="top">
/// <td>IPGeocodeSourceUri</td>
/// <td><b>see note</b></td>
/// <td>
/// The URL where the current IP to location Geocoding database from <a href="http://maxmind.com">maxmind.com</a>
/// can be downloaded. This must be a decompressed GeoIP City or GeoLite City database file encrypted
/// into a <see cref="SecureFile" />. This setting defaults to: http://www.lilltek.com/Config/GeoTracker/IP2City.encrypted.dat
/// </td>
/// </tr>
/// <tr valign="top">
/// <td>IPGeocodeSourceRsaKey</td>
/// <td><b>See note</b></td>
/// <td>
/// The private RSA key used to decrypt the downloaded Geocoding database. (defaults to
/// the value used to manually encrypt the file hosted on http://www.lilltek.com).
/// </td>
/// </tr>
/// <tr valign="top">
/// <td>IPGeocodeSourcePollInterval</td>
/// <td><b>1d</b></td>
/// <td>
/// The interval at which server instances will poll for updates to the IP Geocode database,
/// Note that server will use the HTTP <b>If-Modified-Since</b> header for efficency since this
/// database is updated only once a month.
/// </td>
/// </tr>
/// <tr valign="top">
/// <td>IPGeocodeSourceTimeout</td>
/// <td><b>5m</b></td>
/// <td>
/// The maximum time the server will wait for the download of the IP Geocode database file.
/// </td>
/// </tr>
/// <tr valign="top">
/// <td>SweepInterval</td>
/// <td><b>2.5m</b></td>
/// <td>
/// The interval at which old <see cref="GeoFix" />es, entities, and groups will be
/// swept up and discarded by the server.
/// </td>
/// </tr>
/// <tr valign="top">
/// <td>BkInterval</td>
/// <td><b>1s</b></td>
/// <td>
/// The minimum interval at which background activities will be scheduled.
/// </td>
/// </tr>
/// </table>
/// </div>
/// </remarks>
public static GeoTrackerServerSettings LoadConfig(string keyPrefix)
{
var config = new Config(keyPrefix);
var settings = new GeoTrackerServerSettings();
settings.ServerEP = config.Get("ServerEP", settings.ServerEP);
settings.ClusterEP = config.Get("ClusterEP", settings.ClusterEP);
settings.ClusterTopology = config.Get("ClusterTopology", settings.ClusterTopology);
settings.ClusterArgs = new ArgCollection(config.Get("ClusterArgs", settings.ClusterArgs.ToString()), '=', '\n');
settings.GeoFixArchiver = config.Get("GeoFixArchiver", settings.GeoFixArchiver);
settings.GeoFixArchiverArgs = ArgCollection.Parse(config.Get("GeoFixArchiverArgs", settings.GeoFixArchiverArgs.ToString()), '=', '\n');
settings.GeoFixRetentionInterval = config.Get("GeoFixRetentionInterval", settings.GeoFixRetentionInterval);
settings.GeoFixPurgeInterval = config.Get("GeoFixPurgeInterval", settings.GeoFixPurgeInterval);
settings.MaxEntityGeoFixes = config.Get("MaxEntityGeoFixes", settings.MaxEntityGeoFixes);
settings.IndexHighWatermarkLimit = config.Get("IndexHighWatermarkLimit", settings.IndexHighWatermarkLimit);
settings.IndexLowWatermarkLimit = config.Get("IndexLowWatermarkLimit", settings.IndexLowWatermarkLimit);
settings.IndexMaxGroupTableLevel = config.Get("IndexMaxGroupTableLevel", settings.IndexMaxGroupTableLevel);
settings.IndexBalancingInterval = config.Get("IndexBalancingInterval", settings.IndexBalancingInterval);
settings.IPGeocodeEnabled = config.Get("IPGeocodeEnabled", settings.IPGeocodeEnabled);
settings.IPGeocodeSourceUri = config.Get("IPGeocodeSourceUri", settings.IPGeocodeSourceUri);
settings.IPGeocodeSourceRsaKey = config.Get("IPGeocodeSourceRsaKey", settings.IPGeocodeSourceRsaKey);
settings.IPGeocodeSourcePollInterval = config.Get("IPGeocodeSourcePollInterval", settings.IPGeocodeSourcePollInterval);
settings.IPGeocodeSourceTimeout = config.Get("IPGeocodeSourceTimeout", settings.IPGeocodeSourceTimeout);
settings.SweepInterval = config.Get("SweepInterval", settings.SweepInterval);
settings.BkInterval = config.Get("BkInterval", settings.BkInterval);
return(settings);
}
/// <summary>
/// Constructs and starts the cache instance.
/// </summary>
/// <param name="settings">The GeoTracker settings.</param>
public GeoFixCache(GeoTrackerServerSettings settings)
{
this.settings = settings;
this.isRunning = true;
this.rwLock = new ReaderWriterLockSlim();
this.purgeTimer = new PolledTimer(settings.GeoFixPurgeInterval);
this.bkThread = new Thread(new ThreadStart(BkThread));
this.bkThread.Name = "GeoTracker: Cache";
this.bkThread.Start();
}
/// <summary>
/// Associates the service handler with a message router by registering
/// the necessary application message handlers.
/// </summary>
/// <param name="router">The message router.</param>
/// <param name="settings">The configuration settings.</param>
/// <param name="perfCounters">The application's performance counter set (or <c>null</c>).</param>
/// <param name="perfPrefix">The string to prefix any performance counter names (or <c>null</c>).</param>
/// <remarks>
/// <para>
/// Applications that expose performance counters will pass a non-<c>null</c> <b>perfCounters</b>
/// instance. The service handler should add any counters it implements to this set.
/// If <paramref name="perfPrefix" /> is not <c>null</c> then any counters added should prefix their
/// names with this parameter.
/// </para>
/// </remarks>
public void Start(MsgRouter router, GeoTrackerServerSettings settings, PerfCounterSet perfCounters, string perfPrefix)
{
if (this.isRunning)
{
throw new InvalidOperationException("This node has already been started.");
}
if (router == null)
{
throw new ArgumentNullException("router");
}
// Initialize the performance counters
this.startTime = DateTime.UtcNow;
this.perf = new Perf(perfCounters, perfPrefix);
// General initialization
this.settings = settings;
this.bkTimer = new GatedTimer(new TimerCallback(OnBkTimer), null, settings.BkInterval);
this.ipGeocoder = new IPGeocoder(this);
this.clusterClient = Helper.CreateInstance <ITopologyProvider>(settings.ClusterTopology);
this.clusterServer = Helper.CreateInstance <ITopologyProvider>(settings.ClusterTopology);
this.fixCache = new GeoFixCache(settings);
this.archiver = Helper.CreateInstance <IGeoFixArchiver>(settings.GeoFixArchiver);
EntityState.MaxEntityFixes = settings.MaxEntityGeoFixes;
try
{
// Initialize the router
this.router = router;
this.router.Dispatcher.AddTarget(this, "GeoTrackerServerEP", new SimpleEPMunger(settings.ServerEP), null);
// Initialize the cluster
this.clusterClient.OpenClient(router, settings.ClusterEP, settings.ClusterArgs);
this.clusterServer.OpenServer(router, "GeoTrackerClusterEP", settings.ClusterEP, this, settings.ClusterArgs);
// Start the archiver.
archiver.Start(this, settings.GeoFixArchiverArgs);
this.isRunning = true;
}
catch
{
Stop();
throw;
}
}
/// <summary>
/// Initalizes the fix archiver instance.
/// </summary>
/// <param name="node">The parent <see cref="GeoTrackerNode" /> instance.</param>
/// <param name="args">This implementation recognizes special arguments (see the remarks).</param>
/// <remarks>
/// <para>
/// This archiver recognizes the following arguments:
/// </para>
/// <list type="table">
/// <item>
/// <term><b>ConnectionString</b></term>
/// <description>
/// <b>Required:</b> This specifies the SQL connection string the archiver will use
/// to connect to the database server.
/// </description>
/// </item>
/// <item>
/// <term><b>BufferSize</b></term>
/// <description>
/// <b>Optional:</b> This specifies the number of fixes the archiver will buffer before
/// submitting the fixes to the database in a batch. (defaults to <b>1000</b>).
/// </description>
/// </item>
/// <item>
/// <term><b>BufferInterval</b></term>
/// <description>
/// <b>Optional: </b> The maximum length of time the archiver will buffer fixes before
/// flushing to the database, regardless of how full thbe buffer is. (defaults to <b>5 minutes</b>).
/// </description>
/// </item>
/// <item>
/// <term><b>AddScript</b></term>
/// <description>
/// <para>
/// The SQL script template the archiver will use for submitting fixes to the database.
/// This script will include macros where the archiver will substitute the values
/// from the <see cref="GeoFix" /> being persisted. Here are the supported macros:
/// </para>
/// <list type="table">
/// <item>
/// <term><b>@(TimeUtc)</b></term>
/// <description>
/// The time (UTC) the fix was taken.
/// </description>
/// </item>
/// <item>
/// <term><b>@(EntityID)</b></term>
/// <description>
/// The ID of the entity being tracked.
/// </description>
/// </item>
/// <item>
/// <term><b>@(GroupID)</b></term>
/// <description>
/// <b>Nullable:</b> The ID of the group the entity belongs to.
/// </description>
/// </item>
/// <item>
/// <term><b>@(Technology)</b></term>
/// <description>
/// The technology used to obtain the position.
/// </description>
/// </item>
/// <item>
/// <term><b>@(Latitude)</b></term>
/// <description>
/// The latitude.
/// </description>
/// </item>
/// <item>
/// <term><b>@(Longitude)</b></term>
/// <description>
/// The longitude.
/// </description>
/// </item>
/// <item>
/// <term><b>@(Altitude)</b></term>
/// <description>
/// <b>Nullable:</b> The altitude in meters.
/// </description>
/// </item>
/// <item>
/// <term><b>@(Course)</b></term>
/// <description>
/// <b>Nullable:</b> The direction of travel in degrees from true north.
/// </description>
/// </item>
/// <item>
/// <term><b>@(Speed)</b></term>
/// <description>
/// <b>Nullable:</b> The speed in kilometers per hour.
/// </description>
/// </item>
/// <item>
/// <term><b>@(HorizontalAccuracy)</b></term>
/// <description>
/// <b>Nullable:</b> The estimated horizontal accuracy of the fix in meters.
/// </description>
/// </item>
/// <item>
/// <term><b>@(VerticalAccurancy)</b></term>
/// <description>
/// <b>Nullable:</b> The estimated vertical accuracy of the fix in meters.
/// </description>
/// </item>
/// <item>
/// <term><b>@(NetworkStatus)</b></term>
/// <description>
/// Identifies how or if the entity was connected to the Internet when the fix was taken.
/// </description>
/// </item>
/// </list>
/// <note>
/// We're using the <b>"@"</b> rather than the usual <b>"$"</b> character to mark the
/// macro name to avoid conflicting with environment variable expansions performed
/// when loading the application configuration.
/// </note>
/// </description>
/// </item>
/// </list>
/// <para>
/// The most important archiver parameter is <b>AddScript</b>. This is the template the archiver
/// will use to generate the SQL statements that will add <see cref="GeoFix" />es to the database.
/// The script can be as simple as an ad-hoc table <b>insert</b> or a call to a stored procedure.
/// Add one or more of the macro identifiers listed above to the script. The archiver will replace
/// these macros with actual fields from the <see cref="GeoFix" />, quoting them as necessary.
/// Null <see cref="GeoFix" /> fields or numeric fields set to <see cref="double.NaN" /> will
/// be generated as <c>null</c>. See the <b>Nullable</b> tags in the macro definitions above.
/// </para>
/// <para>
/// Here's an exmple of a simple sample script that will insert the <see cref="GeoFix.TimeUtc" />,
/// <b>entity ID</b>, <see cref="GeoFix.Latitude" />, and <see cref="GeoFix.Longitude" />
/// <see cref="GeoFix" /> fields into a table.
/// </para>
/// <code language="none">
/// insert into FixArchive(Time,Entity,Lat,Lon) values (@(TimeUtc),@(EntityID),@(Latitude),@(longitude))
/// </code>
/// <note>
/// <see cref="IGeoFixArchiver" /> implementations must silently handle any internal
/// error conditions. <see cref="GeoTrackerNode" /> does not expect to see any
/// exceptions raised from calls to any of these methods. Implementations should
/// catch any exceptions thrown internally and log errors or warnings as necessary.
/// </note>
/// </remarks>
public void Start(GeoTrackerNode node, ArgCollection args)
{
lock (syncLock)
{
try
{
if (isRunning)
{
throw new InvalidProgramException("SqlGeoFixArchiver: Archiver has already been started.");
}
if (isStopped)
{
throw new InvalidOperationException("SqlGeoFixArchiver: Cannot restart a stopped archiver.");
}
this.node = node;
this.settings = node.Settings;
this.bufferedFixes = new List <FixRecord>();
this.conString = args.Get("ConnectionString", string.Empty);
this.bufferSize = args.Get("BufferSize", 1000);
this.bufferInterval = args.Get("BufferInterval", TimeSpan.FromMinutes(5));
this.addScript = args.Get("AddScript", string.Empty).Replace("@(", "$(");
if (string.IsNullOrWhiteSpace(this.conString))
{
throw new ArgumentException("SqlGeoFixArchiver: The [ConnectionString] argument is required.");
}
if (string.IsNullOrWhiteSpace(this.addScript))
{
throw new ArgumentException("SqlGeoFixArchiver: The [AddScript] argument is required.");
}
this.isRunning = true;
this.flushTimer = new PolledTimer(bufferInterval, false);
this.flushThread = new Thread(new ThreadStart(FlushThread));
this.flushThread.Start();
}
catch (Exception e)
{
SysLog.LogException(e);
}
}
}
/// <summary>
/// Constructs the instance and starts the background downloading thread.
/// </summary>
/// <param name="node">The parent <see cref="GeoTrackerNode" /> instance.</param>
/// <remarks>
/// <note>
/// The constructor will not start a background thread if IP geocoding is disabled.
/// </note>
/// </remarks>
public IPGeocoder(GeoTrackerNode node)
{
settings = node.Settings;
if (!settings.IPGeocodeEnabled)
{
return;
}
// Initialize the service including loading the MaxMind database if present.
running = true;
stopPending = false;
pollDataNow = false;
maxMind = null;
try
{
if (File.Exists(dataPath))
{
maxMind = new LookupService(dataPath, LookupService.GEOIP_MEMORY_CACHE);
maxMind.close();
}
}
catch (Exception e)
{
// Assume that the database file is corrupted if there's an exception
// and delete it so the download thread will download a new copy.
SysLog.LogException(e);
SysLog.LogError("GeoTracker: The MaxMind database file [{0}] appears to be corrupted. This will be deleted so the downloader can get a fresh copy.", dataPath);
Helper.DeleteFile(dataPath);
}
// Start the background downloader thread.
downloadThread = new Thread(new ThreadStart(DownloadThread));
downloadThread.Name = "GeoTracker: GeoData Downloader";
downloadThread.Start();
}
private List <EntityState> entities; // The entities within the block (or null)
/// <summary>
/// Constructor.
/// </summary>
/// <param name="settings">The GeoTracker server settings.</param>
/// <param name="bounds">The boundry for the area indexed by this instance.</param>
/// <param name="depth">The depth of of this instance in the hierarchy (0 for the top level).</param>
/// <remarks>
/// <note>
/// This constructor initially creates a leaf index block. Subsequent calls to <see cref="Balance" />
/// may split this block into sub-blocks or join sub-blocks back into a leaf node.
/// </note>
/// </remarks>
public MercatorBlock(GeoTrackerServerSettings settings, GeoRectangle bounds, int depth)
{
this.Bounds = bounds;
this.Depth = depth;
this.subBlocks = null;
this.entities = new List <EntityState>();
if (depth == 0)
{
edgeLength = 10.0;
}
else
{
edgeLength = 10.0 / (Math.Pow(BlockDimension, depth));
}
if (depth <= settings.IndexMaxGroupTableLevel)
{
groupMap = new Dictionary <string, bool>();
}
}
/// <summary>
/// Balances the entities within the block by creating sub-blocks if this is a leaf and the
/// number of entities exceeds the high watermark or collapsing sub-blocks back into a leaf
/// if the number of entities is less than the low watermark.
/// </summary>
public void Balance(GeoTrackerServerSettings settings)
{
}
/// <summary>
/// Associates the service handler with a message router by registering
/// the necessary application message handlers.
/// </summary>
/// <param name="router">The message router.</param>
/// <param name="keyPrefix">The configuration key prefix or (null to use <b>LillTek.GeoTracker.Server</b>).</param>
/// <param name="perfCounters">The application's performance counter set (or <c>null</c>).</param>
/// <param name="perfPrefix">The string to prefix any performance counter names (or <c>null</c>).</param>
/// <remarks>
/// <para>
/// Applications that expose performance counters will pass a non-<c>null</c> <b>perfCounters</b>
/// instance. The service handler should add any counters it implements to this set.
/// If <paramref name="perfPrefix" /> is not <c>null</c> then any counters added should prefix their
/// names with this parameter.
/// </para>
/// </remarks>
public void Start(MsgRouter router, string keyPrefix, PerfCounterSet perfCounters, string perfPrefix)
{
Start(router, GeoTrackerServerSettings.LoadConfig(keyPrefix ?? ConfigPrefix), perfCounters, perfPrefix);
}
