/// <summary>
/// Sends a batch of commands to the server and processes the results.
/// </summary>
private static void SendBatch()
{
// Create a new web client that will serve as the connection point to the server.
WebClient webClient = new WebClient();
webClient.Timeout = TableLoader.timeout;
// Call the web services to execute the command batch.
remoteBatch.Merge(webClient.Execute(remoteBatch));
// Any error during the batch will terminate the execution of the remaining records in the data file.
if (remoteBatch.HasExceptions)
{
// Display each error on the console.
foreach (RemoteException remoteException in remoteBatch.Exceptions)
{
Console.WriteLine(remoteException.Message);
}
// This will signal the error exit from this loader.
hasErrors = true;
}
// Clearing out the batch indicates that a new header should be created for the next set of commands.
remoteBatch = null;
remoteTransaction = null;
remoteAssembly = null;
remoteType = null;
}
/// <summary>
/// Execute a command batch and synchronizes the client data model with the newer records from the server.
/// </summary>
public static void Execute(RemoteBatch remoteBatch)
{
// This 'try' block will insure that the mutual exclusion locks are released.
try
{
// Maure sure only one thread at a time tries to refresh the data model.
ClientMarketData.refreshMutex.WaitOne();
// IMPORTANT CONCEPT: The rowVersion keeps track of the state of the client in-memory database. The server returns
// a value for us to use on the next cycle. If, for any reason, the merge of the data on the client should fail,
// we won't use the new value on the next cycle. That is, we're going to keep on asking for the same incremental
// set of records until we've successfully merged them. This guarantees that the client and server databases stay
// consistent with each other.
DateTime timeStamp = ClientMarketData.timeStamp;
long rowVersion = ClientMarketData.rowVersion;
// IMPORTANT CONCEPT: Executing the 'Reconcile' Web Service with a rowVersion returns to the client a DataSet with
// only records that are the same age or younger than the rowVersion. This reduces the traffic on the network to
// include only the essential records. This set also contains the deleted records. When the DataSet that is
// returned from this Web Service is merged with the current data, the client will be reconciled with the server as
// of the 'rowVersion' returned from the server.
DataSet resultSet;
WebClient webClient = new WebClient();
DataSet dataSet = webClient.ExecuteAndReconcile(ref timeStamp, ref rowVersion, remoteBatch, out resultSet);
remoteBatch.Merge(resultSet);
// If the time stamps are out of sync, then the server has reset since our last refresh (or this is the first time
// refreshing the data mdoel). This will reset the client side of the data model so that, after this refresh, the
// client and the server will be in sync again.
if (ClientMarketData.timeStamp != timeStamp)
{
ClientMarketData.rowVersion = 0;
ClientMarketData.timeStamp = timeStamp;
ClientMarketData.Clear();
}
// Optimization: Don't merge the results if there's nothing to merge.
if (rowVersion > ClientMarketData.rowVersion)
{
// IMPORTANT CONCEPT: This broadcast can be used to set up conditions for the data event handlers. Often,
// optimizing algorithms will be used to consolidate the results of a merge. This will allow the event driven
// logic to clear previous results and set up initial states for handling the bulk update of data.
ClientMarketData.OnBeginMerge(typeof(ClientMarketData));
// This 'try' block will insure that the locks are released if there's an error.
try
{
// IMPORTANT CONCEPT: Make sure that there aren't any nested locks. Table locks have to be aquired in the
// same order for every thread that uses the shared data model. It's possible that the current thread may
// have locked table 'B' if the preamble, then try to lock table 'A' during the processing of a command
// batch. If another thread aquires the lock on table 'A' and blocks on table 'B', we will have a
// deadlock. This is designed to catch situations where the 'Execute' method is called while tables are
// already locked.
Debug.Assert(!ClientMarketData.AreLocksHeld);
// IMPORTANT CONCEPT: Since the results will still be on the server if the client misses a refresh cycle,
// we take the attitude that this update process doesn't have to wait for locks. That is, if we can't get
// all the tables locked quickly, we'll just wait until the next refresh period to get the results. This
// effectively prevents deadlocking on the client. Make sure all the tables are locked before populating
// them.
foreach (TableLock tableLock in ClientMarketData.TableLocks)
{
tableLock.AcquireWriterLock(ClientTimeout.LockWait);
}
// IMPORANT CONCEPT: Once all the write locks have been obtained, we can merge the results. This will
// trigger the events associated with the tables for updated and deleted rows. Also, if
// "AcceptChanges" is invoked for a DataSet or a Table, every single record in the DataSet or DataTable
// will be "Committed", even though they are unchanged. This is a VERY inefficient operation. To get
// around this, an ArrayList of modified records is constructed during the Merge operation. After the
// merge, only the new records are committed.
ClientMarketData.mergedRows.Clear();
ClientMarketData.Merge(dataSet);
for (int index = 0; index < ClientMarketData.mergedRows.Count; index++)
{
((DataRow)ClientMarketData.mergedRows[index]).AcceptChanges();
}
// If the merge operation was successful, then we can use the new rowVersion for the next cycle. Any
// exception before this point will result in a request of the same set of data becaue the rowVersion was
// never updated.
ClientMarketData.rowVersion = rowVersion;
}
catch (ConstraintException)
{
// Write out the exact location of the error.
foreach (DataTable dataTable in ClientMarketData.Tables)
{
foreach (DataRow dataRow in dataTable.Rows)
{
if (dataRow.HasErrors)
{
Console.WriteLine("Error in '{0}': {1}", dataRow.Table.TableName, dataRow.RowError);
}
}
}
}
catch (Exception exception)
{
// Write the error and stack trace out to the debug listener
Debug.WriteLine(String.Format("{0}, {1}", exception.Message, exception.StackTrace));
}
finally
{
// No matter what happens above, we need to release the locks acquired above.
foreach (TableLock tableLock in ClientMarketData.TableLocks)
{
if (tableLock.IsWriterLockHeld)
{
tableLock.ReleaseWriterLock();
}
}
}
// IMPORTANT CONCEPT: When the merge is complete and the tables are unlocked, this will broadcast an event
// which allows optimization code to consolidate the results, examine the changed values and update reports
// based on the changed data.
ClientMarketData.OnEndMerge(typeof(ClientMarketData));
}
}
finally
{
// Other threads can now request a refresh of the data model.
ClientMarketData.refreshMutex.ReleaseMutex();
}
// Throw a specialized exception if the server returned any errors in the RemoteBatch structure.
if (remoteBatch.HasExceptions)
{
throw new BatchException(remoteBatch);
}
}
static int Main(string[] args)
{
// If this flag is set during the processing of the file, the program will exit with an error code.
bool hasErrors = false;
try
{
// Defaults
assemblyName = "Service.External";
namespaceName = "Shadows.WebService.External";
externalConfigurationCode = "DEFAULT";
argumentState = ArgumentState.FileName;
// Parse the command line for arguments.
foreach (string argument in args)
{
// Decode the current argument into a state.
if (argument == "-c")
{
argumentState = ArgumentState.ConfigurationCode; continue;
}
if (argument == "-i")
{
argumentState = ArgumentState.FileName; continue;
}
if (argument == "-id")
{
argumentState = ArgumentState.StylesheetId; continue;
}
if (argument == "-t")
{
argumentState = ArgumentState.StylesheetTypeCode; continue;
}
// The parsing state will determine which variable is read next.
switch (argumentState)
{
// Read the command line argument into the proper variable based on the parsing state.
case ArgumentState.ConfigurationCode: externalConfigurationCode = argument; break;
case ArgumentState.StylesheetTypeCode: stylesheetTypeCode = argument; break;
case ArgumentState.FileName: fileName = argument; break;
case ArgumentState.Name: name = argument; break;
case ArgumentState.StylesheetId: StylesheetId = argument; break;
}
// The default state for the parser is to look for a file name.
argumentState = ArgumentState.FileName;
}
// If no file name was specified, we return an error.
if (fileName == null)
{
throw new Exception("Usage: Loader.Stylesheet -i <FileName>");
}
// Load up an XML document with the contents of the file specified on the command line.
XmlDocument stylesheet = new XmlDocument();
stylesheet.Load(fileName);
// The XML document has several nodes that need to be read -- and then removed -- that contain attributes of the
// stylesheet. These nodes can be found easily using the XSL Path functions which need a namespace manager to sort
// out the tag prefixes.
XmlNamespaceManager namespaceManager = new XmlNamespaceManager(stylesheet.NameTable);
namespaceManager.AddNamespace("xsl", "http://www.w3.org/1999/XSL/Transform");
namespaceManager.AddNamespace("sss", "urn:schemas-shadows-com:shadows:stylesheet");
// The spreadhseet source has several nodes which contain information about how the data in the XML document should
// be loaded into the server, such as the stylesheet identifier, the name and the stylesheet style. They are found
// at this node. After these information nodes have been read, they are removed from the stylesheet source.
XmlNode stylesheetNode = stylesheet.SelectSingleNode("xsl:stylesheet", namespaceManager);
if (stylesheetNode == null)
{
throw new Exception("Syntax Error: missing stylesheet declaration.");
}
// Find the StylesheetId node.
XmlNode StylesheetIdNode = stylesheetNode.SelectSingleNode("sss:stylesheetId", namespaceManager);
if (StylesheetIdNode == null)
{
throw new Exception("Syntax Error: missing StylesheetId declaration.");
}
// Find the StylesheetStyle node.
XmlNode stylesheetTypeCodeNode = stylesheetNode.SelectSingleNode("sss:stylesheetTypeCode", namespaceManager);
if (stylesheetTypeCodeNode == null)
{
throw new Exception("Syntax Error: missing StylesheetStyle declaration.");
}
// Find the name node.
XmlNode nameNode = stylesheetNode.SelectSingleNode("sss:name", namespaceManager);
if (nameNode == null)
{
throw new Exception("Syntax Error: missing name declaration.");
}
// Extract the data from the XML nodes.
StylesheetId = StylesheetIdNode.InnerText;
stylesheetTypeCode = stylesheetTypeCodeNode.InnerText;
name = nameNode.InnerText;
// Remove the stylesheet nodes from the XSL spreadsheet before loading it into the server.
stylesheetNode.RemoveChild(StylesheetIdNode);
stylesheetNode.RemoveChild(stylesheetTypeCodeNode);
stylesheetNode.RemoveChild(nameNode);
// Create a command to load the stylesheet from the data loaded from the file.
RemoteBatch remoteBatch = new RemoteBatch();
RemoteAssembly remoteAssembly = remoteBatch.Assemblies.Add(assemblyName);
RemoteType remoteType = remoteAssembly.Types.Add(string.Format("{0}.{1}", namespaceName, "Stylesheet"));
RemoteMethod remoteMethod = remoteType.Methods.Add("Load");
remoteMethod.Parameters.Add("StylesheetId", StylesheetId);
remoteMethod.Parameters.Add("stylesheetTypeCode", stylesheetTypeCode);
remoteMethod.Parameters.Add("name", name);
remoteMethod.Parameters.Add("text", stylesheet.InnerXml);
// Create a new web client that will serve as the connection point to the server and call the web services to
// execute the command batch.
WebClient webClient = new WebClient();
remoteBatch.Merge(webClient.Execute(remoteBatch));
// Display the each of the exceptions and set a global flag that shows that there was an exception to the normal
// execution.
if (remoteBatch.HasExceptions)
{
foreach (RemoteException exception in remoteBatch.Exceptions)
{
Console.WriteLine(String.Format("{0}: {1}", remoteMethod.Parameters["StylesheetId"].Value, exception.Message));
}
hasErrors = true;
}
}
catch (Exception exception)
{
// Show the system error and exit with an error.
Console.WriteLine(exception.Message);
hasErrors = true;
}
// Any errors will cause an abnormal exit.
if (hasErrors)
{
return(1);
}
// Display the template that was loaded and exit with a successful code.
Console.WriteLine(String.Format("{0} Stylesheet: {1}, Loaded", DateTime.Now.ToString("u"), name));
return(0);
}
static int Main(string[] args)
{
// If this flag is set during the processing of the file, the program will exit with an error code.
bool hasErrors = false;
try
{
// Defaults
batchSize = 100;
assemblyName = "Service.External";
nameSpaceName = "Shadows.WebService.External";
// The command line parser is driven by different states that are triggered by the flags read. Unless a flag has been
// read, the command line parser assumes that it's reading the file name from the command line.
argumentState = ArgumentState.FileName;
// Parse the command line for arguments.
foreach (string argument in args)
{
// Decode the current argument into a state change (or some other action).
if (argument == "-a")
{
argumentState = ArgumentState.Assembly; continue;
}
if (argument == "-b")
{
argumentState = ArgumentState.BatchSize; continue;
}
if (argument == "-n")
{
argumentState = ArgumentState.NameSpace; continue;
}
if (argument == "-i")
{
argumentState = ArgumentState.FileName; continue;
}
// The parsing state will determine which variable is read next.
switch (argumentState)
{
case ArgumentState.Assembly: assemblyName = argument; break;
case ArgumentState.BatchSize: batchSize = Convert.ToInt32(argument); break;
case ArgumentState.FileName: fileName = argument; break;
case ArgumentState.NameSpace: nameSpaceName = argument; break;
}
// The default state is to look for the input file name on the command line.
argumentState = ArgumentState.FileName;
}
// Throw a usage message back at the user if no file name was given.
if (fileName == null)
{
throw new Exception("Usage: Loader.Algorithm -i <FileName>");
}
// Open up the file containing all the broker.
BrokerReader brokerReader = new BrokerReader(fileName);
// Create a new web client that will serve as the connection point to the server.
WebClient webClient = new WebClient();
// Loading the database involves creating a batch of commands and sending them off as a transaction. This gives
// the server a chance to pipeline a large chunk of processing, without completely locking up the server for the
// entire set of data. This will construct a header for the command batch which gives information about which
// assembly contains the class that is used to load the data.
RemoteBatch remoteBatch = new RemoteBatch();
RemoteTransaction remoteTransaction = remoteBatch.Transactions.Add();
RemoteAssembly remoteAssembly = remoteBatch.Assemblies.Add(assemblyName);
RemoteType remoteType = remoteAssembly.Types.Add(string.Format("{0}.{1}", nameSpaceName, "Broker"));
// Read the file until an EOF is reached.
while (true)
{
// This counter keeps track of the number of records sent. When the batch is full, it's sent to the server to be
// executed as a single transaction.
int batchCounter = 0;
// Read the next broker from the input stream. A 'null' is returned when we've read past the end of file.
Broker broker = brokerReader.ReadBroker();
if (broker != null)
{
// Construct a call to the 'Load' method to populate the broker record.
RemoteMethod remoteMethod = remoteType.Methods.Add("Load");
remoteMethod.Transaction = remoteTransaction;
remoteMethod.Parameters.Add("brokerId", broker.Symbol);
remoteMethod.Parameters.Add("name", broker.Name);
remoteMethod.Parameters.Add("symbol", broker.Symbol);
}
// This will check to see if it's time to send the batch. A batch is sent when the 'batchSize' has been
// reached, or if the last record has just been converted into a command.
if (++batchCounter % batchSize == 0 || broker == null)
{
// Call the web services to execute the command batch.
remoteBatch.Merge(webClient.Execute(remoteBatch));
// Any error during the batch will terminate the execution of the remaining records in the data file.
if (remoteBatch.HasExceptions)
{
// Display each error on the console.
foreach (RemoteMethod remoteMethod in remoteBatch.Methods)
{
foreach (RemoteException remoteException in remoteMethod.Exceptions)
{
Console.WriteLine(String.Format("{0}: {1}", remoteMethod.Parameters["brokerId"].Value,
remoteException.Message));
}
}
// This will signal the error exit from this loader.
hasErrors = true;
}
// If the end of file was reached, break out of the loop and exit the application.
if (broker == null)
{
break;
}
// After each batch has been send, check to see if there are additional records to be send. If so,
// regenerate the remote batch and set up the header.
remoteBatch = new RemoteBatch();
remoteTransaction = remoteBatch.Transactions.Add();
remoteAssembly = remoteBatch.Assemblies.Add(assemblyName);
remoteType = remoteAssembly.Types.Add(string.Format("{0}.{1}", nameSpaceName, "Broker"));
}
}
}
catch (Exception exception)
{
// Show the system error and exit with an error.
Console.WriteLine(exception.Message);
hasErrors = true;
}
// Any errors will cause an abnormal exit.
if (hasErrors)
{
return(1);
}
// Write a status message when a the file is loaded successfully.
Console.WriteLine(String.Format("{0} Data: Brokers, Loaded", DateTime.Now.ToString("u")));
// If we reached here, the file was imported without issue.
return(0);
}
