public void TransferPackageFile(StitchGroupName groupName, string toNodeId, string filePath, string fileName, InstanceAdaptorDetails adaptor, string jobId, string taskId)
{
// TODO: More validation and error handling
// TODO: We need to get more sophisticated about this, such as doing the transfer in chunks and allowing restarts
if (!groupName.IsValid() || !groupName.IsVersionGroup())
{
throw new Exception("Must use a valid version name for a package upload file");
}
if (adaptor == null)
{
throw new Exception("Adaptor details must be provided");
}
var bytes = File.ReadAllBytes(filePath);
var envelope = new FileTransferEnvelope
{
Contents = bytes,
GroupName = groupName.VersionString,
JobId = jobId,
TaskId = taskId,
PacketNumber = 1,
TotalNumberOfPackets = 1,
FileName = fileName,
Adaptor = adaptor
};
var message = new ClusterMessageBuilder()
.ToNode(toNodeId)
.FromNode()
.WithInternalObjectPayload(envelope)
.Build();
Send(message);
}
public void AddRemoteStitch(string nodeId, string networkNodeId, string id, StitchGroupName groupName)
{
var summary = new StitchSummary
{
GroupName = groupName,
Id = id,
Locale = StitchLocaleType.Remote,
NodeId = nodeId,
NetworkNodeId = networkNodeId
};
if (!_remoteStitches.ContainsKey(nodeId))
{
_remoteStitches.Add(nodeId, new List <StitchSummary>
{
summary
});
return;
}
var summaries = _remoteStitches[nodeId]
.Where(ss => ss.Id != id)
.Concat(new[] { summary })
.ToList();
_remoteStitches[nodeId] = summaries;
_allStitches = null;
}
public CommandResponse Handle(CommandRequest request)
{
if (string.IsNullOrEmpty(request.Target))
{
return(CommandResponse.Create(false));
}
var groupName = new StitchGroupName(request.Target);
var job = _jobManager.CreateJob("Command=" + request.Command);
var stitches = _data.GetStitchesInGroup(groupName);
foreach (var stitch in stitches)
{
var subtask = job.CreateSubtask(CommandType, stitch.Id, _nodeId);
if (stitch.NodeId == _nodeId)
{
SendLocal(job, subtask, stitch);
}
else
{
SendRemote(job, subtask, stitch);
}
}
// Save the job to get an Id
_jobManager.Save(job);
return(CommandResponse.Started(job.Id));
}
public PackageFileUploadResponse UploadStitchPackageFile(PackageFileUploadRequest request)
{
if (!request.IsValidLocalRequest())
{
return(new PackageFileUploadResponse(false, null, null));
}
// Save the file and generate a unique Version name
var result = _fileSystem.SavePackageToLibrary(request.GroupName.Application, request.GroupName.Component, request.Contents);
var groupName = new StitchGroupName(request.GroupName.Application, request.GroupName.Component, result.Version);
var packageFile = new PackageFile
{
Id = groupName.ToString(),
Adaptor = request.Adaptor,
FileName = request.FileName,
GroupName = groupName
};
bool ok = _data.Save(packageFile, true);
if (!ok)
{
_log.LogError("Could not save PackageFile Id={0}", groupName.ToString());
}
_log.LogDebug("Uploaded package file {0}", groupName);
return(new PackageFileUploadResponse(ok, groupName, result.FilePath));
}
public Result UnzipLibraryPackageToRunningBase(StitchGroupName groupName, string instanceId)
{
string libraryDirectoryPath = Path.Combine(_config.AppLibraryBasePath, groupName.Application, groupName.Component);
if (!Directory.Exists(libraryDirectoryPath))
{
return(Result.Failure());
}
string libraryFilePath = Path.Combine(libraryDirectoryPath, groupName.Version + ".zip");
var runningDirectory = GetInstanceRunningDirectory(instanceId);
if (Directory.Exists(runningDirectory))
{
Directory.Delete(runningDirectory, true);
}
Directory.CreateDirectory(runningDirectory);
using (FileStream fileStream = File.Open(libraryFilePath, FileMode.Open, FileAccess.Read))
{
using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Read, true))
{
archive.ExtractToDirectory(runningDirectory);
}
}
return(new Result
{
Success = true,
Path = runningDirectory
});
}
public void SendPackageFile(string networkNodeId, StitchGroupName groupName, string fileName, string filePath, InstanceAdaptorDetails adaptor, string jobId, string taskId)
{
_messageBus.Publish(new FileTransferRequest
{
FilePath = filePath,
NetworkNodeId = networkNodeId,
GroupName = groupName,
JobId = jobId,
TaskId = taskId,
FileName = fileName,
Adaptor = adaptor
});
//var message = new ClusterMessageBuilder()
// .FromNode()
// .ToNode(networkNodeId)
// .WithObjectPayload(new FileTransferRequest
// {
// FilePath = filePath,
// GroupName = groupName,
// JobId = jobId,
// TaskId = taskId,
// FileName = fileName
// })
// .Build();
//_messageBus.Publish(ClusterMessage.SendEventName, message);
}
private static void StartBuiltinStitch(CrossStitchCore core)
{
var group3 = new StitchGroupName("StitchStart", "BuiltIn", "1");
var packageResult3 = core.MessageBus.RequestWait <DataRequest <PackageFile>, DataResponse <PackageFile> >(DataRequest <PackageFile> .Save(new PackageFile
{
Id = group3.ToString(),
GroupName = group3,
Adaptor = new InstanceAdaptorDetails
{
Type = AdaptorType.BuildInClassV1,
Parameters = new Dictionary <string, string>
{
{ CrossStitch.Stitch.BuiltInClassV1.Parameters.TypeName, typeof(StitchStartBuiltInStitch).AssemblyQualifiedName }
}
},
}, true));
var createResult3 = core.MessageBus.RequestWait <LocalCreateInstanceRequest, LocalCreateInstanceResponse>(new LocalCreateInstanceRequest
{
Name = "StitchStart.BuiltIn",
GroupName = group3,
NumberOfInstances = 1
});
core.MessageBus.RequestWait <InstanceRequest, InstanceResponse>(InstanceRequest.ChannelStart, new InstanceRequest
{
Id = createResult3.CreatedIds.FirstOrDefault()
});
}
private IEnumerable <StitchDataMessage> AddressApplicationMessage(StitchDataMessage message)
{
var messages = new List <StitchDataMessage>();
var groupName = new StitchGroupName(message.ToStitchGroup);
var stitches = _stitches
.Where(si => groupName.Contains(si.GroupName))
.Where(si => si.Id != message.FromStitchInstanceId)
.ToList();
foreach (var stitch in stitches)
{
messages.Add(new StitchDataMessage
{
// Leave the ToNodeID and NetworkID empty, so it gets routed locally
Data = message.Data,
ToStitchInstanceId = stitch.Id,
FromNetworkId = message.FromNetworkId,
FromNodeId = message.FromNodeId,
FromStitchInstanceId = message.FromStitchInstanceId,
ToNetworkId = stitch.NetworkNodeId,
ToNodeId = stitch.NodeId
});
}
return(messages);
}
private static void StartProcessStitch(CrossStitchCore core, string name, MessageChannelType channelType)
{
var group1 = new StitchGroupName("StitchStart", name, "1");
var packageResult1 = core.MessageBus.RequestWait <DataRequest <PackageFile>, DataResponse <PackageFile> >(DataRequest <PackageFile> .Save(new PackageFile
{
Id = group1.ToString(),
GroupName = group1,
Adaptor = new InstanceAdaptorDetails
{
Type = AdaptorType.ProcessV1,
Parameters = new Dictionary <string, string>
{
{ Parameters.RunningDirectory, "." },
{ Parameters.ExecutableName, "StitchStart.Client.exe" }
},
RequiresPackageUnzip = false,
Channel = channelType
}
}, true));
var createResult1 = core.MessageBus.RequestWait <LocalCreateInstanceRequest, LocalCreateInstanceResponse>(new LocalCreateInstanceRequest
{
Name = "StitchStart." + name,
GroupName = group1,
NumberOfInstances = 1,
});
core.MessageBus.RequestWait <InstanceRequest, InstanceResponse>(InstanceRequest.ChannelStart, new InstanceRequest
{
Id = createResult1.CreatedIds.FirstOrDefault()
});
}
public void ConstructFromParts_Version_Test()
{
var target = new StitchGroupName("A", "B", "C");
target.Application.Should().Be("A");
target.Component.Should().Be("B");
target.Version.Should().Be("C");
}
public void StitchDeleted(string id, StitchGroupName groupName)
{
_messageBus.Publish(StitchInstanceEvent.ChannelCreated, new StitchInstanceEvent
{
InstanceId = id,
GroupName = groupName
});
}
public void ConstructFromGroupName_Application_Test()
{
var target = new StitchGroupName("A");
target.Application.Should().Be("A");
target.Component.Should().BeNull();
target.Version.Should().BeNull();
}
public void ToString_Test()
{
var target = new StitchGroupName("A.B.C");
target.ToString().Should().Be("A.B.C");
target = new StitchGroupName("A", "B", "C");
target.ToString().Should().Be("A.B.C");
}
public void IsValid_Test()
{
var target = new StitchGroupName()
{
VersionString = null
};
target.IsValid().Should().BeFalse();
}
private static void TestStep1(IMessageBus messageBus, string serverBNodeId, string serverBNetworkId)
{
Thread.Sleep(5000);
_testLog.LogInformation("ServerB has joined the cluster.");
// Zip up the Stitch.js file
_testLog.LogInformation("Zipping the stitch");
var stream = new MemoryStream();
using (var zip = new ZipArchive(stream, ZipArchiveMode.Create, true))
{
var entry = zip.CreateEntry("Stitch.js");
using (var entryStream = entry.Open())
using (var writer = new StreamWriter(entryStream))
{
writer.Write(File.ReadAllText(".\\Stitch.js"));
}
}
stream.Seek(0, SeekOrigin.Begin);
// Subscribe to the Job complete event, so we can move to the next step as soon as it is ready
messageBus.Subscribe <JobCompleteEvent>(b => b
.WithTopic(JobCompleteEvent.ChannelSuccess)
.Invoke(m => TestStep2(messageBus, m))
.OnThreadPool()
.MaximumEvents(1));
// "Upload" the stitch file to ServerA, which will broadcast to ServerB
_testLog.LogInformation("Uploading the Stitch.zip package file");
var response = messageBus.RequestWait <PackageFileUploadRequest, PackageFileUploadResponse>(new PackageFileUploadRequest
{
Contents = stream,
FileName = "Stitch.zip",
GroupName = new StitchGroupName("StitchIntegration.Stitch"),
LocalOnly = false,
Adaptor = new InstanceAdaptorDetails
{
RequiresPackageUnzip = true,
Type = AdaptorType.ProcessV1,
Parameters = new Dictionary <string, string>
{
{ Parameters.ArgumentsFormat, "{ExecutableName} {CoreArgs} -- {CustomArgs}" },
{ Parameters.ExecutableFormat, "C:\\Program Files\\nodejs\\node.exe" },
{ Parameters.ExecutableName, "Stitch.js" }
}
},
});
_groupName = response.GroupName;
_testLog.LogInformation("Uploaded version {0}", _groupName);
if (!response.IsSuccess)
{
_testLog.LogError("Could not upload package file");
}
}
static void Main(string[] args)
{
var nodeConfig = NodeConfiguration.GetDefault();
using (var core = new CrossStitchCore(nodeConfig))
{
// Setup an in-memory data store
var store = new InMemoryDataStorage();
core.AddModule(new DataModule(core.MessageBus, store));
// Backplane so we can cluster
core.AddModule(new BackplaneModule(core));
// Stitches so we can host the stitch instances
var stitchesConfig = StitchesConfiguration.GetDefault();
core.AddModule(new StitchesModule(core, stitchesConfig));
// Setup logging.
var logger = new LoggerFactory().AddConsole(LogLevel.Debug).CreateLogger <Program>();
core.AddModule(new LoggingModule(core, logger));
// Create a stitch instance to run on startup
var groupName = new StitchGroupName("PenPal.StitchA.1");
var packageFile = new PackageFile
{
Id = groupName.ToString(),
GroupName = groupName,
Adaptor = new InstanceAdaptorDetails
{
Type = AdaptorType.ProcessV1,
Parameters = new Dictionary <string, string>
{
{ Parameters.RunningDirectory, "." },
{ Parameters.ExecutableArguments, "" },
{ Parameters.ExecutableName, "PenPal.StitchA.exe" }
}
},
};
store.Save(packageFile, true);
var stitch = new StitchInstance
{
Name = "StitchA",
GroupName = groupName,
OwnerNodeName = core.Name,
OwnerNodeId = core.NodeId,
State = InstanceStateType.Running
};
store.Save(stitch, true);
core.Start();
Console.ReadKey();
core.Stop();
}
}
static void Main(string[] args)
{
var nodeConfig = NodeConfiguration.GetDefault();
using (var core = new CrossStitchCore(nodeConfig))
{
var httpServer = new NancyHttpModule(core.MessageBus);
core.AddModule(httpServer);
var dataStorage = new InMemoryDataStorage();
var data = new DataModule(core.MessageBus, dataStorage);
core.AddModule(data);
var stitchesConfiguration = StitchesConfiguration.GetDefault();
var stitches = new StitchesModule(core, stitchesConfiguration);
core.AddModule(stitches);
var groupName = new StitchGroupName("HttpTest", "Stitch", "1");
var packageFile = new PackageFile
{
Id = groupName.ToString(),
GroupName = groupName,
Adaptor = new InstanceAdaptorDetails
{
Type = AdaptorType.ProcessV1,
Parameters = new Dictionary <string, string>
{
{ Parameters.RunningDirectory, "." },
{ Parameters.ExecutableName, "HttpTest.Stitch.exe" },
//{ Parameters.ArgumentsFormat, "{ExecutableName} {CoreArgs} -- {CustomArgs}" },
//{ Parameters.ExecutableFormat, "dotnet" },
}
},
};
dataStorage.Save(packageFile, true);
var stitch = new StitchInstance
{
Name = "HttpTest.Stitch",
GroupName = groupName,
State = InstanceStateType.Running
};
dataStorage.Save(stitch, true);
var logger = new LoggerFactory().AddConsole(LogLevel.Debug).CreateLogger <Program>();
core.AddModule(new LoggingModule(core, logger));
core.Log.LogInformation("Started");
core.Start();
Console.ReadKey();
core.Stop();
}
}
static void Main(string[] args)
{
var nodeConfig = NodeConfiguration.GetDefault();
using (var core = new CrossStitchCore(nodeConfig))
{
var httpServer = new NancyHttpModule(core.MessageBus);
core.AddModule(httpServer);
var dataStorage = new InMemoryDataStorage();
var data = new DataModule(core.MessageBus, dataStorage);
core.AddModule(data);
var stitchesConfiguration = StitchesConfiguration.GetDefault();
var stitches = new StitchesModule(core, stitchesConfiguration);
core.AddModule(stitches);
var groupName = new StitchGroupName("HttpTest", "Stitch", "1");
var packageFile = new PackageFile
{
Id = groupName.ToString(),
GroupName = groupName,
Adaptor = new InstanceAdaptorDetails
{
Type = AdaptorType.ProcessV1,
Parameters = new Dictionary <string, string>
{
{ CrossStitch.Stitch.ProcessV1.Parameters.DirectoryPath, "." },
{ CrossStitch.Stitch.ProcessV1.Parameters.ExecutableName, "HttpTest.Stitch.exe" }
}
},
};
dataStorage.Save(packageFile, true);
var stitch = new StitchInstance
{
Name = "HttpTest.Stitch",
GroupName = groupName,
State = InstanceStateType.Running,
LastHeartbeatReceived = 0
};
dataStorage.Save(stitch, true);
core.AddModule(new LoggingModule(core, Common.Logging.LogManager.GetLogger("CrossStitch")));
core.Start();
Console.ReadKey();
core.Stop();
}
}
static void Main(string[] args)
{
var config = NodeConfiguration.GetDefault();
using (var core = new CrossStitchCore(config))
{
var dataStorage = new InMemoryDataStorage();
var groupName = new StitchGroupName("JsStitch", "Stitch", "1");
dataStorage.Save(new PackageFile
{
Id = groupName.ToString(),
GroupName = groupName,
Adaptor = new InstanceAdaptorDetails
{
Type = AdaptorType.ProcessV1,
Parameters = new Dictionary <string, string>
{
{ Parameters.RunningDirectory, "." },
{ Parameters.ExecutableName, "JsStitch.Stitch.js" }
}
},
}, true);
dataStorage.Save(new StitchInstance
{
Name = "JsStitch.Stitch",
GroupName = groupName,
State = InstanceStateType.Running
}, true);
var data = new DataModule(core.MessageBus, dataStorage);
core.AddModule(data);
var stitchesConfiguration = StitchesConfiguration.GetDefault();
var stitches = new StitchesModule(core, stitchesConfiguration);
core.AddModule(stitches);
var logger = new LoggerFactory().AddConsole(LogLevel.Debug).CreateLogger <Program>();
core.AddModule(new LoggingModule(core, logger));
core.Start();
Console.ReadKey();
core.Stop();
}
}
public void AddLocalStitch(string id, StitchGroupName groupName)
{
var locals = _localStitches
.Where(si => si.Id != id)
.Concat(new[]
{
new StitchSummary
{
Id = id,
GroupName = groupName,
Locale = StitchLocaleType.Local,
NodeId = _nodeId
}
})
.ToList();
_localStitches = locals;
_allStitches = null;
}
public List <StitchSummary> GetStitchSummaries(StitchSummaryRequest request)
{
IEnumerable <StitchSummary> query = _data.GetAllStitchSummaries();
if (!string.IsNullOrEmpty(request.NodeId))
{
query = query.Where(ss => ss.NodeId == request.NodeId);
}
if (!string.IsNullOrEmpty(request.StitchGroupName))
{
var name = new StitchGroupName(request.StitchGroupName);
query = query.Where(ss => name.Contains(ss.GroupName));
}
if (!string.IsNullOrEmpty(request.StitchId))
{
query = query.Where(ss => ss.Id == request.StitchId);
}
return(query.ToList());
}
public bool Contains(StitchGroupName otherGroup)
{
if (Application != otherGroup.Application)
{
return(false);
}
if (IsApplicationGroup())
{
return(true);
}
if (Component != otherGroup.Component)
{
return(false);
}
if (IsComponentGroup())
{
return(true);
}
return(Version == otherGroup.Version);
}
public PackageFileUploadResponse UploadStitchPackageFile(StitchGroupName groupName, string filePath, PackageFileUploadRequest request)
{
// Send this to all nodes which are running the Stitches module
var nodes = _data.GetAll <NodeStatus>()
.Where(n => n.Id != _core.NodeId)
.Where(ns => ns.RunningModules.Contains(ModuleNames.Stitches))
.ToList();
if (nodes.Count == 0)
{
return(new PackageFileUploadResponse(true, groupName, filePath));
}
var job = _jobManager.CreateJob("Command=PackageFileUpload");
foreach (var node in nodes)
{
var task = job.CreateSubtask(CommandType.UploadPackageFile, node.Id, node.Id);
_clusterSender.SendPackageFile(node.NetworkNodeId, groupName, request.FileName, filePath, request.Adaptor, job.Id, task.Id);
}
_jobManager.Save(job);
return(new PackageFileUploadResponse(true, groupName, filePath, job.Id));
}
static void Main(string[] args)
{
var config = NodeConfiguration.GetDefault();
using (var core = new CrossStitchCore(config))
{
var stitchesConfiguration = StitchesConfiguration.GetDefault();
var stitches = new StitchesModule(core, stitchesConfiguration);
core.AddModule(stitches);
var log = Common.Logging.LogManager.GetLogger("CrossStitch");
var logging = new LoggingModule(core, log);
core.AddModule(logging);
core.Start();
// First stitch is a processV1
var group1 = new StitchGroupName("StitchStart", "Client", "1");
var packageResult1 = core.MessageBus.Request <DataRequest <PackageFile>, DataResponse <PackageFile> >(DataRequest <PackageFile> .Save(new PackageFile
{
Id = group1.ToString(),
GroupName = group1,
Adaptor = new InstanceAdaptorDetails
{
Type = AdaptorType.ProcessV1,
Parameters = new Dictionary <string, string>
{
{ Parameters.DirectoryPath, "." },
{ Parameters.ExecutableName, "StitchStart.Client.exe" }
},
RequiresPackageUnzip = false
}
}, true));
var createResult1 = core.MessageBus.Request <LocalCreateInstanceRequest, LocalCreateInstanceResponse>(new LocalCreateInstanceRequest
{
Name = "StitchStart.Client",
GroupName = group1,
NumberOfInstances = 1,
});
core.MessageBus.Request <InstanceRequest, InstanceResponse>(InstanceRequest.ChannelStart, new InstanceRequest
{
Id = createResult1.CreatedIds.FirstOrDefault()
});
// Second stitch is a built-in class
var group2 = new StitchGroupName("StitchStart", "BuiltIn", "1");
var packageResult2 = core.MessageBus.Request <DataRequest <PackageFile>, DataResponse <PackageFile> >(DataRequest <PackageFile> .Save(new PackageFile
{
Id = group2.ToString(),
GroupName = group2,
Adaptor = new InstanceAdaptorDetails
{
Type = AdaptorType.BuildInClassV1,
Parameters = new Dictionary <string, string>
{
{ CrossStitch.Stitch.BuiltInClassV1.Parameters.TypeName, typeof(StitchStartBuiltInStitch).AssemblyQualifiedName }
}
},
}, true));
var createResult2 = core.MessageBus.Request <LocalCreateInstanceRequest, LocalCreateInstanceResponse>(new LocalCreateInstanceRequest
{
Name = "StitchStart.BuiltIn",
GroupName = group2,
NumberOfInstances = 1
});
core.MessageBus.Request <InstanceRequest, InstanceResponse>(InstanceRequest.ChannelStart, new InstanceRequest
{
Id = createResult2.CreatedIds.FirstOrDefault()
});
Console.ReadKey();
core.Stop();
}
}
static void Main(string[] args)
{
var config = NodeConfiguration.GetDefault();
using (var core = new CrossStitchCore(config))
{
var dataStorage = new InMemoryDataStorage();
var pingGroup = new StitchGroupName("PingPong", "Ping", "1");
var pingPackage = new PackageFile
{
Id = pingGroup.ToString(),
GroupName = pingGroup,
Adaptor = new InstanceAdaptorDetails
{
Type = AdaptorType.ProcessV1,
Parameters = new Dictionary <string, string>
{
{ Parameters.RunningDirectory, "." },
{ Parameters.ExecutableName, "PingPong.Ping.exe" }
}
},
};
var ping = new StitchInstance
{
Name = "PingPong.Ping",
GroupName = pingGroup,
State = InstanceStateType.Running
};
var pongGroup = new StitchGroupName("PingPong", "Pong", "1");
var pongPackage = new PackageFile
{
Id = pongGroup.ToString(),
GroupName = pongGroup,
Adaptor = new InstanceAdaptorDetails
{
Type = AdaptorType.ProcessV1,
Parameters = new Dictionary <string, string>
{
{ Parameters.RunningDirectory, "." },
{ Parameters.ExecutableName, "PingPong.Pong.exe" }
}
},
};
var pong = new StitchInstance
{
Name = "PingPong.Pong",
GroupName = pongGroup,
State = InstanceStateType.Running
};
dataStorage.Save(pingPackage, true);
dataStorage.Save(ping, true);
dataStorage.Save(pongPackage, true);
dataStorage.Save(pong, true);
var data = new DataModule(core.MessageBus, dataStorage);
core.AddModule(data);
var stitchesConfiguration = StitchesConfiguration.GetDefault();
var stitches = new StitchesModule(core, stitchesConfiguration);
core.AddModule(stitches);
var logger = new LoggerFactory().AddConsole(LogLevel.Debug).CreateLogger <Program>();
core.AddModule(new LoggingModule(core, logger));
// TODO: We need a way to start for initialization to complete, either having Start
// block or providing an Initialized event which waits for all modules to report
// being initialized
core.Start();
Console.ReadKey();
core.Stop();
}
}
public PackageFileUploadResponse(bool isSuccess, StitchGroupName groupName, string filePath, string jobId = null)
{
IsSuccess = isSuccess;
GroupName = groupName;
FilePath = filePath;
}
public List <StitchSummary> GetStitchesInGroup(StitchGroupName group)
{
// TODO: Is there ever a case where we fall back to the data module?
return(StitchCache.GetStitchSummaries().Where(si => group.Contains(si.GroupName)).ToList());
}