private SysRandom(Random rnd)
{
Contracts.AssertValue(rnd);
_rnd = rnd;
}
/// <summary>
/// Make sure the given assemblies are loaded and that their loadable classes have been catalogued.
/// </summary>
public static void LoadAndRegister(IHostEnvironment env, string[] assemblies)
{
Contracts.AssertValue(env);
if (Utils.Size(assemblies) > 0)
{
foreach (string path in assemblies)
{
Exception ex = null;
try
{
// REVIEW: Will LoadFrom ever return null?
Contracts.CheckNonEmpty(path, nameof(path));
var assem = LoadAssembly(env, path);
if (assem != null)
{
continue;
}
}
catch (Exception e)
{
ex = e;
}
// If it is a zip file, load it that way.
ZipArchive zip;
try
{
zip = ZipFile.OpenRead(path);
}
catch (Exception e)
{
// Couldn't load as an assembly and not a zip, so warn the user.
ex = ex ?? e;
Console.Error.WriteLine("Warning: Could not load '{0}': {1}", path, ex.Message);
continue;
}
string dir;
try
{
dir = CreateTempDirectory();
}
catch (Exception e)
{
throw Contracts.ExceptIO(e, "Creating temp directory for extra assembly zip extraction failed: '{0}'", path);
}
try
{
zip.ExtractToDirectory(dir);
}
catch (Exception e)
{
throw Contracts.ExceptIO(e, "Extracting extra assembly zip failed: '{0}'", path);
}
LoadAssembliesInDir(env, dir, false);
}
}
}
/// <summary>
/// Create an output "file" and return a handle to it.
/// </summary>
public static IFileHandle CreateOutputFile(this IHostEnvironment env, string path)
{
Contracts.AssertValue(env);
Contracts.CheckNonWhiteSpace(path, nameof(path));
return(new SimpleFileHandle(env, path, needsWrite: true, autoDelete: false));
}
/// <summary>
/// Query all progress and:
/// * If there's any checkpoint/start/stop event, print all of them.
/// * If there's none, print a dot.
/// * If there's <see cref="_maxDots"/> dots, print the current status for all running calculations.
/// </summary>
public void GetAndPrintAllProgress(ProgressReporting.ProgressTracker progressTracker)
{
Contracts.AssertValue(progressTracker);
var entries = progressTracker.GetAllProgress();
if (entries.Count == 0)
{
// There's no calculation running. Don't even print a dot.
return;
}
var checkpoints = entries.Where(
x => x.Kind != ProgressReporting.ProgressEvent.EventKind.Progress || x.ProgressEntry.IsCheckpoint);
lock (_lock)
{
bool anyCheckpoint = false;
foreach (var ev in checkpoints)
{
anyCheckpoint = true;
EnsureNewLine();
// We assume that things like status counters, which contain only things
// like loss function values, counts of rows, counts of items, etc., are
// not sensitive.
WriteAndReturnLinePrefix(MessageSensitivity.None, _out);
switch (ev.Kind)
{
case ProgressReporting.ProgressEvent.EventKind.Start:
PrintOperationStart(_out, ev);
break;
case ProgressReporting.ProgressEvent.EventKind.Stop:
PrintOperationStop(_out, ev);
break;
case ProgressReporting.ProgressEvent.EventKind.Progress:
_out.Write("[{0}] ", ev.Index);
PrintProgressLine(_out, ev);
break;
}
}
if (anyCheckpoint)
{
// At least one checkpoint has been printed, so there's no need for dots.
return;
}
if (PrintDot())
{
// We need to print an extended status line. At this point, every event should be
// a non-checkpoint progress event.
bool needPrepend = entries.Count > 1;
foreach (var ev in entries)
{
Contracts.Assert(ev.Kind == ProgressReporting.ProgressEvent.EventKind.Progress);
Contracts.Assert(!ev.ProgressEntry.IsCheckpoint);
if (needPrepend)
{
EnsureNewLine();
WriteAndReturnLinePrefix(MessageSensitivity.None, _out);
_out.Write("[{0}] ", ev.Index);
}
else
{
// This is the only case we are printing something at the end of the line of dots.
// So, we need to reset the dots counter.
_dots = 0;
}
PrintProgressLine(_out, ev);
}
}
}
}
internal ModelOperationsCatalog(IHostEnvironment env)
{
Contracts.AssertValue(env);
Environment = env;
}
/// <summary>
/// This loads assemblies that are in our "root" directory (where this assembly is) and caches
/// information for the loadable classes in loaded assemblies.
/// </summary>
private static void CacheLoadedAssemblies()
{
// The target assembly is the one containing LoadableClassAttributeBase. If an assembly doesn't reference
// the target, then we don't want to scan its assembly attributes (there's no point in doing so).
var target = typeof(LoadableClassAttributeBase).Assembly;
lock (_lock)
{
if (_assemblyQueue == null)
{
// Create the loaded assembly queue and dictionary, set up the AssemblyLoad / AssemblyResolve
// event handlers and populate the queue / dictionary with all assemblies that are currently loaded.
Contracts.Assert(_assemblyQueue == null);
Contracts.Assert(_loadedAssemblies == null);
_assemblyQueue = new ConcurrentQueue <Assembly>();
_loadedAssemblies = new ConcurrentDictionary <string, Assembly>();
AppDomain.CurrentDomain.AssemblyLoad += CurrentDomainAssemblyLoad;
AppDomain.CurrentDomain.AssemblyResolve += CurrentDomainAssemblyResolve;
foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies())
{
// Ignore dynamic assemblies.
if (a.IsDynamic)
{
continue;
}
_assemblyQueue.Enqueue(a);
if (!_loadedAssemblies.TryAdd(a.FullName, a))
{
// Duplicate loading.
Console.Error.WriteLine("Duplicate loaded assembly '{0}'", a.FullName);
}
}
// Load all assemblies in our directory.
var moduleName = typeof(ComponentCatalog).Module.FullyQualifiedName;
// If were are loaded in the context of SQL CLR then the FullyQualifiedName and Name properties are set to
// string "<Unknown>" and we skip scanning current directory.
if (moduleName != "<Unknown>")
{
string dir = Path.GetDirectoryName(moduleName);
LoadAssembliesInDir(dir, true);
dir = Path.Combine(dir, "AutoLoad");
LoadAssembliesInDir(dir, true);
}
}
Contracts.AssertValue(_assemblyQueue);
Contracts.AssertValue(_loadedAssemblies);
Assembly assembly;
while (_assemblyQueue.TryDequeue(out assembly))
{
if (!_cachedAssemblies.Add(assembly.FullName))
{
continue;
}
if (assembly != target)
{
bool found = false;
var targetName = target.GetName();
foreach (var name in assembly.GetReferencedAssemblies())
{
if (name.Name == targetName.Name)
{
found = true;
break;
}
}
if (!found)
{
continue;
}
}
#if TRACE_ASSEMBLY_LOADING
// The "" no-op argument is necessary because WriteLine has multiple overloads, and with two strings
// it will be the one that is message/category, rather than format string with
System.Diagnostics.Debug.WriteLine("*** Caching classes in {0}", assembly.FullName, "");
#endif
int added = 0;
foreach (LoadableClassAttributeBase attr in assembly.GetCustomAttributes(typeof(LoadableClassAttributeBase)))
{
MethodInfo getter = null;
ConstructorInfo ctor = null;
MethodInfo create = null;
bool requireEnvironment = false;
if (attr.InstanceType != typeof(void) && !TryGetIniters(attr.InstanceType, attr.LoaderType, attr.CtorTypes, out getter, out ctor, out create, out requireEnvironment))
{
Console.Error.WriteLine(
"CacheClassesFromAssembly: can't instantiate loadable class {0} with name {1}",
attr.InstanceType.Name, attr.LoadNames[0]);
Contracts.Assert(getter == null && ctor == null && create == null);
}
var info = new LoadableClassInfo(attr, getter, ctor, create, requireEnvironment);
AddClass(info, attr.LoadNames);
added++;
}
#if TRACE_ASSEMBLY_LOADING
System.Diagnostics.Debug.WriteLine(" Found {0} entries in {1}", added, assembly.FullName);
#endif
}
}
}
