Renamed TlcEnvironment to Console. Also introduced LocalEnvironment (#923)

Author: Zruty0

src/Microsoft.ML.Api/TypedCursor.cs

@@ -658,7 +658,7 @@ public static ICursorable<TRow> AsCursorable<TRow>(this IDataView data, bool ign
             where TRow : class, new()
         {
             // REVIEW: Take an env as a parameter.
-            var env = new TlcEnvironment();
+            var env = new ConsoleEnvironment();
             return data.AsCursorable<TRow>(env, ignoreMissingColumns, schemaDefinition);
         }
 
@@ -699,7 +699,7 @@ public static IEnumerable<TRow> AsEnumerable<TRow>(this IDataView data, bool reu
             where TRow : class, new()
         {
             // REVIEW: Take an env as a parameter.
-            var env = new TlcEnvironment();
+            var env = new ConsoleEnvironment();
             return data.AsEnumerable<TRow>(env, reuseRowObject, ignoreMissingColumns, schemaDefinition);
         }
     }

src/Microsoft.ML.Core/Data/ProgressReporter.cs

@@ -17,7 +17,7 @@ namespace Microsoft.ML.Runtime.Data
     public static class ProgressReporting
     {
         /// <summary>
-        /// The progress channel for <see cref="TlcEnvironment"/>.
+        /// The progress channel for <see cref="ConsoleEnvironment"/>.
         /// This is coupled with a <see cref="ProgressTracker"/> that aggregates all events and returns them on demand.
         /// </summary>
         public sealed class ProgressChannel : IProgressChannel

src/Microsoft.ML.Core/Environment/TlcEnvironment.cs renamed to src/Microsoft.ML.Core/Environment/ConsoleEnvironment.cs

@@ -15,14 +15,14 @@ namespace Microsoft.ML.Runtime.Data
 {
     using Stopwatch = System.Diagnostics.Stopwatch;
 
-    public sealed class TlcEnvironment : HostEnvironmentBase<TlcEnvironment>
+    public sealed class ConsoleEnvironment : HostEnvironmentBase<ConsoleEnvironment>
     {
         public const string ComponentHistoryKey = "ComponentHistory";
 
         private sealed class ConsoleWriter
         {
             private readonly object _lock;
-            private readonly TlcEnvironment _parent;
+            private readonly ConsoleEnvironment _parent;
             private readonly TextWriter _out;
             private readonly TextWriter _err;
 
@@ -34,7 +34,7 @@ private sealed class ConsoleWriter
             private const int _maxDots = 50;
             private int _dots;
 
-            public ConsoleWriter(TlcEnvironment parent, TextWriter outWriter, TextWriter errWriter)
+            public ConsoleWriter(ConsoleEnvironment parent, TextWriter outWriter, TextWriter errWriter)
             {
                 Contracts.AssertValue(parent);
                 Contracts.AssertValue(outWriter);
@@ -331,7 +331,7 @@ private bool PrintDot()
         private sealed class Channel : ChannelBase
         {
             public readonly Stopwatch Watch;
-            public Channel(TlcEnvironment root, ChannelProviderBase parent, string shortName,
+            public Channel(ConsoleEnvironment root, ChannelProviderBase parent, string shortName,
                 Action<IMessageSource, ChannelMessage> dispatch)
                 : base(root, parent, shortName, dispatch)
             {
@@ -358,20 +358,36 @@ protected override void DisposeCore()
         private volatile ConsoleWriter _consoleWriter;
         private readonly MessageSensitivity _sensitivityFlags;
 
-        public TlcEnvironment(int? seed = null, bool verbose = false,
+        /// <summary>
+        /// Create an ML.NET <see cref="IHostEnvironment"/> for local execution, with console feedback.
+        /// </summary>
+        /// <param name="seed">Random seed. Set to <c>null</c> for a non-deterministic environment.</param>
+        /// <param name="verbose">Set to <c>true</c> for fully verbose logging.</param>
+        /// <param name="sensitivity">Allowed message sensitivity.</param>
+        /// <param name="conc">Concurrency level. Set to 1 to run single-threaded. Set to 0 to pick automatically.</param>
+        /// <param name="outWriter">Text writer to print normal messages to.</param>
+        /// <param name="errWriter">Text writer to print error messages to.</param>
+        public ConsoleEnvironment(int? seed = null, bool verbose = false,
             MessageSensitivity sensitivity = MessageSensitivity.All, int conc = 0,
             TextWriter outWriter = null, TextWriter errWriter = null)
             : this(RandomUtils.Create(seed), verbose, sensitivity, conc, outWriter, errWriter)
         {
         }
 
+        // REVIEW: do we really care about custom random? If we do, let's make this ctor public.
         /// <summary>
-        /// This takes ownership of the random number generator.
+        /// Create an ML.NET environment for local execution, with console feedback.
         /// </summary>
-        public TlcEnvironment(IRandom rand, bool verbose = false,
+        /// <param name="rand">An custom source of randomness to use in the environment.</param>
+        /// <param name="verbose">Set to <c>true</c> for fully verbose logging.</param>
+        /// <param name="sensitivity">Allowed message sensitivity.</param>
+        /// <param name="conc">Concurrency level. Set to 1 to run single-threaded. Set to 0 to pick automatically.</param>
+        /// <param name="outWriter">Text writer to print normal messages to.</param>
+        /// <param name="errWriter">Text writer to print error messages to.</param>
+        private ConsoleEnvironment(IRandom rand, bool verbose = false,
             MessageSensitivity sensitivity = MessageSensitivity.All, int conc = 0,
             TextWriter outWriter = null, TextWriter errWriter = null)
-            : base(rand, verbose, conc, nameof(TlcEnvironment))
+            : base(rand, verbose, conc, nameof(ConsoleEnvironment))
         {
             Contracts.CheckValueOrNull(outWriter);
             Contracts.CheckValueOrNull(errWriter);
@@ -401,7 +417,7 @@ protected override IFileHandle CreateTempFileCore(IHostEnvironment env, string s
             return base.CreateTempFileCore(env, suffix, "TLC_" + prefix);
         }
 
-        protected override IHost RegisterCore(HostEnvironmentBase<TlcEnvironment> source, string shortName, string parentFullName, IRandom rand, bool verbose, int? conc)
+        protected override IHost RegisterCore(HostEnvironmentBase<ConsoleEnvironment> source, string shortName, string parentFullName, IRandom rand, bool verbose, int? conc)
         {
             Contracts.AssertValue(rand);
             Contracts.AssertValueOrNull(parentFullName);
@@ -413,15 +429,15 @@ protected override IHost RegisterCore(HostEnvironmentBase<TlcEnvironment> source
         protected override IChannel CreateCommChannel(ChannelProviderBase parent, string name)
         {
             Contracts.AssertValue(parent);
-            Contracts.Assert(parent is TlcEnvironment);
+            Contracts.Assert(parent is ConsoleEnvironment);
             Contracts.AssertNonEmpty(name);
             return new Channel(this, parent, name, GetDispatchDelegate<ChannelMessage>());
         }
 
         protected override IPipe<TMessage> CreatePipe<TMessage>(ChannelProviderBase parent, string name)
         {
             Contracts.AssertValue(parent);
-            Contracts.Assert(parent is TlcEnvironment);
+            Contracts.Assert(parent is ConsoleEnvironment);
             Contracts.AssertNonEmpty(name);
             return new Pipe<TMessage>(parent, name, GetDispatchDelegate<TMessage>());
         }
@@ -439,11 +455,11 @@ internal IDisposable RedirectChannelOutput(TextWriter newOutWriter, TextWriter n
 
         private sealed class OutputRedirector : IDisposable
         {
-            private readonly TlcEnvironment _root;
+            private readonly ConsoleEnvironment _root;
             private ConsoleWriter _oldConsoleWriter;
             private readonly ConsoleWriter _newConsoleWriter;
 
-            public OutputRedirector(TlcEnvironment env, TextWriter newOutWriter, TextWriter newErrWriter)
+            public OutputRedirector(ConsoleEnvironment env, TextWriter newOutWriter, TextWriter newErrWriter)
             {
                 Contracts.AssertValue(env);
                 Contracts.AssertValue(newOutWriter);
@@ -467,7 +483,7 @@ public void Dispose()
 
         private sealed class Host : HostBase
         {
-            public Host(HostEnvironmentBase<TlcEnvironment> source, string shortName, string parentFullName, IRandom rand, bool verbose, int? conc)
+            public Host(HostEnvironmentBase<ConsoleEnvironment> source, string shortName, string parentFullName, IRandom rand, bool verbose, int? conc)
                 : base(source, shortName, parentFullName, rand, verbose, conc)
             {
                 IsCancelled = source.IsCancelled;
@@ -489,7 +505,7 @@ protected override IPipe<TMessage> CreatePipe<TMessage>(ChannelProviderBase pare
                 return new Pipe<TMessage>(parent, name, GetDispatchDelegate<TMessage>());
             }
 
-            protected override IHost RegisterCore(HostEnvironmentBase<TlcEnvironment> source, string shortName, string parentFullName, IRandom rand, bool verbose, int? conc)
+            protected override IHost RegisterCore(HostEnvironmentBase<ConsoleEnvironment> source, string shortName, string parentFullName, IRandom rand, bool verbose, int? conc)
             {
                 return new Host(source, shortName, parentFullName, rand, verbose, conc);
             }

src/Microsoft.ML.Data/Utilities/LocalEnvironment.cs

@@ -0,0 +1,131 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+
+namespace Microsoft.ML.Runtime.Data
+{
+    using Stopwatch = System.Diagnostics.Stopwatch;
+
+    /// <summary>
+    /// An ML.NET environment for local execution.
+    /// </summary>
+    public sealed class LocalEnvironment : HostEnvironmentBase<LocalEnvironment>
+    {
+        private sealed class Channel : ChannelBase
+        {
+            public readonly Stopwatch Watch;
+            public Channel(LocalEnvironment root, ChannelProviderBase parent, string shortName,
+                Action<IMessageSource, ChannelMessage> dispatch)
+                : base(root, parent, shortName, dispatch)
+            {
+                Watch = Stopwatch.StartNew();
+                Dispatch(this, new ChannelMessage(ChannelMessageKind.Trace, MessageSensitivity.None, "Channel started"));
+            }
+
+            public override void Done()
+            {
+                Watch.Stop();
+                ChannelFinished();
+                base.Done();
+            }
+
+            private void ChannelFinished()
+                => Dispatch(this, new ChannelMessage(ChannelMessageKind.Trace, MessageSensitivity.None, "Channel finished. Elapsed { 0:c }.", Watch.Elapsed));
+
+            protected override void DisposeCore()
+            {
+                if (IsActive)
+                {
+                    ChannelFinished();
+                    Watch.Stop();
+                }
+
+                Dispatch(this, new ChannelMessage(ChannelMessageKind.Trace, MessageSensitivity.None, "Channel disposed"));
+                base.DisposeCore();
+            }
+        }
+
+        /// <summary>
+        /// Create an ML.NET <see cref="IHostEnvironment"/> for local execution.
+        /// </summary>
+        /// <param name="seed">Random seed. Set to <c>null</c> for a non-deterministic environment.</param>
+        /// <param name="conc">Concurrency level. Set to 1 to run single-threaded. Set to 0 to pick automatically.</param>
+        public LocalEnvironment(int? seed = null, int conc = 0)
+            : base(RandomUtils.Create(seed), verbose: false, conc)
+        {
+        }
+
+        /// <summary>
+        /// Add a custom listener to the messages of ML.NET components.
+        /// </summary>
+        public void AddListener(Action<IMessageSource, ChannelMessage> listener)
+            => AddListener<ChannelMessage>(listener);
+
+        /// <summary>
+        /// Remove a previously added a custom listener.
+        /// </summary>
+        public void RemoveListener(Action<IMessageSource, ChannelMessage> listener)
+            => RemoveListener<ChannelMessage>(listener);
+
+        protected override IFileHandle CreateTempFileCore(IHostEnvironment env, string suffix = null, string prefix = null)
+            => base.CreateTempFileCore(env, suffix, "Local_" + prefix);
+
+        protected override IHost RegisterCore(HostEnvironmentBase<LocalEnvironment> source, string shortName, string parentFullName, IRandom rand, bool verbose, int? conc)
+        {
+            Contracts.AssertValue(rand);
+            Contracts.AssertValueOrNull(parentFullName);
+            Contracts.AssertNonEmpty(shortName);
+            Contracts.Assert(source == this || source is Host);
+            return new Host(source, shortName, parentFullName, rand, verbose, conc);
+        }
+
+        protected override IChannel CreateCommChannel(ChannelProviderBase parent, string name)
+        {
+            Contracts.AssertValue(parent);
+            Contracts.Assert(parent is LocalEnvironment);
+            Contracts.AssertNonEmpty(name);
+            return new Channel(this, parent, name, GetDispatchDelegate<ChannelMessage>());
+        }
+
+        protected override IPipe<TMessage> CreatePipe<TMessage>(ChannelProviderBase parent, string name)
+        {
+            Contracts.AssertValue(parent);
+            Contracts.Assert(parent is LocalEnvironment);
+            Contracts.AssertNonEmpty(name);
+            return new Pipe<TMessage>(parent, name, GetDispatchDelegate<TMessage>());
+        }
+
+        private sealed class Host : HostBase
+        {
+            public Host(HostEnvironmentBase<LocalEnvironment> source, string shortName, string parentFullName, IRandom rand, bool verbose, int? conc)
+                : base(source, shortName, parentFullName, rand, verbose, conc)
+            {
+                IsCancelled = source.IsCancelled;
+            }
+
+            protected override IChannel CreateCommChannel(ChannelProviderBase parent, string name)
+            {
+                Contracts.AssertValue(parent);
+                Contracts.Assert(parent is Host);
+                Contracts.AssertNonEmpty(name);
+                return new Channel(Root, parent, name, GetDispatchDelegate<ChannelMessage>());
+            }
+
+            protected override IPipe<TMessage> CreatePipe<TMessage>(ChannelProviderBase parent, string name)
+            {
+                Contracts.AssertValue(parent);
+                Contracts.Assert(parent is Host);
+                Contracts.AssertNonEmpty(name);
+                return new Pipe<TMessage>(parent, name, GetDispatchDelegate<TMessage>());
+            }
+
+            protected override IHost RegisterCore(HostEnvironmentBase<LocalEnvironment> source, string shortName, string parentFullName, IRandom rand, bool verbose, int? conc)
+            {
+                return new Host(source, shortName, parentFullName, rand, verbose, conc);
+            }
+        }
+    }
+
+}

src/Microsoft.ML.FastTree/FastTree.cs

@@ -110,8 +110,8 @@ private protected FastTreeTrainerBase(IHostEnvironment env, TArgs args)
             ParallelTraining = Args.ParallelTrainer != null ? Args.ParallelTrainer.CreateComponent(env) : new SingleTrainer();
             ParallelTraining.InitEnvironment();
             // REVIEW: CLR 4.6 has a bug that is only exposed in Scope, and if we trigger GC.Collect in scope environment
-            // with memory consumption more than 5GB, GC get stuck in infinite loop. So for now let's call GC only if we call things from TlcEnvironment.
-            AllowGC = (env is HostEnvironmentBase<TlcEnvironment>);
+            // with memory consumption more than 5GB, GC get stuck in infinite loop. So for now let's call GC only if we call things from ConsoleEnvironment.
+            AllowGC = (env is HostEnvironmentBase<ConsoleEnvironment>);
             Tests = new List<Test>();
 
             InitializeThreads(numThreads);

src/Microsoft.ML.Legacy/LearningPipeline.cs

@@ -161,7 +161,7 @@ public PredictionModel<TInput, TOutput> Train<TInput, TOutput>()
             where TInput : class
             where TOutput : class, new()
         {
-            using (var environment = new TlcEnvironment(seed: _seed, conc: _conc))
+            using (var environment = new ConsoleEnvironment(seed: _seed, conc: _conc))
             {
                 Experiment experiment = environment.CreateExperiment();
                 ILearningPipelineStep step = null;

src/Microsoft.ML.Legacy/LearningPipelineDebugProxy.cs

@@ -25,7 +25,7 @@ internal sealed class LearningPipelineDebugProxy
         private const int MaxSlotNamesToDisplay = 100;
 
         private readonly LearningPipeline _pipeline;
-        private readonly TlcEnvironment _environment;
+        private readonly ConsoleEnvironment _environment;
         private IDataView _preview;
         private Exception _pipelineExecutionException;
         private PipelineItemDebugColumn[] _columns;
@@ -39,7 +39,7 @@ public LearningPipelineDebugProxy(LearningPipeline pipeline)
             _pipeline = new LearningPipeline();
 
             // use a ConcurrencyFactor of 1 so other threads don't need to run in the debugger
-            _environment = new TlcEnvironment(conc: 1);
+            _environment = new ConsoleEnvironment(conc: 1);
 
             foreach (ILearningPipelineItem item in pipeline)
             {

src/Microsoft.ML.Legacy/Models/BinaryClassificationEvaluator.cs

@@ -24,7 +24,7 @@ public sealed partial class BinaryClassificationEvaluator
         /// </returns>
         public BinaryClassificationMetrics Evaluate(PredictionModel model, ILearningPipelineLoader testData)
         {
-            using (var environment = new TlcEnvironment())
+            using (var environment = new ConsoleEnvironment())
             {
                 environment.CheckValue(model, nameof(model));
                 environment.CheckValue(testData, nameof(testData));

src/Microsoft.ML.Legacy/Models/ClassificationEvaluator.cs

@@ -25,7 +25,7 @@ public sealed partial class ClassificationEvaluator
         /// </returns>
         public ClassificationMetrics Evaluate(PredictionModel model, ILearningPipelineLoader testData)
         {
-            using (var environment = new TlcEnvironment())
+            using (var environment = new ConsoleEnvironment())
             {
                 environment.CheckValue(model, nameof(model));
                 environment.CheckValue(testData, nameof(testData));

src/Microsoft.ML.Legacy/Models/ClusterEvaluator.cs

@@ -24,7 +24,7 @@ public sealed partial class ClusterEvaluator
         /// </returns>
         public ClusterMetrics Evaluate(PredictionModel model, ILearningPipelineLoader testData)
         {
-            using (var environment = new TlcEnvironment())
+            using (var environment = new ConsoleEnvironment())
             {
                 environment.CheckValue(model, nameof(model));
                 environment.CheckValue(testData, nameof(testData));