Enable a QuantileRegression Test & Fix Duplicated Baseline Files (#1193)

* Enable a QuantileRegression test

Enable CommandTrainScoreEvaluateQuantileRegression, add the dataset and the necessary baselines.

The only baseline changes were of the form that were caused by https://github.com/dotnet/corefx/issues/31847.

* Allow BaseTestBaseline to check Common first.

A lot of baseline tests have duplicated baselines between debug and release. Allow BaseTestBaseline to check the Common baseline directory for a baseline file first.

Fix #410

* PR feedback

Add more info to the README.
Add column headers to the housing.txt data file.

Author: eerhardt

test/BaselineOutput/Common/Command/CommandTrainScoreEvaluateQuantileRegression-1-out.txt

@@ -0,0 +1 @@
+Wrote 506 rows across 3 columns in %Time%

test/BaselineOutput/Common/Command/CommandTrainScoreEvaluateQuantileRegression-2-out.txt

@@ -0,0 +1 @@
+Wrote 506 rows of length 19

test/BaselineOutput/Common/Command/CommandTrainScoreEvaluateQuantileRegression-3-metrics.txt

@@ -0,0 +1,15 @@
+L1(avg):            1.926877
+L2(avg):            10.630326
+RMS(avg):           3.260418
+Loss-fn(avg):       10.630326
+R Squared:          0.874077
+
+OVERALL RESULTS
+---------------------------------------
+L1(avg):            1.926877 (0.0000)
+L2(avg):            10.630326 (0.0000)
+RMS(avg):           3.260418 (0.0000)
+Loss-fn(avg):       10.630326 (0.0000)
+R Squared:          0.874077 (0.0000)
+
+---------------------------------------

test/BaselineOutput/Common/Command/CommandTrainScoreEvaluateQuantileRegression-3-out.txt

@@ -0,0 +1,15 @@
+L1(avg):            1.926877
+L2(avg):            10.630326
+RMS(avg):           3.260418
+Loss-fn(avg):       10.630326
+R Squared:          0.874077
+
+OVERALL RESULTS
+---------------------------------------
+L1(avg):            1.926877 (0.0000)
+L2(avg):            10.630326 (0.0000)
+RMS(avg):           3.260418 (0.0000)
+Loss-fn(avg):       10.630326 (0.0000)
+R Squared:          0.874077 (0.0000)
+
+---------------------------------------

test/BaselineOutput/Common/Command/CommandTrainScoreEvaluateQuantileRegression-4-out.txt

@@ -0,0 +1,13 @@
+maml.exe Train tr=FastForestRegression{dt+} loader=Text{header+} data=%Data% out=%Output%
+Not adding a normalizer.
+Making per-feature arrays
+Changing data from row-wise to column-wise on disk
+Processed 506 instances
+Binning and forming Feature objects
+Reserved memory for tree learner: 290472 bytes
+Starting to train ...
+Not training a calibrator because it is not needed.
+Physical memory usage(MB): %Number%
+Virtual memory usage(MB): %Number%
+%DateTime%	 Time elapsed(s): %Number%
+

test/BaselineOutput/Common/Command/CommandTrainScoreEvaluateQuantileRegression-data.txt

@@ -9,7 +9,6 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.Runtime.InteropServices;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading;
@@ -31,12 +30,8 @@ protected BaseTestBaseline(ITestOutputHelper output) : base(output)
 
         internal const string RawSuffix = ".raw";
         private const string LogSuffix = ".log";
-        private readonly string _baselineRootRelPath = Path.Combine(TestDir, "BaselineOutput", BuildString); // Relative to Root.
         private readonly string _logRootRelPath = Path.Combine("Logs", BuildString); // Relative to OutDir.
-        private readonly string ScopeRootRelPath = Path.Combine("Samples", "scope"); // Root of files required for Scope related tests. Used primarily for local runs
-        private readonly string TestExtDir = Path.Combine("Tests", "Ext"); // Directory with external binaries checked in. Eg libvw.dll
 
-        private const string SamplesRootRelPath = @"Samples"; // Root location of Samples. Used primarily for local runs
         private const string TestDir = @"test";
 
         private const string DataRootRegExp = @"[a-z]:\\[^/\t ]+\\test\\data" + @"\\[^/\t ]+";
@@ -70,10 +65,9 @@ protected BaseTestBaseline(ITestOutputHelper output) : base(output)
         /// </summary>
         protected const string ProgressLogLine = "--- Progress log ---";
 
-        private static readonly char[] _seperators = { '\t', ' ', '=', '%', '(', ')' };
-
-        // Full paths to the directories.
-        private string _baseDir;
+        // Full paths to the baseline directories.
+        private string _baselineCommonDir;
+        private string _baselineBuildStringDir;
 
         // The writer to write to test log files.
         protected StreamWriter LogWriter;
@@ -86,13 +80,15 @@ protected override void Initialize()
             base.Initialize();
 
             // Create the output and log directories.
-            Contracts.Check(Directory.Exists(Path.Combine(RootDir, TestDir, "BaselineOutput")));
+            string baselineRootDir = Path.Combine(RootDir, TestDir, "BaselineOutput");
+            Contracts.Check(Directory.Exists(baselineRootDir));
+
+            _baselineCommonDir = Path.Combine(baselineRootDir, "Common");
+            _baselineBuildStringDir = Path.Combine(baselineRootDir, BuildString);
+
             string logDir = Path.Combine(OutDir, _logRootRelPath);
             Directory.CreateDirectory(logDir);
 
-            // Find the sample data and baselines.
-            _baseDir = Path.Combine(RootDir, _baselineRootRelPath);
-
             string logPath = Path.Combine(logDir, FullTestName + LogSuffix);
             LogWriter = OpenWriter(logPath);
             _passed = true;
@@ -125,19 +121,6 @@ protected override void Cleanup()
 
         protected bool IsPassing { get { return _passed; } }
 
-        // Return the location of the local Samples folder
-        // Used primarily for local Scope runs
-        protected string SamplesDir { get { return Path.Combine(RootDir, SamplesRootRelPath); } }
-
-        // Return the location of the local scope folder under Samples. Used primarily 
-        // by Scope scripts and for Scope tests
-        protected string ScopeSamplesDir { get { return Path.Combine(RootDir, ScopeRootRelPath); } }
-
-        // Return the local of the directory where external binaries for test purposes are located
-        protected string ExternalTestBinariesDir { get { return Path.Combine(RootDir, TestExtDir); } }
-
-        protected string TestDirectory { get { return Path.Combine(RootDir, TestDir); } }
-
         // Called by a test to signal normal completion. If this is not called before the
         // TestScope is disposed, we assume the test was aborted.
         protected void Done()
@@ -198,31 +181,28 @@ protected void Log(string fmt, params object[] args)
             Output.WriteLine(fmt, args);
         }
 
-        protected string GetBaselineDir(string subDir)
+        protected string GetBaselinePath(string name)
         {
             Contracts.Assert(IsActive);
-            if (string.IsNullOrWhiteSpace(subDir))
+            if (string.IsNullOrWhiteSpace(name))
                 return null;
-            return Path.GetFullPath(Path.Combine(_baseDir, subDir));
-            //return Path.Combine(_baseDir, subDir);
+
+            return GetBaselinePath(string.Empty, name);
         }
 
         protected string GetBaselinePath(string subDir, string name)
         {
             Contracts.Assert(IsActive);
-            if (string.IsNullOrWhiteSpace(subDir))
-                return GetBaselinePath(name);
-            return Path.GetFullPath(Path.Combine(_baseDir, subDir, name));
-            //return Path.Combine(_baseDir, subDir, name);
-        }
+            subDir = subDir ?? string.Empty;
 
-        protected string GetBaselinePath(string name)
-        {
-            Contracts.Assert(IsActive);
-            if (string.IsNullOrWhiteSpace(name))
-                return null;
-            //return Path.Combine(_baseDir, name);
-            return Path.GetFullPath(Path.Combine(_baseDir, name));
+            // first check the Common folder, and use it if it exists
+            string commonBaselinePath = Path.GetFullPath(Path.Combine(_baselineCommonDir, subDir, name));
+            if (File.Exists(commonBaselinePath))
+            {
+                return commonBaselinePath;
+            }
+
+            return Path.GetFullPath(Path.Combine(_baselineBuildStringDir, subDir, name));
         }
 
         // Inverts the _passed flag. Do not ever use this except in rare conditions. Eg. Recording failure of a test as a success.
@@ -393,36 +373,6 @@ protected bool CheckEqualityCore(string dir, string name, string nameBase, bool
             return res;
         }
 
-        /// <summary>
-        /// Check whether two files are same ignoring volatile differences (path, dates, times, etc),
-        /// skipping the given number of lines on the output, and finding the corresponding line
-        /// in the baseline.
-        /// </summary>
-        protected bool CheckEqualityNormalizedFromPaths(string desc, string basePath, string outPath, int skip = 0)
-        {
-            Contracts.Assert(IsActive);
-            Contracts.AssertNonEmpty(basePath);
-            Contracts.AssertNonEmpty(outPath);
-            Contracts.Assert(skip >= 0);
-
-            if (!CheckOutFile(outPath))
-                return false;
-
-            // Normalize the output file.
-            Normalize(outPath);
-
-            if (!CheckBaseFile(basePath))
-                return false;
-
-            bool res = CheckEqualityFromPathsCore(desc, basePath, outPath, skip);
-
-            // No need to keep the raw (unnormalized) output file.
-            if (res)
-                File.Delete(outPath + RawSuffix);
-
-            return res;
-        }
-
         private bool FirstIsSuffix<T>(IEnumerator<T> suffix, IEnumerator<T> seq, Func<T, T, bool> equalFunc = null)
         {
             Contracts.AssertValue(suffix);
@@ -464,11 +414,6 @@ private bool FirstIsSuffix<T>(IEnumerator<T> suffix, IEnumerator<T> seq, Func<T,
             return true;
         }
 
-        private IEnumerator<string> LineEnumerator(TextReader reader)
-        {
-            return LineEnumerator(reader, x => false);
-        }
-
         private IEnumerator<string> LineEnumerator(TextReader reader, Func<string, bool> stop)
         {
             string result;
@@ -496,41 +441,6 @@ protected bool CheckOutputIsSuffix(string basePath, string outPath, int skip = 0
             }
         }
 
-        /// <summary>
-        /// Check whether two files are same ignoring volatile differences (path, dates, times, etc),
-        /// skipping the given number of lines on the output, and finding the corresponding line
-        /// in the baseline.
-        /// </summary>
-        protected bool CheckEqualityNormalized(string dir, string name, string suffix, int skip, int digitsOfPrecision = DigitsOfPrecision)
-        {
-            Contracts.Assert(IsActive);
-            Contracts.AssertValue(dir); // Can be empty.
-            Contracts.AssertNonEmpty(name);
-            Contracts.AssertNonEmpty(suffix);
-            Contracts.Assert(skip >= 0);
-
-            string relPath = Path.Combine(dir, name + suffix);
-            string basePath = GetBaselinePath(dir, name);
-            string outPath = GetOutputPath(dir, name + suffix);
-
-            if (!CheckOutFile(outPath))
-                return false;
-
-            // Normalize the output file.
-            Normalize(outPath);
-
-            if (!CheckBaseFile(basePath))
-                return false;
-
-            bool res = CheckEqualityFromPathsCore(relPath, basePath, outPath, skip, digitsOfPrecision);
-
-            // No need to keep the raw (unnormalized) output file.
-            if (res)
-                File.Delete(outPath + RawSuffix);
-
-            return res;
-        }
-
         protected bool CheckEqualityFromPathsCore(string relPath, string basePath, string outPath, int skip = 0, int digitsOfPrecision = DigitsOfPrecision)
         {
             Contracts.Assert(skip >= 0);
@@ -838,7 +748,7 @@ protected static StreamReader OpenReader(string path)
         {
             Contracts.CheckNonWhiteSpace(path, nameof(path));
 #if CORECLR
-            return new StreamReader(File.Open(path, FileMode.Open, FileAccess.Read));
+            return new StreamReader(File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read));
 #else
             return new StreamReader(path);
 #endif
@@ -857,16 +767,6 @@ protected static int MainForTest(string args)
                 return result;
             }
         }
-
-        protected static string GetEnvironmentVariable(string name)
-        {
-            return Environment.GetEnvironmentVariable(name, EnvironmentVariableTarget.Process);
-        }
-
-        protected static void SetEnvironmentVariable(string name, string value)
-        {
-            Environment.SetEnvironmentVariable(name, value, EnvironmentVariableTarget.Process);
-        }
     }
 
     public partial class TestBaselineNormalize : BaseTestBaseline

test/BaselineOutput/Common/Command/CommandTrainScoreEvaluateQuantileRegression-out.txt

@@ -1662,15 +1662,15 @@ public void CommandTrainScoreEvaluateSdcaRegression()
         }
 
         [TestCategory(Cat), TestCategory("FastForest")]
-        [Fact(Skip = "Need CoreTLC specific baseline update")]
+        [Fact]
         public void CommandTrainScoreEvaluateQuantileRegression()
         {
             RunMTAThread(() =>
             {
                 // First run a training.
-                string pathData = GetDataPath(@"..\Housing (regression)", "housing.txt");
+                string pathData = GetDataPath("housing.txt");
                 OutputPath trainModel = ModelPath();
-                TestCore("train", pathData, "loader=text", "lab=Label feat=Features tr=FastForestRegression{dt+}");
+                TestCore("train", pathData, "loader=Text{header+}", "lab=Label feat=Features tr=FastForestRegression{dt+}");
 
                 // Then, run the score.
                 _step++;

test/Microsoft.ML.TestFramework/BaseTestBaseline.cs

@@ -65,6 +65,14 @@ The dataset is under a CC-by 4.0 license.
 }
 ```
 
+### Boston Housing Data
+
+Redistributing the dataset "[housing.txt](housing.txt)" with attribution:
+
+ > Harrison, D. and Rubinfeld, D.L. 'Hedonic prices and the demand for clean air', J. Environ. Economics & Management, vol.5, 81-102, 1978.
+
+More information: https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.names
+
 # Images
 
 ### Located in `images` folder