Rebase changes

Author: franchuterivera

autosklearn/util/backend.py

@@ -89,11 +89,6 @@ def __init__(self,
         # call the setup_logger with this port and update self.logger
         self.logger = None  # type: Optional[PicklableClientLogger]
         self.create_directories()
-        # This is the first place the logger gets created.
-        # We want to make sure any logging forward sets the correct directory
-        # were all files should be created
-        logging.setup_logger(output_dir=self._temporary_directory)
-        self._logger = logging.get_logger(__name__)
 
     def setup_logger(self, port: int) -> None:
         self._logger = get_named_client_logger(

autosklearn/util/logging_.py

@@ -10,6 +10,7 @@
 import socketserver
 import struct
 import threading
+import uuid
 from typing import Any, Dict, Optional, Type
 
 import yaml
@@ -39,61 +40,7 @@ def setup_logger(
             output_dir, distributedlog_filename
         )
 
-    # Applying the configuration is expensive, because logging.config.dictConfig
-    # reconstruct the logging singletons each time it is called. We only call it when
-    # needed, and to do so, we created a initialized attributed to control if a logger
-    # was created
-    if not is_logging_config_applied(logging_config):
-        logging.config.dictConfig(logging_config)
-        for logger_name in list(logging_config['loggers'].keys()) + ['root']:
-            if logger_name == 'root':
-                logger = logging.getLogger()
-            else:
-                logger = logging.getLogger(logger_name)
-            setattr(logger, 'initialized', True)
-
-
-def is_logging_config_applied(logging_config: Dict) -> bool:
-    """
-    This functions check if the provided logging config is already applied to the environment.
-    if it is not the case, it returns false.
-
-    The motivation towards this checking is that in multiprocessing the loggers might be lost,
-    because a new worker in a new node might not have the logger configuration setup.
-
-    Parameters
-    ----------
-    logging_config: (Dict)
-        A logging configuration following the format specified in
-        https://docs.python.org/3/library/logging.config.html
-
-    Returns
-    -------
-    (bool)
-        True if a configuration has already been applied to the environment
-    """
-
-    # The logging config is a dictionary with
-    # dict_keys(['version', 'disable_existing_loggers', 'formatters', 'handlers',
-    # 'root', 'loggers']) . There are 2 things to check. Whether the logger changed
-    # or whether the handlers changed
-
-    # Check the loggers
-    for logger_name in list(logging_config['loggers'].keys()) + ['root']:
-
-        if logger_name == 'root':
-            logger = logging.getLogger()
-        else:
-            logger = logging.getLogger(logger_name)
-
-        # Checking for the contents of the logging config is not feasible as the
-        # logger uses a hierarchical structure, so depending on where this function is
-        # called, the logger_name requires the full hierarchy (like autosklearn.automl)
-        # But because loggers are singletons we can rely on the initialized attribute we
-        # set on them on creation
-        if not hasattr(logger, 'initialized'):
-            return False
-    return True
+    logging.config.dictConfig(logging_config)
 
 
 def _create_logger(name: str) -> logging.Logger:
@@ -240,13 +187,7 @@ def _get_named_client_logger(
 
     # local_logger is mainly used to create a TCP record which is going to be formatted
     # and handled by the main logger server. The main logger server sets up the logging format
-    # that is desired, so we just make sure of two things.
-    # First the local_logger below should not have extra handlers, or else we will be unecessarily
-    # dumping more messages, in addition to the Socket handler we create below
-    # Second, during each multiprocessing spawn, a logger is created
-    # via the logger __setstate__, which is expensive. This is better handled with using
-    # the multiprocessing logger
-    local_logger = multiprocessing.get_logger()
+    local_logger = _create_logger(str(uuid.uuid4()))
 
     # Under this perspective, we print every msg (DEBUG) and let the server decide what to
     # dump. Also, the no propagate disable the root setup to interact with the client
@@ -257,7 +198,8 @@ def _get_named_client_logger(
     # The logger is a singleton, but the logger.handlers is a list. So we need to
     # check if it already has a socket handler on it
     socketHandler = logging.handlers.SocketHandler(host, port)
-    if not any([handler for handler in local_logger.handlers if 'SocketHandler' in str(handler)]):
+    registered_ports = [hdr.port for hdr in local_logger.handlers if 'SocketHandler' in str(hdr)]
+    if port not in registered_ports:
         local_logger.addHandler(socketHandler)
 
     return local_logger

test/test_util/test_logging.py

@@ -12,10 +12,6 @@
 
 
 class LoggingTest(unittest.TestCase):
-    def setUp(self):
-        # We need a clean start for the logging
-        logging.shutdown()
-        reload(logging)
 
     def test_setup_logger(self):
         # Test that setup_logger function correctly configures the logger
@@ -42,8 +38,6 @@ def test_setup_logger(self):
         self.assertEqual(logging.getLogger().getEffectiveLevel(), 10)
 
         # Make sure we log to the desired directory
-        # We have to call setup again, to clean the past logging
-        self.setUp()
         logging_.setup_logger(output_dir=os.path.dirname(__file__),
                               filename='test.log'
                               )
@@ -53,30 +47,3 @@ def test_setup_logger(self):
         with open(os.path.join(os.path.dirname(__file__), 'test.log')) as fh:
             self.assertIn('test_setup_logger', ''.join(fh.readlines()))
         os.remove(os.path.join(os.path.dirname(__file__), 'test.log'))
-
-    def test_setup_logger_called_when_needed(self):
-
-        # Create the testing configuration
-        with open(os.path.join(os.path.dirname(__file__), 'example_config.yaml'), 'r') as fh:
-            example_config = yaml.safe_load(fh)
-
-        logging_config = copy.deepcopy(example_config)
-        output_dir = tempfile.gettempdir()
-        filename = logging_config['handlers']['file_handler']['filename']
-        logging_config['handlers']['file_handler']['filename'] = os.path.join(
-            output_dir, filename
-        )
-        distributedlog_filename = logging_config['handlers']['distributed_logfile']['filename']
-        logging_config['handlers']['distributed_logfile']['filename'] = os.path.join(
-            output_dir, distributedlog_filename
-        )
-
-        # We have not yet sourced any configuration
-        self.assertFalse(logging_.is_logging_config_applied(logging_config=logging_config))
-
-        # Then setup the logging config
-        logging_.setup_logger(logging_config=example_config,
-                              output_dir=tempfile.gettempdir())
-
-        # Then here we do not expect the configuration to change
-        self.assertTrue(logging_.is_logging_config_applied(logging_config=logging_config))