gh-93162: Add ability to configure QueueHandler/QueueListener together (GH-93269)

vsajip · web-flow · commit 1b7480399162 · 2022-06-07T10:20:35.000+02:00
Also, provide getHandlerByName() and getHandlerNames() APIs. Closes #93162.
diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst
@@ -661,6 +661,76 @@ it with :func:`staticmethod`. For example::
 You don't need to wrap with :func:`staticmethod` if you're setting the import
 callable on a configurator *instance*.
 
+.. _configure-queue:
+
+Configuring QueueHandler and QueueListener
+""""""""""""""""""""""""""""""""""""""""""
+
+If you want to configure a :class:`~logging.handlers.QueueHandler`, noting that this
+is normally used in conjunction with a :class:`~logging.handlers.QueueListener`, you
+can configure both together. After the configuration, the ``QueueListener`` instance
+will be available as the :attr:`~logging.handlers.QueueHandler.listener` attribute of
+the created handler, and that in turn will be available to you using
+:func:`~logging.getHandlerByName` and passing the name you have used for the
+``QueueHandler`` in your configuration. The dictionary schema for configuring the pair
+is shown in the example YAML snippet below.
+
+.. code-block:: yaml
+
+    handlers:
+      qhand:
+        class: logging.handlers.QueueHandler
+        queue: my.module.queue_factory
+        listener: my.package.CustomListener
+        handlers:
+          - hand_name_1
+          - hand_name_2
+          ...
+
+The ``queue`` and ``listener`` keys are optional.
+
+If the ``queue`` key is present, the corresponding value can be one of the following:
+
+* An actual instance of :class:`queue.Queue` or a subclass thereof. This is of course
+  only possible if you are constructing or modifying the configuration dictionary in
+  code.
+
+* A string that resolves to a callable which, when called with no arguments, returns
+  the :class:`queue.Queue` instance to use. That callable could be a
+  :class:`queue.Queue` subclass or a function which returns a suitable queue instance,
+  such as ``my.module.queue_factory()``.
+
+* A dict with a ``'()'`` key which is constructed in the usual way as discussed in
+  :ref:`logging-config-dict-userdef`. The result of this construction should be a
+  :class:`queue.Queue` instance.
+
+If the  ``queue`` key is absent, a standard unbounded :class:`queue.Queue` instance is
+created and used.
+
+If the ``listener`` key is present, the corresponding value can be one of the following:
+
+* A subclass of :class:`logging.handlers.QueueListener`. This is of course only
+  possible if you are constructing or modifying the configuration dictionary in
+  code.
+
+* A string which resolves to a class which is a subclass of ``QueueListener``, such as
+  ``'my.package.CustomListener'``.
+
+* A dict with a ``'()'`` key which is constructed in the usual way as discussed in
+  :ref:`logging-config-dict-userdef`. The result of this construction should be a
+  callable with the same signature as the ``QueueListener`` initializer.
+
+If the ``listener`` key is absent, :class:`logging.handlers.QueueListener` is used.
+
+The values under the ``handlers`` key are the names of other handlers in the
+configuration (not shown in the above snippet) which will be passed to the queue
+listener.
+
+Any custom queue handler and listener classes will need to be defined with the same
+initialization signatures as :class:`~logging.handlers.QueueHandler` and
+:class:`~logging.handlers.QueueListener`.
+
+.. versionadded:: 3.12
 
 .. _logging-config-fileformat:
 
diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst
@@ -1051,7 +1051,13 @@ possible, while any potentially slow operations (such as sending an email via
       want to override this if you want to use blocking behaviour, or a
       timeout, or a customized queue implementation.
 
+   .. attribute:: listener
 
+      When created via configuration using :func:`~logging.config.dictConfig`, this
+      attribute will contain a :class:`QueueListener` instance for use with this
+      handler. Otherwise, it will be ``None``.
+
+      .. versionadded:: 3.12
 
 .. _queue-listener:
 
diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst
@@ -1164,6 +1164,19 @@ functions.
       This undocumented behaviour was considered a mistake, and was removed in
       Python 3.4, but reinstated in 3.4.2 due to retain backward compatibility.
 
+.. function:: getHandlerByName(name)
+
+   Returns a handler with the specified *name*, or ``None`` if there is no handler
+   with that name.
+
+   .. versionadded:: 3.12
+
+.. function:: getHandlerNames()
+
+   Returns an immutable set of all known handler names.
+
+   .. versionadded:: 3.12
+
 .. function:: makeLogRecord(attrdict)
 
    Creates and returns a new :class:`LogRecord` instance whose attributes are
diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2001-2019 by Vinay Sajip. All Rights Reserved.
+# Copyright 2001-2022 by Vinay Sajip. All Rights Reserved.
 #
 # Permission to use, copy, modify, and distribute this software and its
 # documentation for any purpose and without fee is hereby granted,
@@ -18,7 +18,7 @@
 Logging package for Python. Based on PEP 282 and comments thereto in
 comp.lang.python.
 
-Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved.
+Copyright (C) 2001-2022 Vinay Sajip. All Rights Reserved.
 
 To use, simply 'import logging' and log away!
 """
@@ -38,7 +38,8 @@
            'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass',
            'info', 'log', 'makeLogRecord', 'setLoggerClass', 'shutdown',
            'warn', 'warning', 'getLogRecordFactory', 'setLogRecordFactory',
-           'lastResort', 'raiseExceptions', 'getLevelNamesMapping']
+           'lastResort', 'raiseExceptions', 'getLevelNamesMapping',
+           'getHandlerByName', 'getHandlerNames']
 
 import threading
 
@@ -885,6 +886,23 @@ def _addHandlerRef(handler):
     finally:
         _releaseLock()
 
+
+def getHandlerByName(name):
+    """
+    Get a handler with the specified *name*, or None if there isn't one with
+    that name.
+    """
+    return _handlers.get(name)
+
+
+def getHandlerNames():
+    """
+    Return all known handler names as an immutable set.
+    """
+    result = set(_handlers.keys())
+    return frozenset(result)
+
+
 class Handler(Filterer):
     """
     Handler instances dispatch logging events to specific destinations.
diff --git a/Lib/logging/config.py b/Lib/logging/config.py
@@ -1,4 +1,4 @@
-# Copyright 2001-2019 by Vinay Sajip. All Rights Reserved.
+# Copyright 2001-2022 by Vinay Sajip. All Rights Reserved.
 #
 # Permission to use, copy, modify, and distribute this software and its
 # documentation for any purpose and without fee is hereby granted,
@@ -19,15 +19,17 @@
 is based on PEP 282 and comments thereto in comp.lang.python, and influenced
 by Apache's log4j system.
 
-Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved.
+Copyright (C) 2001-2022 Vinay Sajip. All Rights Reserved.
 
 To use, simply 'import logging' and log away!
 """
 
 import errno
+import functools
 import io
 import logging
 import logging.handlers
+import queue
 import re
 import struct
 import threading
@@ -563,7 +565,7 @@ def configure(self):
                         handler.name = name
                         handlers[name] = handler
                     except Exception as e:
-                        if 'target not configured yet' in str(e.__cause__):
+                        if ' not configured yet' in str(e.__cause__):
                             deferred.append(name)
                         else:
                             raise ValueError('Unable to configure handler '
@@ -702,6 +704,21 @@ def add_filters(self, filterer, filters):
             except Exception as e:
                 raise ValueError('Unable to add filter %r' % f) from e
 
+    def _configure_queue_handler(self, klass, **kwargs):
+        if 'queue' in kwargs:
+            q = kwargs['queue']
+        else:
+            q = queue.Queue()  # unbounded
+        rhl = kwargs.get('respect_handler_level', False)
+        if 'listener' in kwargs:
+            lklass = kwargs['listener']
+        else:
+            lklass = logging.handlers.QueueListener
+        listener = lklass(q, *kwargs['handlers'], respect_handler_level=rhl)
+        handler = klass(q)
+        handler.listener = listener
+        return handler
+
     def configure_handler(self, config):
         """Configure a handler from a dictionary."""
         config_copy = dict(config)  # for restoring in case of error
@@ -721,26 +738,83 @@ def configure_handler(self, config):
             factory = c
         else:
             cname = config.pop('class')
-            klass = self.resolve(cname)
-            #Special case for handler which refers to another handler
+            if callable(cname):
+                klass = cname
+            else:
+                klass = self.resolve(cname)
             if issubclass(klass, logging.handlers.MemoryHandler) and\
                 'target' in config:
+                # Special case for handler which refers to another handler
                 try:
-                    th = self.config['handlers'][config['target']]
+                    tn = config['target']
+                    th = self.config['handlers'][tn]
                     if not isinstance(th, logging.Handler):
                         config.update(config_copy)  # restore for deferred cfg
                         raise TypeError('target not configured yet')
                     config['target'] = th
                 except Exception as e:
-                    raise ValueError('Unable to set target handler '
-                                     '%r' % config['target']) from e
+                    raise ValueError('Unable to set target handler %r' % tn) from e
+            elif issubclass(klass, logging.handlers.QueueHandler):
+                # Another special case for handler which refers to other handlers
+                if 'handlers' not in config:
+                    raise ValueError('No handlers specified for a QueueHandler')
+                if 'queue' in config:
+                    qspec = config['queue']
+                    if not isinstance(qspec, queue.Queue):
+                        if isinstance(qspec, str):
+                            q = self.resolve(qspec)
+                            if not callable(q):
+                                raise TypeError('Invalid queue specifier %r' % qspec)
+                            q = q()
+                        elif isinstance(qspec, dict):
+                            if '()' not in qspec:
+                                raise TypeError('Invalid queue specifier %r' % qspec)
+                            q = self.configure_custom(dict(qspec))
+                        else:
+                            raise TypeError('Invalid queue specifier %r' % qspec)
+                        config['queue'] = q
+                if 'listener' in config:
+                    lspec = config['listener']
+                    if isinstance(lspec, type):
+                        if not issubclass(lspec, logging.handlers.QueueListener):
+                            raise TypeError('Invalid listener specifier %r' % lspec)
+                    else:
+                        if isinstance(lspec, str):
+                            listener = self.resolve(lspec)
+                            if isinstance(listener, type) and\
+                                not issubclass(listener, logging.handlers.QueueListener):
+                                raise TypeError('Invalid listener specifier %r' % lspec)
+                        elif isinstance(lspec, dict):
+                            if '()' not in lspec:
+                                raise TypeError('Invalid listener specifier %r' % lspec)
+                            listener = self.configure_custom(dict(lspec))
+                        else:
+                            raise TypeError('Invalid listener specifier %r' % lspec)
+                        if not callable(listener):
+                            raise TypeError('Invalid listener specifier %r' % lspec)
+                        config['listener'] = listener
+                hlist = []
+                try:
+                    for hn in config['handlers']:
+                        h = self.config['handlers'][hn]
+                        if not isinstance(h, logging.Handler):
+                            config.update(config_copy)  # restore for deferred cfg
+                            raise TypeError('Required handler %r '
+                                            'is not configured yet' % hn)
+                        hlist.append(h)
+                except Exception as e:
+                    raise ValueError('Unable to set required handler %r' % hn) from e
+                config['handlers'] = hlist
             elif issubclass(klass, logging.handlers.SMTPHandler) and\
                 'mailhost' in config:
                 config['mailhost'] = self.as_tuple(config['mailhost'])
             elif issubclass(klass, logging.handlers.SysLogHandler) and\
                 'address' in config:
                 config['address'] = self.as_tuple(config['address'])
-            factory = klass
+            if issubclass(klass, logging.handlers.QueueHandler):
+                factory = functools.partial(self._configure_queue_handler, klass)
+            else:
+                factory = klass
         props = config.pop('.', None)
         kwargs = {k: config[k] for k in config if valid_ident(k)}
         try:
diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py
@@ -1424,6 +1424,7 @@ def __init__(self, queue):
         """
         logging.Handler.__init__(self)
         self.queue = queue
+        self.listener = None  # will be set to listener if configured via dictConfig()
 
     def enqueue(self, record):
         """
diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
diff --git a/Misc/NEWS.d/next/Library/2022-05-26-09-24-41.gh-issue-93162.W1VuhU.rst b/Misc/NEWS.d/next/Library/2022-05-26-09-24-41.gh-issue-93162.W1VuhU.rst
