chore(integrations): internalize get_version, patch, and unpatch for all integrations [3.0] (#11916)

Requires: [chore: avoids using deprecated modules in tests and log
injection [3.0
prep]](#11835).

Follow up to: #11918

This change internalizes the following functions:
- `ddtrace.contrib.<integration_name>.patch()`
- `ddtrace.contrib.<integration_name>.unpatch()`
- `ddtrace.contrib.<integration_name>.get_version()`

This change also internalizes all integrations that only expose
`patch()`, `unpatch()`, and `get_version()` functions.

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Author: mabdinur

ddtrace/_monkey.py

@@ -176,9 +176,13 @@ def _on_import_factory(module, prefix="ddtrace.contrib", raise_errors=True, patc
 
     def on_import(hook):
         # Import and patch module
-        path = "%s.%s" % (prefix, module)
         try:
-            imported_module = importlib.import_module(path)
+            try:
+                imported_module = importlib.import_module("%s.internal.%s.patch" % (prefix, module))
+            except ImportError:
+                # Some integrations do not have an internal patch module, so we use the public one
+                # FIXME: This is a temporary solution until we refactor the patching logic.
+                imported_module = importlib.import_module("%s.%s" % (prefix, module))
             imported_module.patch()
             if hasattr(imported_module, "patch_submodules"):
                 imported_module.patch_submodules(patch_indicator)
@@ -204,7 +208,8 @@ def on_import(hook):
                     telemetry.telemetry_writer.add_integration(
                         name, True, PATCH_MODULES.get(module) is True, "", version=v
                     )
-            else:
+            elif hasattr(imported_module, "get_version"):
+                # TODO: Ensure every integration defines either get_version or get_versions in their patch.py module
                 version = imported_module.get_version()
                 telemetry.telemetry_writer.add_integration(
                     module, True, PATCH_MODULES.get(module) is True, "", version=version

ddtrace/_trace/trace_handlers.py

@@ -22,7 +22,7 @@
 from ddtrace.constants import SPAN_MEASURED_KEY
 from ddtrace.contrib import trace_utils
 from ddtrace.contrib.internal.botocore.constants import BOTOCORE_STEPFUNCTIONS_INPUT_KEY
-from ddtrace.contrib.trace_utils import _set_url_tag
+from ddtrace.contrib.internal.trace_utils import _set_url_tag
 from ddtrace.ext import SpanKind
 from ddtrace.ext import db
 from ddtrace.ext import http

ddtrace/_trace/utils_redis.py

@@ -10,7 +10,7 @@
 from ddtrace.constants import SPAN_KIND
 from ddtrace.constants import SPAN_MEASURED_KEY
 from ddtrace.contrib import trace_utils
-from ddtrace.contrib.redis_utils import _extract_conn_tags
+from ddtrace.contrib.internal.redis_utils import _extract_conn_tags
 from ddtrace.ext import SpanKind
 from ddtrace.ext import SpanTypes
 from ddtrace.ext import db

ddtrace/appsec/_handlers.py

@@ -6,8 +6,8 @@
 from ddtrace.appsec._asm_request_context import get_blocked
 from ddtrace.appsec._constants import SPAN_DATA_NAMES
 from ddtrace.contrib import trace_utils
-from ddtrace.contrib.trace_utils import _get_request_header_user_agent
-from ddtrace.contrib.trace_utils import _set_url_tag
+from ddtrace.contrib.internal.trace_utils import _get_request_header_user_agent
+from ddtrace.contrib.internal.trace_utils import _set_url_tag
 from ddtrace.ext import SpanTypes
 from ddtrace.ext import http
 from ddtrace.internal import core

ddtrace/appsec/_processor.py

@@ -101,7 +101,7 @@ def get_rules() -> str:
 
 
 def _set_headers(span: Span, headers: Any, kind: str, only_asm_enabled: bool = False) -> None:
-    from ddtrace.contrib.trace_utils import _normalize_tag_name
+    from ddtrace.contrib.internal.trace_utils import _normalize_tag_name
 
     for k in headers:
         if isinstance(k, tuple):
@@ -218,7 +218,7 @@ def rasp_sqli_enabled(self) -> bool:
         return WAF_DATA_NAMES.SQLI_ADDRESS in self._addresses_to_keep
 
     def on_span_start(self, span: Span) -> None:
-        from ddtrace.contrib import trace_utils
+        from ddtrace.contrib.internal import trace_utils
 
         if not hasattr(self, "_ddwaf"):
             self.delayed_init()

ddtrace/appsec/_trace_utils.py

@@ -11,7 +11,7 @@
 from ddtrace.appsec._constants import LOGIN_EVENTS_MODE
 from ddtrace.appsec._constants import WAF_ACTIONS
 from ddtrace.appsec._utils import _hash_user_id
-from ddtrace.contrib.trace_utils import set_user
+from ddtrace.contrib.internal.trace_utils import set_user
 from ddtrace.ext import SpanTypes
 from ddtrace.ext import user
 from ddtrace.internal import core

ddtrace/appsec/_utils.py

@@ -22,7 +22,7 @@ def parse_response_body(raw_body):
 
     from ddtrace.appsec import _asm_request_context
     from ddtrace.appsec._constants import SPAN_DATA_NAMES
-    from ddtrace.contrib.trace_utils import _get_header_value_case_insensitive
+    from ddtrace.contrib.internal.trace_utils import _get_header_value_case_insensitive
 
     if not raw_body:
         return

ddtrace/contrib/__init__.py

@@ -1,4 +1,73 @@
 from ddtrace._trace import trace_handlers  # noqa:F401
+from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning
 from ddtrace.internal.utils.importlib import func_name  # noqa:F401
 from ddtrace.internal.utils.importlib import module_name  # noqa:F401
 from ddtrace.internal.utils.importlib import require_modules  # noqa:F401
+from ddtrace.vendor.debtcollector import deprecate
+
+
+def __getattr__(name):
+    if name in (
+        "aiohttp",
+        "asgi",
+        "bottle",
+        "celery",
+        "cherrypy",
+        "falcon",
+        "flask_cache",
+        "pylibmc",
+        "pyramid",
+        "requests",
+        "sqlalchemy",
+        "wsgi",
+        "trace_utils",
+        "internal",
+    ):
+        # following packages/modules are not deprecated and will not be removed in 3.0
+        pass
+    elif name in ("trace_handlers", "func_name", "module_name", "require_modules"):
+        # the following attributes are exposed in ddtrace.contrib.__init__ and should be
+        # removed in v3.0
+        deprecate(
+            ("ddtrace.contrib.%s is deprecated" % name),
+            category=DDTraceDeprecationWarning,
+            removal_version="3.0.0",
+        )
+    elif name in ("aiobotocore", "httplib", "kombu", "snowflake", "sqlalchemy", "tornado", "urllib3"):
+        # following integrations are not enabled by default and require a unique deprecation message
+        deprecate(
+            f"ddtrace.contrib.{name} is deprecated",
+            message="Avoid using this package directly. "
+            f"Set DD_TRACE_{name.upper()}_ENABLED=true and use ``ddtrace.auto`` or the "
+            "``ddtrace-run`` command to enable and configure this integration.",
+            category=DDTraceDeprecationWarning,
+            removal_version="3.0.0",
+        )
+    elif name in ("redis_utils", "trace_utils_redis", "trace_utils_async"):
+        deprecate(
+            f"The ddtrace.contrib.{name} module is deprecated",
+            message="Import from ``ddtrace.contrib.trace_utils`` instead.",
+            category=DDTraceDeprecationWarning,
+            removal_version="3.0.0",
+        )
+    elif name == "flask_login":
+        deprecate(
+            """The flask_login integration is deprecated and will be deleted.
+        We recommend customers to switch to manual instrumentation.
+        https://docs.datadoghq.com/security/application_security/threats/add-user-info/?tab=loginsuccess&code-lang=python#adding-business-logic-information-login-success-login-failure-any-business-logic-to-traces
+        """,
+            message="",
+            category=DDTraceDeprecationWarning,
+        )
+    else:
+        deprecate(
+            f"ddtrace.contrib.{name} is deprecated",
+            message="Avoid using this package directly. "
+            f"Use ``import ddtrace.auto`` or the ``ddtrace-run`` command to enable and configure {name}.",
+            category=DDTraceDeprecationWarning,
+            removal_version="3.0.0",
+        )
+
+    if name in globals():
+        return globals()[name]
+    raise AttributeError("%s has no attribute %s", __name__, name)

ddtrace/contrib/aiobotocore/__init__.py

@@ -35,9 +35,6 @@
     _w.simplefilter("ignore", DeprecationWarning)
     from . import patch as _  # noqa: F401, I001
 
-# Expose public methods
-from ddtrace.contrib.internal.aiobotocore.patch import get_version
-from ddtrace.contrib.internal.aiobotocore.patch import patch
 
-
-__all__ = ["patch", "get_version"]
+from ddtrace.contrib.internal.aiobotocore.patch import get_version  # noqa: F401
+from ddtrace.contrib.internal.aiobotocore.patch import patch  # noqa: F401

ddtrace/contrib/aiohttp/__init__.py

@@ -94,11 +94,25 @@ async def home_handler(request):
     _w.simplefilter("ignore", DeprecationWarning)
     from . import patch as _  # noqa: F401, I001
 from ddtrace.contrib.internal.aiohttp.middlewares import trace_app
-from ddtrace.contrib.internal.aiohttp.patch import get_version
+from ddtrace.contrib.internal.aiohttp.patch import get_version  # noqa: F401
+from ddtrace.contrib.internal.aiohttp.patch import patch  # noqa: F401
+from ddtrace.contrib.internal.aiohttp.patch import unpatch  # noqa: F401
+from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning
+from ddtrace.vendor.debtcollector import deprecate
 
-# Expose public methods
-from ddtrace.contrib.internal.aiohttp.patch import patch
-from ddtrace.contrib.internal.aiohttp.patch import unpatch
 
+def __getattr__(name):
+    if name in ("patch", "get_version", "unpatch"):
+        deprecate(
+            ("%s.%s is deprecated" % (__name__, name)),
+            message="Use ``import ddtrace.auto`` or the ``ddtrace-run`` command to configure this integration.",
+            category=DDTraceDeprecationWarning,
+            removal_version="3.0.0",
+        )
 
-__all__ = ["patch", "unpatch", "trace_app", "get_version"]
+    if name in globals():
+        return globals()[name]
+    raise AttributeError("%s has no attribute %s", __name__, name)
+
+
+__all__ = ["trace_app"]

ddtrace/contrib/aiohttp_jinja2/__init__.py

@@ -23,9 +23,6 @@
     _w.simplefilter("ignore", DeprecationWarning)
     from . import patch as _  # noqa: F401, I001
 
-from ddtrace.contrib.internal.aiohttp_jinja2.patch import get_version
-from ddtrace.contrib.internal.aiohttp_jinja2.patch import patch
-from ddtrace.contrib.internal.aiohttp_jinja2.patch import unpatch
-
-
-__all__ = ["patch", "unpatch", "get_version"]
+from ddtrace.contrib.internal.aiohttp_jinja2.patch import get_version  # noqa: F401
+from ddtrace.contrib.internal.aiohttp_jinja2.patch import patch  # noqa: F401
+from ddtrace.contrib.internal.aiohttp_jinja2.patch import unpatch  # noqa: F401

ddtrace/contrib/aiomysql/__init__.py

@@ -45,9 +45,6 @@
     _w.simplefilter("ignore", DeprecationWarning)
     from . import patch as _  # noqa: F401, I001
 
-from ddtrace.contrib.internal.aiomysql.patch import get_version
-from ddtrace.contrib.internal.aiomysql.patch import patch
-from ddtrace.contrib.internal.aiomysql.patch import unpatch
-
-
-__all__ = ["patch", "unpatch", "get_version"]
+from ddtrace.contrib.internal.aiomysql.patch import get_version  # noqa: F401
+from ddtrace.contrib.internal.aiomysql.patch import patch  # noqa: F401
+from ddtrace.contrib.internal.aiomysql.patch import unpatch  # noqa: F401

ddtrace/contrib/aiopg/__init__.py

@@ -26,8 +26,5 @@
     _w.simplefilter("ignore", DeprecationWarning)
     from . import patch as _  # noqa: F401, I001
 
-from ddtrace.contrib.internal.aiopg.patch import get_version
-from ddtrace.contrib.internal.aiopg.patch import patch
-
-
-__all__ = ["patch", "get_version"]
+from ddtrace.contrib.internal.aiopg.patch import get_version  # noqa: F401
+from ddtrace.contrib.internal.aiopg.patch import patch  # noqa: F401

ddtrace/contrib/aioredis/__init__.py

@@ -1,12 +1,3 @@
-from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning
-from ddtrace.vendor.debtcollector import deprecate
-
-
-deprecate(
-    "The aioredis integration is deprecated.",
-    message="Please use the redis integration with redis>=4.2.0 instead.",
-    category=DDTraceDeprecationWarning,
-)
 """
 The aioredis integration instruments aioredis requests. Version 1.3 and above are fully
 supported.
@@ -75,11 +66,8 @@
 with _w.catch_warnings():
     _w.simplefilter("ignore", DeprecationWarning)
     # Required to allow users to import from  `ddtrace.contrib.aioredis.patch` directly
-    from . import patch as _  # noqa: F401, I001
-
-from ddtrace.contrib.internal.aioredis.patch import get_version  # noqa: E402
-from ddtrace.contrib.internal.aioredis.patch import patch  # noqa: E402
-from ddtrace.contrib.internal.aioredis.patch import unpatch  # noqa: E402
-
+    from . import patch as _  # noqa: I001,F401
 
-__all__ = ["patch", "unpatch", "get_version"]
+from ddtrace.contrib.internal.aioredis.patch import get_version  # noqa: F401
+from ddtrace.contrib.internal.aioredis.patch import patch  # noqa: F401
+from ddtrace.contrib.internal.aioredis.patch import unpatch  # noqa: F401

ddtrace/contrib/algoliasearch/__init__.py

@@ -31,9 +31,6 @@
     _w.simplefilter("ignore", DeprecationWarning)
     from . import patch as _  # noqa: F401, I001
 
-from ddtrace.contrib.internal.algoliasearch.patch import get_version
-from ddtrace.contrib.internal.algoliasearch.patch import patch
-from ddtrace.contrib.internal.algoliasearch.patch import unpatch
-
-
-__all__ = ["patch", "unpatch", "get_version"]
+from ddtrace.contrib.internal.algoliasearch.patch import get_version  # noqa: F401
+from ddtrace.contrib.internal.algoliasearch.patch import patch  # noqa: F401
+from ddtrace.contrib.internal.algoliasearch.patch import unpatch  # noqa: F401

ddtrace/contrib/anthropic/__init__.py

@@ -91,9 +91,6 @@
     _w.simplefilter("ignore", DeprecationWarning)
     from . import patch as _  # noqa: F401, I001
 
-from ddtrace.contrib.internal.anthropic.patch import get_version
-from ddtrace.contrib.internal.anthropic.patch import patch
-from ddtrace.contrib.internal.anthropic.patch import unpatch
-
-
-__all__ = ["patch", "unpatch", "get_version"]
+from ddtrace.contrib.internal.anthropic.patch import get_version  # noqa: F401
+from ddtrace.contrib.internal.anthropic.patch import patch  # noqa: F401
+from ddtrace.contrib.internal.anthropic.patch import unpatch  # noqa: F401

ddtrace/contrib/aredis/__init__.py

@@ -75,8 +75,5 @@ async def example():
     _w.simplefilter("ignore", DeprecationWarning)
     from . import patch as _  # noqa: F401, I001
 
-from ddtrace.contrib.internal.aredis.patch import get_version
-from ddtrace.contrib.internal.aredis.patch import patch
-
-
-__all__ = ["patch", "get_version"]
+from ddtrace.contrib.internal.aredis.patch import get_version  # noqa: F401
+from ddtrace.contrib.internal.aredis.patch import patch  # noqa: F401

ddtrace/contrib/asgi/__init__.py

@@ -57,8 +57,24 @@ def handle_request(scope, send):
 
 
 from ddtrace.contrib.internal.asgi.middleware import TraceMiddleware
-from ddtrace.contrib.internal.asgi.middleware import get_version
+from ddtrace.contrib.internal.asgi.middleware import get_version  # noqa: F401
 from ddtrace.contrib.internal.asgi.middleware import span_from_scope
+from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning
+from ddtrace.vendor.debtcollector import deprecate
 
 
-__all__ = ["TraceMiddleware", "span_from_scope", "get_version"]
+def __getattr__(name):
+    if name in ("get_version",):
+        deprecate(
+            ("%s.%s is deprecated" % (__name__, name)),
+            message="Use ``import ddtrace.auto`` or the ``ddtrace-run`` command to configure this integration.",
+            category=DDTraceDeprecationWarning,
+            removal_version="3.0.0",
+        )
+
+    if name in globals():
+        return globals()[name]
+    raise AttributeError("%s has no attribute %s", __name__, name)
+
+
+__all__ = ["TraceMiddleware", "span_from_scope"]

ddtrace/contrib/asyncio/__init__.py

@@ -3,23 +3,20 @@
 of concurrent execution of ``asyncio.Task``.
 """
 # Required to allow users to import from  `ddtrace.contrib.asyncio.patch` directly
-# Expose public methods
+
 import warnings as _w  # noqa:E402
 
 
 with _w.catch_warnings():
     _w.simplefilter("ignore", DeprecationWarning)
     from . import patch as _  # noqa: F401, I001
 from ddtrace._trace.provider import DefaultContextProvider
-from ddtrace.contrib.internal.asyncio.helpers import ensure_future
-from ddtrace.contrib.internal.asyncio.helpers import run_in_executor
-from ddtrace.contrib.internal.asyncio.helpers import set_call_context
-from ddtrace.contrib.internal.asyncio.patch import get_version
-from ddtrace.contrib.internal.asyncio.patch import patch
+from ddtrace.contrib.internal.asyncio.helpers import ensure_future  # noqa: F401
+from ddtrace.contrib.internal.asyncio.helpers import run_in_executor  # noqa: F401
+from ddtrace.contrib.internal.asyncio.helpers import set_call_context  # noqa: F401
+from ddtrace.contrib.internal.asyncio.patch import get_version  # noqa: F401
+from ddtrace.contrib.internal.asyncio.patch import patch  # noqa: F401
 from ddtrace.contrib.internal.asyncio.patch import unpatch  # noqa: F401
 
 
 context_provider = DefaultContextProvider()
-
-
-__all__ = ["context_provider", "set_call_context", "ensure_future", "run_in_executor", "patch", "get_version"]

ddtrace/contrib/asyncpg/__init__.py

@@ -53,9 +53,6 @@
     _w.simplefilter("ignore", DeprecationWarning)
     from . import patch as _  # noqa: F401, I001
 
-from ddtrace.contrib.internal.asyncpg.patch import get_version
-from ddtrace.contrib.internal.asyncpg.patch import patch
-from ddtrace.contrib.internal.asyncpg.patch import unpatch
-
-
-__all__ = ["patch", "unpatch", "get_version"]
+from ddtrace.contrib.internal.asyncpg.patch import get_version  # noqa: F401
+from ddtrace.contrib.internal.asyncpg.patch import patch  # noqa: F401
+from ddtrace.contrib.internal.asyncpg.patch import unpatch  # noqa: F401

ddtrace/contrib/avro/__init__.py

@@ -15,9 +15,6 @@
 ~~~~~~~~~~~~~
 
 """
-# Expose public methods
-from ..internal.avro.patch import get_version
-from ..internal.avro.patch import patch
 
-
-__all__ = ["patch", "get_version"]
+from ..internal.avro.patch import get_version  # noqa: F401
+from ..internal.avro.patch import patch  # noqa: F401