Merge branch '1.x' into feat/debugging-py311-support

Author: P403n1x87

.github/CODEOWNERS

@@ -28,6 +28,8 @@ ddtrace/appsec/                     @DataDog/asm-python
 tests/appsec/                       @DataDog/asm-python
 ddtrace/internal/remoteconfig       @DataDog/apm-core-python @DataDog/asm-python
 tests/internal/remoteconfig         @DataDog/apm-core-python @DataDog/asm-python
+tests/contrib/flask/test_flask_appsec.py    @DataDog/asm-python
+tests/contrib/django/test_django_appsec.py  @DataDog/asm-python
 
 # Profiling
 ddtrace/profiling                   @DataDog/apm-core-python @DataDog/debugger-python

ddtrace/appsec/_remoteconfiguration.py

@@ -10,8 +10,15 @@
 if TYPE_CHECKING:  # pragma: no cover
     from typing import Any
     from typing import Callable
+
+    try:
+        from typing import Literal
+    except ImportError:
+        # Python < 3.8. The "type ignore" is to avoid a runtime check just to silence mypy.
+        from typing_extensions import Literal  # type: ignore
     from typing import Mapping
     from typing import Optional
+    from typing import Union
 
     from ddtrace import Tracer
     from ddtrace.internal.remoteconfig.client import ConfigMetadata
@@ -26,9 +33,46 @@ def enable_appsec_rc():
 
     if _appsec_rc_features_is_enabled():
         from ddtrace.internal.remoteconfig import RemoteConfig
+        from ddtrace.internal.remoteconfig.constants import ASM_DATA_PRODUCT
         from ddtrace.internal.remoteconfig.constants import ASM_FEATURES_PRODUCT
 
         RemoteConfig.register(ASM_FEATURES_PRODUCT, appsec_rc_reload_features(tracer))
+        RemoteConfig.register(ASM_DATA_PRODUCT, appsec_rc_reload_features(tracer))
+
+
+def _appsec_rules_data(tracer, features):
+    # type: (Tracer, Union[Literal[False], Mapping[str, Any]]) -> None
+    if features and tracer._appsec_processor:
+        rules_data = features.get("rules_data", [])
+        if rules_data:
+            log.debug("Reloading Appsec rules data: %s", rules_data)
+            tracer._appsec_processor._update_rules(rules_data)
+
+
+def _appsec_1click_activation(tracer, features):
+    # type: (Tracer, Union[Literal[False], Mapping[str, Any]]) -> None
+    if features is False:
+        rc_appsec_enabled = False
+    else:
+        rc_appsec_enabled = features.get("asm", {}).get("enabled")
+
+    if rc_appsec_enabled is not None:
+        from ddtrace.internal.remoteconfig import RemoteConfig
+        from ddtrace.internal.remoteconfig.constants import ASM_DATA_PRODUCT
+
+        log.debug("Reloading Appsec 1-click: %s", rc_appsec_enabled)
+        _appsec_enabled = True
+
+        if not (APPSEC_ENV not in os.environ and rc_appsec_enabled) and (
+            not asbool(os.environ.get(APPSEC_ENV)) or not rc_appsec_enabled
+        ):
+            _appsec_enabled = False
+            RemoteConfig.unregister(ASM_DATA_PRODUCT)
+        else:
+            RemoteConfig.register(ASM_DATA_PRODUCT, appsec_rc_reload_features(tracer))
+
+        if tracer._appsec_enabled != _appsec_enabled:
+            tracer.configure(appsec_enabled=_appsec_enabled)
 
 
 def appsec_rc_reload_features(tracer):
@@ -50,16 +94,8 @@ def _reload_features(metadata, features):
         """
 
         if features is not None:
-            log.debug("Reloading appsec rc: %s", features)
-            rc_appsec_enabled = features.get("asm", {}).get("enabled") if features is not False else False
-
-            _appsec_enabled = True
-
-            if not (APPSEC_ENV not in os.environ and rc_appsec_enabled is True) and (
-                asbool(os.environ.get(APPSEC_ENV)) is False or rc_appsec_enabled is False
-            ):
-                _appsec_enabled = False
-
-            tracer.configure(appsec_enabled=_appsec_enabled)
+            log.debug("Updating ASM Remote Configuration: %s", features)
+            _appsec_rules_data(tracer, features)
+            _appsec_1click_activation(tracer, features)
 
     return _reload_features

ddtrace/appsec/processor.py

@@ -2,6 +2,7 @@
 import json
 import os
 import os.path
+from typing import Any
 from typing import List
 from typing import Set
 from typing import TYPE_CHECKING
@@ -210,6 +211,13 @@ def __attrs_post_init__(self):
         # we always need the response headers
         self._mark_needed(_Addresses.SERVER_RESPONSE_HEADERS_NO_COOKIES)
 
+    def _update_rules(self, new_rules):
+        # type: (List[Dict[str, Any]]) -> None
+        try:
+            self._ddwaf.update_rules(new_rules)
+        except TypeError:
+            log.debug("Error updating ASM rules", exc_info=True)
+
     def on_span_start(self, span):
         # type: (Span) -> None
         pass

ddtrace/appsec/utils.py

@@ -1,6 +1,9 @@
+import base64
 import os
+import sys
 
 from ddtrace.constants import APPSEC_ENV
+from ddtrace.internal.compat import to_bytes_py2
 from ddtrace.internal.utils.formats import asbool
 
 
@@ -13,13 +16,32 @@ def _appsec_rc_features_is_enabled():
 
 def _appsec_rc_capabilities():
     # type: () -> str
-    """return the bit of the composed capabilities in base64
+    r"""return the bit representation of the composed capabilities in base64
     bit 0: Reserved
-    bit 1: ASM Activation
+    bit 1: ASM 1-click Activation
     bit 2: ASM Ip blocking
 
-    TODO: refactor to compose the string and encode it
+    Int Number  -> binary number    -> bytes representation -> base64 representation
+    ASM Activation:
+    2           -> 10               -> b'\x02'              -> "Ag=="
+    ASM Ip blocking:
+    4           -> 100              -> b'\x04'              -> "BA=="
+    ASM Activation and ASM Ip blocking:
+    6           -> 110              -> b'\x06'              -> "Bg=="
+    ...
+    256         -> 100000000        -> b'\x01\x00'          -> b'AQA='
     """
+    value = 0b0
+
     if _appsec_rc_features_is_enabled():
-        return "Ag=="
-    return ""
+        value |= 1 << 1
+        value |= 1 << 2
+
+    result = ""
+    if sys.version_info.major < 3:
+        bytes_res = to_bytes_py2(value, (value.bit_length() + 7) // 8, "big")
+        result = base64.b64encode(bytes_res)  # type: ignore[assignment, arg-type]
+    else:
+        result = str(base64.b64encode(value.to_bytes((value.bit_length() + 7) // 8, "big")), encoding="utf-8")
+
+    return result

ddtrace/contrib/gevent/patch.py

@@ -25,6 +25,10 @@ def patch():
     This action ensures that if a user extends the ``Greenlet``
     class, the ``TracedGreenlet`` is used as a parent class.
     """
+    if getattr(gevent, "__datadog_patch", False):
+        return
+    setattr(gevent, "__datadog_patch", True)
+
     _replace(TracedGreenlet, TracedIMap, TracedIMapUnordered)
     ddtrace.tracer.configure(context_provider=GeventContextProvider())
 
@@ -35,6 +39,10 @@ def unpatch():
     before executing application code, otherwise the ``DatadogGreenlet``
     class may be used during initialization.
     """
+    if not getattr(gevent, "__datadog_patch", False):
+        return
+    setattr(gevent, "__datadog_patch", False)
+
     _replace(__Greenlet, __IMap, __IMapUnordered)
     ddtrace.tracer.configure(context_provider=DefaultContextProvider())
 

ddtrace/internal/compat.py

@@ -275,6 +275,23 @@ def maybe_stringify(obj):
     return None
 
 
+def to_bytes_py2(n, length, byteorder):
+    # type: (int, int, str) -> Text
+    """
+    Convert a string to bytes in the format expected by the remote config
+    capabilities string, considering the byteorder, which is needed
+    for Python 2.
+    """
+    if byteorder == "little":
+        order = range(length)
+    elif byteorder == "big":
+        order = reversed(range(length))  # type: ignore[assignment]
+    else:
+        raise ValueError("byteorder must be either 'little' or 'big'")
+
+    return "".join(chr((n >> i * 8) & 0xFF) for i in order)
+
+
 NoneType = type(None)
 
 BUILTIN_SIMPLE_TYPES = frozenset([int, float, str, bytes, bool, NoneType, type, long])

ddtrace/internal/remoteconfig/__init__.py

@@ -63,6 +63,13 @@ def register(cls, product, handler):
         except Exception:
             log.warning("error starting the RCM client", exc_info=True)
 
+    @classmethod
+    def unregister(cls, product):
+        try:
+            cls._worker._client.unregister_product(product)
+        except Exception:
+            log.warning("error starting the RCM client", exc_info=True)
+
     @classmethod
     def disable(cls):
         # type: () -> None

ddtrace/internal/remoteconfig/client.py

@@ -25,6 +25,8 @@
 from ddtrace.internal.runtime import container
 from ddtrace.internal.utils.time import parse_isoformat
 
+from ..utils.version import _pep440_to_semver
+
 
 if TYPE_CHECKING:  # pragma: no cover
     from typing import Callable
@@ -187,33 +189,6 @@ class RemoteConfigClient(object):
     and dispatches configurations to registered products.
     """
 
-    @staticmethod
-    def _get_version():
-        # type: () -> str
-        # The library uses a PEP 440-compliant versioning scheme, but the
-        # RCM spec requires that we use a SemVer-compliant version.
-        #
-        # However, we may have versions like:
-        #
-        #   - 1.7.1.dev3+gf258c7d9
-        #   - 1.7.1rc2.dev3+gf258c7d9
-        #
-        # Which are not Semver-compliant.
-        #
-        # The easiest fix is to replace the first occurrence of "rc" or
-        # ".dev" with "-rc" or "-dev" to make them compliant.
-        #
-        # Other than X.Y.Z, we are allowed `-<dot separated pre-release>+<build identifier>`
-        # https://semver.org/#backusnaur-form-grammar-for-valid-semver-versions
-        #
-        # e.g. 1.7.1-rc2.dev3+gf258c7d9 is valid
-        tracer_version = ddtrace.__version__
-        if "rc" in tracer_version:
-            tracer_version = tracer_version.replace("rc", "-rc", 1)
-        elif ".dev" in tracer_version:
-            tracer_version = tracer_version.replace(".dev", "-dev", 1)
-        return tracer_version
-
     def __init__(self):
         # type: () -> None
         self.id = str(uuid.uuid4())
@@ -230,7 +205,7 @@ def __init__(self):
         self._client_tracer = dict(
             runtime_id=runtime.get_runtime_id(),
             language="python",
-            tracer_version=self._get_version(),
+            tracer_version=_pep440_to_semver(),
             service=ddtrace.config.service,
             env=ddtrace.config.env,
             app_version=ddtrace.config.version,

ddtrace/internal/remoteconfig/constants.py

@@ -1,2 +1,3 @@
 ASM_FEATURES_PRODUCT = "ASM_FEATURES"
+ASM_DATA_PRODUCT = "ASM_DATA"
 REMOTE_CONFIG_AGENT_ENDPOINT = "v0.7/config"

ddtrace/internal/telemetry/writer.py

@@ -16,6 +16,7 @@
 from ..periodic import PeriodicService
 from ..runtime import get_runtime_id
 from ..service import ServiceStatus
+from ..utils.formats import asbool
 from ..utils.formats import parse_tags_str
 from ..utils.time import StopWatch
 from .data import get_application
@@ -54,11 +55,14 @@ def __init__(self, agent_url=None):
         self._integrations_queue = []  # type: List[Dict]
         self._lock = forksafe.Lock()  # type: forksafe.ResetObject
         self._forked = False  # type: bool
+        # Debug flag that enables payload debug mode.
+        self._debug = asbool(os.environ.get("DD_TELEMETRY_DEBUG", "false"))
         forksafe.register(self._fork_writer)
 
         self._headers = {
             "Content-type": "application/json",
             "DD-Telemetry-API-Version": "v1",
+            "DD-Telemetry-Debug-Enabled": str(self._debug).lower(),
         }  # type: Dict[str, str]
         additional_header_str = os.environ.get("_DD_TELEMETRY_WRITER_ADDITIONAL_HEADERS")
         if additional_header_str is not None:
@@ -254,6 +258,7 @@ def _create_telemetry_request(self, payload, payload_type, sequence_id):
             "runtime_id": get_runtime_id(),
             "api_version": "v1",
             "seq_id": sequence_id,
+            "debug": self._debug,
             "application": get_application(config.service, config.version, config.env),
             "host": get_host_info(),
             "payload": payload,

ddtrace/internal/utils/version.py

@@ -1,7 +1,10 @@
 import typing
+from typing import Optional
 
 import packaging.version
 
+from ddtrace.version import get_version
+
 
 def parse_version(version):
     # type: (str) -> typing.Tuple[int, int, int]
@@ -47,3 +50,31 @@ def parse_version(version):
         parsed.release[1] if len(parsed.release) >= 2 else 0,
         parsed.release[2] if len(parsed.release) >= 3 else 0,
     )
+
+
+def _pep440_to_semver(version=None):
+    # type: (Optional[str]) -> str
+    # The library uses a PEP 440-compliant (https://peps.python.org/pep-0440/) versioning
+    # scheme, but the RCM spec requires that we use a SemVer-compliant version.
+    #
+    # However, we may have versions like:
+    #
+    #   - 1.7.1.dev3+gf258c7d9
+    #   - 1.7.1rc2.dev3+gf258c7d9
+    #
+    # Which are not Semver-compliant.
+    #
+    # The easiest fix is to replace the first occurrence of "rc" or
+    # ".dev" with "-rc" or "-dev" to make them compliant.
+    #
+    # Other than X.Y.Z, we are allowed `-<dot separated pre-release>+<build identifier>`
+    # https://semver.org/#backusnaur-form-grammar-for-valid-semver-versions
+    #
+    # e.g. 1.7.1-rc2.dev3+gf258c7d9 is valid
+
+    tracer_version = version or get_version()
+    if "rc" in tracer_version and "-rc" not in tracer_version:
+        tracer_version = tracer_version.replace("rc", "-rc", 1)
+    elif ".dev" in tracer_version:
+        tracer_version = tracer_version.replace(".dev", "-dev", 1)
+    return tracer_version