From dd6ebee2aa27b2ecb7274b3b5f6a7db348f1abd8 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Wed, 9 Sep 2026 11:31:01 +0100 Subject: [PATCH 01/30] Add sender queue that expires oldest first --- datadog/dogstatsd/base.py | 182 +++++--- datadog/dogstatsd/sender_queue.py | 247 +++++++++++ .../test_sender_queue_benchmark.py | 406 ++++++++++++++++++ tests/unit/dogstatsd/test_statsd.py | 258 ++++++++++- 4 files changed, 1032 insertions(+), 61 deletions(-) create mode 100644 datadog/dogstatsd/sender_queue.py create mode 100644 tests/performance/test_sender_queue_benchmark.py diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index 5922e1183..fc969d953 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -23,12 +23,6 @@ if sys.version_info[:2] >= (3, 5): from typing import TYPE_CHECKING # noqa: F401 -try: - import queue -except ImportError: - # pypy has the same module, but capitalized. - import Queue as queue # type: ignore[no-redef] - # pylint: disable=unused-import if sys.version_info[:2] >= (3, 5): @@ -49,6 +43,13 @@ ) from datadog.dogstatsd.route import get_default_route from datadog.dogstatsd.container import Cgroup +from datadog.dogstatsd.sender_queue import ( + SenderQueue, + PendingPayload, + Stop, + PENDING_PAYLOAD_EXPIRY_SECONDS, + coalesce_enqueue_time, +) from datadog.util.compat import text, urlparse from datadog.util.format import normalize_tags, validate_cardinality from datadog.version import __version__ @@ -224,8 +225,6 @@ def reverse(self): ] ) + "\n" -Stop = object() - SUPPORTS_FORKING = hasattr(os, "register_at_fork") and not os.environ.get("DD_DOGSTATSD_DISABLE_FORK_SUPPORT", None) TRACK_INSTANCES = not os.environ.get("DD_DOGSTATSD_DISABLE_INSTANCE_TRACKING", None) @@ -490,16 +489,19 @@ def __init__( Default: True. :type disable_background_sender: boolean - :param sender_queue_size: Set the maximum number of packets to queue for the sender. Optional - How may packets to queue before blocking or dropping the packet if the packet queue is already full. + :param sender_queue_size: Set the maximum number of packets to queue for the sender. Optional. + Once the queue is full, adding a new packet drops the oldest queued packet (and any additional + expired packets at the front of the queue) to make room, instead of blocking or dropping the new + packet. Packets aren't held indefinitely either: a queued packet that hasn't been sent within + PENDING_PAYLOAD_EXPIRY_SECONDS is dropped when it's pulled off the queue, unless it carries its own + explicit timestamp (e.g. gauge_with_timestamp, or count/service_check/event with an explicit + timestamp), in which case it's kept until it can actually be sent. Default: 0 (unlimited). :type sender_queue_size: integer - :param sender_queue_timeout: Set timeout for packet queue operations, in seconds. Optional. - How long the application thread is willing to wait for the queue clear up before dropping the metric packet. - If set to None, wait forever. - If set to zero drop the packet immediately if the queue is full. - Default: 0 (no wait) + :param sender_queue_timeout: Deprecated and ignored. The sender queue no longer blocks: it always + makes room for a new packet by dropping older or expired entries instead. Kept only for backwards + compatibility with existing call sites. :type sender_queue_timeout: float :param track_instance: Keep track of this instance and automatically handle cleanup when os.fork() is called, @@ -634,7 +636,7 @@ def __init__( else: log.debug("Statsd buffering and aggregation is disabled") - self._queue = None # type: Optional[queue.Queue[Union[str, object]]] + self._queue = None # type: Optional[SenderQueue] self._sender_thread = None # type: Optional[threading.Thread] self._sender_enabled = False @@ -716,25 +718,19 @@ def enable_background_sender(self, sender_queue_size=0, sender_queue_timeout=0): to os.fork(). :param sender_queue_size: Set the maximum number of packets to queue for the sender. - How many packets to queue before blocking or dropping the packet if the packet queue is already full. + Once the queue is full, adding a new packet drops the oldest queued packet (and any additional + expired packets at the front of the queue) to make room, instead of blocking or dropping the new + packet. Default: 0 (unlimited). :type sender_queue_size: integer, optional - :param sender_queue_timeout: Set timeout for packet queue operations, in seconds. - How long the application thread is willing to wait for the queue clear up before dropping the metric packet. - If set to None, wait forever. If set to zero drop the packet immediately if the queue is full. - Default: 0 (no wait). + :param sender_queue_timeout: Deprecated and ignored: the sender queue no longer blocks. Kept only + for backwards compatibility with existing call sites. :type sender_queue_timeout: float, optional """ with self._config_lock: self._sender_enabled = True self._sender_queue_size = sender_queue_size - if sender_queue_timeout is None: - self._queue_blocking = True - self._queue_timeout = None - else: - self._queue_blocking = sender_queue_timeout > 0 - self._queue_timeout = max(0, sender_queue_timeout) self._start_sender_thread() @@ -1169,6 +1165,10 @@ def _reset_buffer(self): with self._buffer_lock: self._current_buffer_total_size = 0 self._buffer = [] + # A freshly (re)started buffer starts out replay-safe; it's + # downgraded to False as soon as anything not-replay-safe is + # appended to it. See _send_to_buffer(). + self._buffer_replay_safe = True def flush(self): # type: () -> None @@ -1182,7 +1182,7 @@ def flush_buffered_metrics(self): with self._buffer_lock: # Only send packets if there are packets to send if self._buffer: - self._send_to_server("\n".join(self._buffer)) + self._send_to_server("\n".join(self._buffer), self._buffer_replay_safe) self._reset_buffer() def flush_aggregated_metrics(self): @@ -1569,8 +1569,13 @@ def _report(self, metric, metric_type, value, tags, sample_rate, timestamp=0, sa metric, metric_type, value, tags, sample_rate, timestamp, cardinality ) + # A metric carrying its own explicit timestamp is replay-safe: sending + # it late (e.g. after sitting in the background sender queue) doesn't + # change what it means. + replay_safe = timestamp > 0 + # Send it - self._send(payload) + self._send(payload, replay_safe) def _reset_telemetry(self): # type: () -> None @@ -1580,21 +1585,35 @@ def _reset_telemetry(self): self.bytes_sent = 0 self.bytes_dropped_queue = 0 self.bytes_dropped_writer = 0 + self.bytes_dropped_expired = 0 self.packets_sent = 0 self.packets_dropped_queue = 0 self.packets_dropped_writer = 0 + self.packets_dropped_expired = 0 self._last_flush_time = time.time() # Aliases for backwards compatibility. @property def packets_dropped(self): # type: () -> int - return self.packets_dropped_queue + self.packets_dropped_writer + return self.packets_dropped_queue + self.packets_dropped_writer + self.packets_dropped_expired @property def bytes_dropped(self): # type: () -> int - return self.bytes_dropped_queue + self.bytes_dropped_writer + return self.bytes_dropped_queue + self.bytes_dropped_writer + self.bytes_dropped_expired + + def _account_dropped_queue_full(self, item): + # type: (PendingPayload) -> None + """A payload was evicted from the sender queue to make room for a new one.""" + self.packets_dropped_queue += 1 + self.bytes_dropped_queue += len(item.payload.encode(self.encoding)) + + def _account_dropped_expired(self, item): + # type: (PendingPayload) -> None + """A payload sat in the sender queue longer than PENDING_PAYLOAD_EXPIRY_SECONDS.""" + self.packets_dropped_expired += 1 + self.bytes_dropped_expired += len(item.payload.encode(self.encoding)) def _flush_telemetry(self): # type: () -> str @@ -1633,26 +1652,34 @@ def _is_telemetry_flush_time(self): return self._telemetry and \ self._last_flush_time + self._telemetry_flush_interval < time.time() - def _send_to_server(self, packet): - # type: (str) -> None + def _send_to_server(self, packet, replay_safe=False): + # type: (str, bool) -> None # Skip the lock if the queue is None. There is no race with enable_background_sender. if self._queue is not None: # Prevent a race with disable_background_sender. with self._buffer_lock: packet_with_newline = packet + '\n' if self._queue is not None: - try: - self._queue.put(packet_with_newline, self._queue_blocking, self._queue_timeout) - except queue.Full: - self.packets_dropped_queue += 1 - self.bytes_dropped_queue += len(packet_with_newline.encode(self.encoding)) + # replay_safe payloads never have their enqueued_at read + # (see SenderQueue._expired()'s short-circuit), so skip + # both the clock read and the float allocation for them. + enqueued_at = None if replay_safe else coalesce_enqueue_time() + self._queue.put(PendingPayload(packet_with_newline, enqueued_at, replay_safe)) return self._xmit_packet_with_telemetry(packet + '\n') - def _xmit_packet_with_telemetry(self, packet): - # type: (str) -> None - self._xmit_packet(packet, False) + def _xmit_packet_with_telemetry(self, packet, queue_mode=False): + # type: (str, bool) -> Optional[bool] + """Send one packet, optionally piggy-backing a telemetry flush. + + :param queue_mode: True when called from the background sender + thread on behalf of a queued PendingPayload. In that mode, a + connection failure is reported back as None (rather than being + accounted for and dropped) so the caller can requeue the payload + and retry once reconnected, instead of losing it. + """ + sent = self._xmit_packet(packet, False, queue_mode=queue_mode) if self._is_telemetry_flush_time(): telemetry = self._flush_telemetry() @@ -1666,6 +1693,8 @@ def _xmit_packet_with_telemetry(self, packet): self.bytes_dropped_writer += len(telemetry) self.packets_dropped_writer += 1 + return sent + def _installed_socket(self, is_telemetry): # type: (bool) -> Optional[_Socket] """ @@ -1680,8 +1709,16 @@ def _installed_socket(self, is_telemetry): return self.telemetry_socket return self.socket - def _xmit_packet(self, packet, is_telemetry): - # type: (str, bool) -> bool + def _xmit_packet(self, packet, is_telemetry, queue_mode=False): + # type: (str, bool, bool) -> Optional[bool] + """Attempt to send packet, retrying a reconnect within this call as budget allows. + + Returns True if sent. Otherwise returns False for a definitive, + non-retryable failure (already accounted for as a dropped packet), + or -- only when queue_mode is True -- None for a connection failure + that the sender queue should retry by requeuing the payload rather + than have accounted for here as a drop. + """ if is_telemetry and self._dedicated_telemetry_destination(): uses_uds = self.telemetry_socket_path is not None @@ -1693,6 +1730,7 @@ def _xmit_packet(self, packet, is_telemetry): retry_deadline = time.time() + self.socket_connect_timeout backoff = UDS_CONNECT_RETRY_INITIAL_BACKOFF + sent = None # type: Optional[bool] while True: # Cheap fast-path check before even trying to acquire _socket_lock. if ( @@ -1704,6 +1742,7 @@ def _xmit_packet(self, packet, is_telemetry): "Gave up reconnecting after socket_connect_timeout (%ss), dropping the packet", self.socket_connect_timeout, ) + sent = None break sent = self._xmit_packet_attempt( @@ -1730,6 +1769,12 @@ def _xmit_packet(self, packet, is_telemetry): time.sleep(min(backoff, remaining)) backoff = min(backoff * 2, UDS_CONNECT_RETRY_MAX_BACKOFF) + if sent is None and queue_mode: + # Connection trouble, and the caller is the background sender + # queue: let it requeue the payload and retry once reconnected, + # instead of dropping it here. + return None + if not is_telemetry and self._telemetry: self.bytes_dropped_writer += len(packet) self.packets_dropped_writer += 1 @@ -1836,8 +1881,8 @@ def _xmit_packet_attempt(self, packet, is_telemetry, retry_eligible, retry_deadl return False - def _send_to_buffer(self, packet): - # type: (str) -> None + def _send_to_buffer(self, packet, replay_safe=False): + # type: (str, bool) -> None with self._buffer_lock: if self._should_flush(len(packet)): self.flush_buffered_metrics() @@ -1846,6 +1891,10 @@ def _send_to_buffer(self, packet): # Update the current buffer length, including line break to anticipate # the final packet size self._current_buffer_total_size += len(packet) + 1 + # The flushed batch is only as replay-safe as its least safe + # member: if anything in it needs to be treated as time-sensitive, + # treat the whole batch that way. + self._buffer_replay_safe = self._buffer_replay_safe and replay_safe def _should_flush(self, length_to_be_added): # type: (int) -> bool @@ -1938,7 +1987,9 @@ def event( if self._telemetry: self.events_count += 1 - self._send(string) + # An event carrying its own explicit date_happened is replay-safe: + # sending it late doesn't change what it means. + self._send(string, replay_safe=bool(date_happened)) def service_check( self, @@ -1984,7 +2035,9 @@ def service_check( if self._telemetry: self.service_checks_count += 1 - self._send(string) + # A service check carrying its own explicit timestamp is replay-safe: + # sending it late doesn't change what it means. + self._send(string, replay_safe=bool(timestamp)) @staticmethod def _normalize_and_join_tags(tags): @@ -2062,7 +2115,12 @@ def _start_sender_thread(self): if self._queue is not None: return - self._queue = queue.Queue(self._sender_queue_size) + self._queue = SenderQueue( + self._sender_queue_size, + PENDING_PAYLOAD_EXPIRY_SECONDS, + self._account_dropped_queue_full, + self._account_dropped_expired, + ) log.debug("Starting background sender thread") self._sender_thread = threading.Thread( @@ -2086,19 +2144,35 @@ def _stop_sender_thread(self): self._sender_thread.join() self._sender_thread = None - def _sender_main_loop(self, queue): - # type: (queue.Queue[Union[str, object]]) -> None + def _sender_main_loop(self, pending_queue): + # type: (SenderQueue) -> None + backoff = UDS_CONNECT_RETRY_INITIAL_BACKOFF while True: - item = queue.get() + item = pending_queue.get() if item is Stop: - queue.task_done() + pending_queue.task_done() return # next line has type ignore because the type checker cannot # know that 'if item is Stop' is the only case where item is # of object type. - self._xmit_packet_with_telemetry(item) # type: ignore[arg-type] # noqa: F821 - queue.task_done() + sent = self._xmit_packet_with_telemetry(item.payload, queue_mode=True) # type: ignore[attr-defined] # noqa: F821 + + if sent is None: + # Connection trouble: keep the payload for the next attempt + # instead of losing it. The queue's own expiry check (on a + # future get()) is what eventually gives up on a payload + # that's been stuck for too long, unless it's replay-safe. + pending_queue.requeue_front(item) # type: ignore[arg-type] + time.sleep(backoff) + backoff = min(backoff * 2, UDS_CONNECT_RETRY_MAX_BACKOFF) + continue + + # Sent, or a definitive failure that _xmit_packet already + # accounted for as a dropped packet -- either way, this + # payload's story is over. + pending_queue.task_done() + backoff = UDS_CONNECT_RETRY_INITIAL_BACKOFF def wait_for_pending(self): # type: () -> None diff --git a/datadog/dogstatsd/sender_queue.py b/datadog/dogstatsd/sender_queue.py new file mode 100644 index 000000000..8cf202ade --- /dev/null +++ b/datadog/dogstatsd/sender_queue.py @@ -0,0 +1,247 @@ +import collections +import sys +import threading + +try: + # Python 3.3+ + from time import monotonic +except ImportError: + # Python 2: no monotonic clock available, fall back to wall clock. + from time import time as monotonic + +if sys.version_info[:2] >= (3, 5): + from typing import Callable, Optional, Union # noqa: F401 + + +# Sentinel telling the background sender thread to shut down. +Stop = object() + +# How long (in seconds) a non-replay-safe payload may sit in the background +# sender queue before it's considered stale and dropped instead of sent. +# Payloads that carry their own explicit timestamp (replay_safe) are exempt: +# delivering those late doesn't change what they mean, so they're kept +# around until they can actually be sent. +PENDING_PAYLOAD_EXPIRY_SECONDS = 10.0 + +# Granularity for coalesce_enqueue_time() below. Deliberately far below +# PENDING_PAYLOAD_EXPIRY_SECONDS (by two orders of magnitude with the default +# above), so it has no meaningful effect on expiry accuracy, but lets many +# payloads enqueued within the same short window share one float object +# instead of each allocating their own -- which is exactly when it matters: +# under sustained load or a backlog, not when the queue is lightly used. +_TIMESTAMP_COALESCE_SECONDS = 0.1 + +# Bucket + cached value for coalesce_enqueue_time(). Plain module globals, +# not a lock: under a race between threads, the worst outcome is a +# redundant allocation (two threads each compute a fresh reading for the +# same bucket), never an incorrect timestamp. +_coalesce_bucket = None # type: Optional[int] +_coalesce_value = 0.0 # type: float + + +def coalesce_enqueue_time(): + # type: () -> float + """A monotonic() reading coalesced to _TIMESTAMP_COALESCE_SECONDS granularity. + + Only meant for stamping payloads that DO need expiry tracking (see + PendingPayload.enqueued_at). The slop this introduces (at most one + bucket width, 0.1s by default) is negligible next to the multi-second + expiry window it feeds into. + """ + global _coalesce_bucket, _coalesce_value + raw = monotonic() + bucket = int(raw / _TIMESTAMP_COALESCE_SECONDS) + if bucket != _coalesce_bucket: + _coalesce_bucket = bucket + _coalesce_value = raw + return _coalesce_value + + +class PendingPayload(object): + """A single packet queued for the background sender. + + :ivar payload: The already-serialized packet text (including its + trailing newline), ready to be written to the socket. + :ivar enqueued_at: A monotonic timestamp recorded when the payload + became eligible for sending (i.e. when it was put on the queue). + Used to decide whether it has been sitting in the queue for too + long to still be worth sending. None when replay_safe is True: it's + never read in that case (see SenderQueue._expired()'s short-circuit), + so skipping the allocation costs nothing. + :ivar replay_safe: True when delayed delivery preserves the payload's + meaning because it carries its own explicit timestamp. Such + payloads are never dropped for being stale, and never need + enqueued_at. + """ + + __slots__ = ("payload", "enqueued_at", "replay_safe") + + def __init__(self, payload, enqueued_at, replay_safe): + # type: (str, Optional[float], bool) -> None + self.payload = payload + self.enqueued_at = enqueued_at + self.replay_safe = replay_safe + + +class SenderQueue(object): + """Bounded hand-off queue between application threads and the background sender thread. + + Unlike queue.Queue, put() never blocks and never rejects a payload. When + the queue is already at its maximum size, the oldest entry is dropped to + make room, along with any additional expired entries left at the front, + so a backlog of stale payloads can't shut out fresh metrics indefinitely. + + get() drops expired entries lazily too, from the front, before returning + the next payload actually worth handing to the sender. + + A payload that fails to send (e.g. because the connection is down) can be + handed back with requeue_front() so it's retried first. That still + respects both the expiry check and the size limit though: the queue + must never grow past maxsize, and a payload that's gone stale while it + was being (re)tried is dropped rather than requeued. + """ + + def __init__(self, maxsize, expiry_seconds, on_drop_queue_full, on_drop_expired): + # type: (int, float, Callable[[PendingPayload], None], Callable[[PendingPayload], None]) -> None + self._maxsize = maxsize + self._expiry_seconds = expiry_seconds + self._on_drop_queue_full = on_drop_queue_full + self._on_drop_expired = on_drop_expired + self._deque = collections.deque() # type: collections.deque + self._lock = threading.Lock() + self._not_empty = threading.Condition(self._lock) + self._all_tasks_done = threading.Condition(self._lock) + + # Keep track of the tasks that are being processed. A task pulled from the queue may + # be returned if the connection fails, so we don't consider the queue empty until + # all tasks have been dropped or sent. + self._unfinished_tasks = 0 + + def _expired(self, item, now): + # type: (PendingPayload, float) -> bool + if item.replay_safe: + return False + # enqueued_at is only ever None for replay_safe items (see + # PendingPayload), which are already excluded above -- it's a plain + # float here. mypy can't correlate that invariant across the two + # attributes, hence the ignore. + return (now - item.enqueued_at) > self._expiry_seconds # type: ignore[operator] + + def _make_room_locked(self): + # type: () -> None + """Drop the oldest entry, plus any further expired entries at the front. + + Called with self._lock already held, and only when the queue is at + capacity. Never touches the Stop sentinel: by the time it's queued, + nothing else is ever put on the queue again, so it can only ever be + the newest entry, never the one being evicted here. + """ + if not self._deque or self._deque[0] is Stop: + return + + now = monotonic() + oldest = self._deque.popleft() + # The oldest entry is always dropped to make room. If it happens to + # also be expired, attribute it to staleness rather than to the + # queue being full, since that's the more useful signal. + if self._expired(oldest, now): + self._on_drop_expired(oldest) + else: + self._on_drop_queue_full(oldest) + self._finish_task_locked() + + # Keep clearing out additional stale entries left at the front: they + # would otherwise just sit there consuming a slot until they're + # eventually popped. + while self._deque and self._deque[0] is not Stop and self._expired(self._deque[0], now): + self._on_drop_expired(self._deque.popleft()) + self._finish_task_locked() + + def put(self, item): + # type: (Union[PendingPayload, object]) -> None + """Queue a payload (or the Stop sentinel), evicting old entries if needed.""" + with self._not_empty: + if item is not Stop and self._maxsize > 0 and len(self._deque) >= self._maxsize: + self._make_room_locked() + + self._deque.append(item) + self._unfinished_tasks += 1 + self._not_empty.notify() + + def requeue_front(self, item): + # type: (PendingPayload) -> None + """Put an in-flight payload back at the front after a failed send attempt. + + The payload was already accounted for by the put() that originally + queued it (its task isn't done yet), so a successful requeue here + doesn't touch _unfinished_tasks. But it's still subject to the same + rules as any other entry: an item that's expired while it was being + (re)tried is dropped instead of requeued, and the queue is never + allowed to grow past maxsize -- if it's already full, the requeue is + dropped too rather than evicting something else to make room for it. + Either way, a drop here finishes the task that put() started. + """ + with self._not_empty: + if self._expired(item, monotonic()): + self._on_drop_expired(item) + self._finish_task_locked() + return + + if self._maxsize > 0 and len(self._deque) >= self._maxsize: + self._on_drop_queue_full(item) + self._finish_task_locked() + return + + self._deque.appendleft(item) + self._not_empty.notify() + + def get(self): + # type: () -> Union[PendingPayload, object] + """Block for the next payload, silently dropping expired entries along the way.""" + while True: + with self._not_empty: + while not self._deque: + self._not_empty.wait() + item = self._deque.popleft() + + if item is Stop: + return item + + if self._expired(item, monotonic()): + self._on_drop_expired(item) + self.task_done() + continue + + return item + + def _finish_task_locked(self): + # type: () -> None + # Caller already holds self._lock (shared by _not_empty / _all_tasks_done). + unfinished = self._unfinished_tasks - 1 + if unfinished < 0: + raise ValueError("task_done() called too many times") + self._unfinished_tasks = unfinished + if unfinished == 0: + self._all_tasks_done.notify_all() + + def task_done(self): + # type: () -> None + with self._all_tasks_done: + self._finish_task_locked() + + def join(self): + # type: () -> None + with self._all_tasks_done: + while self._unfinished_tasks: + self._all_tasks_done.wait() + + def qsize(self): + # type: () -> int + with self._lock: + return len(self._deque) + + def empty(self): + # type: () -> bool + with self._lock: + return not self._deque + diff --git a/tests/performance/test_sender_queue_benchmark.py b/tests/performance/test_sender_queue_benchmark.py new file mode 100644 index 000000000..b755c569f --- /dev/null +++ b/tests/performance/test_sender_queue_benchmark.py @@ -0,0 +1,406 @@ +""" +Microbenchmark: SenderQueue vs. stdlib queue.Queue. + +This isolates just the hand-off queue's own overhead -- no sockets, no +network variance -- because that's the piece that changed when the +background sender moved off queue.Queue. put() runs synchronously on every +metric emission's calling thread (the application's hot path), so its +*latency* matters at least as much as raw throughput; get() runs on the +background sender thread. + +queue.Queue is used as the baseline throughout via a thin adapter +(_OldStyleQueueAdapter) that reproduces the OLD behavior being replaced: +put_nowait() and drop-with-a-counter on queue.Full, get()+task_done() to +drain. That's the fairest apples-to-apples comparison, since it's literally +what SenderQueue's put()/get() replaced. + +Scenarios: + 1. Unbounded put() then get(), single-threaded (best case for both -- + no eviction, no contention). + 2. Bounded queue, kept permanently full: every put() forces an eviction + for SenderQueue, vs an immediate reject-with-exception for + queue.Queue. This is the main new cost the redesign introduces. + 3. A single put() that has to walk past a large backlog of already-EXPIRED + entries at the front (SenderQueue's opportunistic-cleanup loop has no + upper bound tied to "just free one slot" -- it clears every consecutive + stale entry it finds). Reports cost as a function of backlog size, to + surface whether this can spike a calling thread's latency. + 4. Producer/consumer concurrency: N producer threads hammering put() while + 1 consumer thread drains, measuring achieved producer throughput and + put() latency percentiles under real lock contention. + 5. Per-item memory footprint: PendingPayload wrapper vs a bare str. + +Usage: + python3 tests/performance/test_sender_queue_benchmark.py [--quick] + + --quick shrinks every N so it finishes in a few seconds (CI-friendly); + default sizes are big enough to get low-noise numbers on a quiet + machine. + +This prints numbers and interpretation guidance; it does not hard-fail on +absolute thresholds (those are too hardware/noise dependent to gate CI on +reliably). The one thing it does assert on is the *shape* of the eviction +cost in scenario 3 -- that it's linear in backlog size, not something worse. +Read the printed numbers yourself before/after a change and compare. +""" +import os +import sys +import threading +import time + +try: + import queue as stdlib_queue +except ImportError: + import Queue as stdlib_queue # type: ignore[no-redef] + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + +from datadog.dogstatsd.sender_queue import ( # noqa: E402 + PendingPayload, + SenderQueue, + coalesce_enqueue_time, + monotonic, +) + +QUICK = "--quick" in sys.argv + + +def section(title): + print() + print("=" * 78) + print(title) + print("=" * 78) + + +def note(msg): + print(" . {}".format(msg)) + + +PACKET = "some.metric.name:1|c|#tag1:val1,tag2:val2\n" + + +# -------------------------------------------------------------------------- +# Baseline adapter: reproduces the OLD (pre-SenderQueue) put/get contract on +# top of stdlib queue.Queue, so scenario code can treat both implementations +# uniformly. +# -------------------------------------------------------------------------- +class _OldStyleQueueAdapter(object): + def __init__(self, maxsize): + self._q = stdlib_queue.Queue(maxsize) + self.dropped = 0 + + def put(self, item): + try: + self._q.put_nowait(item) + except stdlib_queue.Full: + self.dropped += 1 + + def get(self): + return self._q.get() + + def task_done(self): + self._q.task_done() + + def qsize(self): + return self._q.qsize() + + +def make_sender_queue(maxsize, expiry_seconds=3600.0): + drops = {"full": 0, "expired": 0} + + def on_full(_item): + drops["full"] += 1 + + def on_expired(_item): + drops["expired"] += 1 + + q = SenderQueue(maxsize, expiry_seconds, on_full, on_expired) + q.drops = drops + return q + + +def percentiles(samples_us): + samples_us = sorted(samples_us) + n = len(samples_us) + + def pct(p): + idx = min(n - 1, int(n * p)) + return samples_us[idx] + + return { + "p50": pct(0.50), + "p90": pct(0.90), + "p99": pct(0.99), + "max": samples_us[-1], + } + + +def time_puts(put_fn, n): + """Time n individual put() calls, returning (total_seconds, [latency_us, ...]).""" + samples = [0.0] * n + t_start = time.perf_counter() + for i in range(n): + t0 = time.perf_counter() + put_fn() + samples[i] = (time.perf_counter() - t0) * 1e6 + total = time.perf_counter() - t_start + return total, samples + + +def report(label, n, total_seconds, samples_us): + p = percentiles(samples_us) + print( + " {:<28s} ops/sec={:>10,.0f} p50={:>7.3f}us p90={:>7.3f}us p99={:>7.3f}us max={:>9.3f}us".format( + label, n / total_seconds, p["p50"], p["p90"], p["p99"], p["max"] + ) + ) + return p + + +# -------------------------------------------------------------------------- +# Scenario 1: unbounded, no contention, no eviction. +# -------------------------------------------------------------------------- +def scenario_1_unbounded_single_threaded(): + section("1. Unbounded put()/get(), single-threaded (best case, no eviction)") + n = 20000 if not QUICK else 2000 + + old = _OldStyleQueueAdapter(maxsize=0) + total, samples = time_puts(lambda: old.put(PACKET), n) + report("queue.Queue (baseline)", n, total, samples) + for _ in range(n): + old.get() + old.task_done() + + new = make_sender_queue(maxsize=0) + total, samples = time_puts(lambda: new.put(PendingPayload(PACKET, coalesce_enqueue_time(), False)), n) + new_p = report("SenderQueue", n, total, samples) + for _ in range(n): + new.get() + new.task_done() + + note("SenderQueue p99 put() latency: {:.3f}us for {:,} plain puts with headroom to spare".format(new_p["p99"], n)) + + +# -------------------------------------------------------------------------- +# Scenario 2: bounded queue, kept permanently full -- every put() evicts. +# -------------------------------------------------------------------------- +def scenario_2_sustained_overflow(): + section("2. Bounded queue kept permanently full: every put() forces eviction (new cost)") + n = 20000 if not QUICK else 2000 + maxsize = 8 + + old = _OldStyleQueueAdapter(maxsize=maxsize) + for _ in range(maxsize): + old.put(PACKET) + total, samples = time_puts(lambda: old.put(PACKET), n) + old_p = report("queue.Queue (baseline)", n, total, samples) + note("queue.Queue just rejects with an exception when full -- O(1), no eviction work at all") + + new = make_sender_queue(maxsize=maxsize) + for _ in range(maxsize): + new.put(PendingPayload(PACKET, coalesce_enqueue_time(), False)) + total, samples = time_puts(lambda: new.put(PendingPayload(PACKET, coalesce_enqueue_time(), False)), n) + new_p = report("SenderQueue", n, total, samples) + + ratio = new_p["p99"] / old_p["p99"] if old_p["p99"] else float("inf") + note("SenderQueue's drop-oldest-and-evict costs {:.1f}x queue.Queue's reject-with-exception at p99".format(ratio)) + note("(each put() here evicts exactly one item -- the mandatory oldest -- since nothing is expired)") + + +# -------------------------------------------------------------------------- +# Scenario 3: one put() that has to walk past a large expired backlog. +# -------------------------------------------------------------------------- +def scenario_3_large_expired_backlog(): + section("3. Cost of ONE put() as a function of an already-expired backlog size") + note("SenderQueue's opportunistic cleanup has no cap tied to 'free just one slot': it clears") + note("every consecutive stale entry at the front. This measures whether that can spike latency.") + + backlog_sizes = [1, 10, 100, 1000, 5000] if not QUICK else [1, 10, 100] + results = [] + for backlog in backlog_sizes: + # expiry_seconds=0 with a backdated enqueued_at makes every backlog + # entry expired the instant it's queued. maxsize=backlog (exactly + # full) so the next put() below is what actually triggers eviction. + q = make_sender_queue(maxsize=backlog, expiry_seconds=0.0) + stale_at = monotonic() - 1000.0 + for _ in range(backlog): + q.put(PendingPayload(PACKET, stale_at, False)) + + t0 = time.perf_counter() + q.put(PendingPayload(PACKET, coalesce_enqueue_time(), False)) + elapsed_us = (time.perf_counter() - t0) * 1e6 + + note("backlog={:>5d} stale entries -> single put() took {:>9.3f}us, evicted {:d}".format( + backlog, elapsed_us, q.drops["expired"] + q.drops["full"] + )) + results.append((backlog, elapsed_us)) + + # Sanity check on the *shape*: cost should scale roughly linearly with + # backlog size, not blow up super-linearly. Compare the per-entry cost + # at the smallest and largest backlog sizes; allow a generous margin for + # fixed overhead and noise, but a large deviation would indicate a real + # algorithmic problem worth investigating. + (small_n, small_us), (large_n, large_us) = results[1], results[-1] + small_per_entry = small_us / small_n + large_per_entry = large_us / large_n + ratio = large_per_entry / small_per_entry if small_per_entry else float("inf") + note( + "per-entry eviction cost: {:.3f}us/entry at backlog={} vs {:.3f}us/entry at backlog={} (ratio={:.2f}x)".format( + small_per_entry, small_n, large_per_entry, large_n, ratio + ) + ) + if ratio > 5.0: + print(" [WARN] per-entry eviction cost grew by {:.1f}x from a small to a large backlog".format(ratio)) + print(" -- that's worse than linear; investigate before shipping.") + else: + print(" [OK] per-entry eviction cost stayed roughly flat as backlog size grew (linear, as expected)") + note("takeaway: a single put() CAN take noticeably longer if a huge stale backlog piles up (e.g. a") + note("long outage with a very large sender_queue_size). Keep sender_queue_size sized to what you're") + note("actually willing to let one put() walk through in the worst case.") + + +# -------------------------------------------------------------------------- +# Scenario 4: producer/consumer concurrency. +# -------------------------------------------------------------------------- +def _run_concurrent(put_fn, get_and_ack_fn, n_producers, n_per_producer, duration_cap=15.0): + latencies = [] + latencies_lock = threading.Lock() + stop = threading.Event() + + def producer(): + local_latencies = [] + for _ in range(n_per_producer): + t0 = time.perf_counter() + put_fn() + local_latencies.append((time.perf_counter() - t0) * 1e6) + with latencies_lock: + latencies.extend(local_latencies) + + def consumer(): + while not stop.is_set(): + get_and_ack_fn() + + consumer_thread = threading.Thread(target=consumer) + consumer_thread.daemon = True + consumer_thread.start() + + producers = [threading.Thread(target=producer) for _ in range(n_producers)] + t0 = time.perf_counter() + for p in producers: + p.start() + for p in producers: + p.join(timeout=duration_cap) + elapsed = time.perf_counter() - t0 + stop.set() + + total_ops = n_producers * n_per_producer + return elapsed, total_ops, latencies + + +def scenario_4_concurrency(): + section("4. Producer/consumer concurrency: N producers hammering put(), 1 consumer draining") + n_producers = 4 + n_per_producer = 5000 if not QUICK else 500 + + old = _OldStyleQueueAdapter(maxsize=1000) + elapsed, total_ops, samples = _run_concurrent( + lambda: old.put(PACKET), + lambda: (old.get(), old.task_done()), + n_producers, + n_per_producer, + ) + old_p = report("queue.Queue (baseline)", total_ops, elapsed, samples) + note("queue.Queue: {} producers x {} puts in {:.3f}s, {} dropped-on-full".format( + n_producers, n_per_producer, elapsed, old.dropped + )) + + new = make_sender_queue(maxsize=1000) + elapsed, total_ops, samples = _run_concurrent( + lambda: new.put(PendingPayload(PACKET, coalesce_enqueue_time(), False)), + lambda: (new.get(), new.task_done()), + n_producers, + n_per_producer, + ) + new_p = report("SenderQueue", total_ops, elapsed, samples) + note("SenderQueue: {} producers x {} puts in {:.3f}s, {} dropped-full, {} dropped-expired".format( + n_producers, n_per_producer, elapsed, new.drops["full"], new.drops["expired"] + )) + + ratio_p99 = new_p["p99"] / old_p["p99"] if old_p["p99"] else float("inf") + note("under real thread contention, SenderQueue's p99 put() latency is {:.2f}x queue.Queue's".format(ratio_p99)) + + +# -------------------------------------------------------------------------- +# Scenario 5: per-item memory footprint. +# -------------------------------------------------------------------------- +def scenario_5_memory_footprint(): + section("5. Per-item memory footprint: PendingPayload wrapper vs a bare str") + note("sys.getsizeof() is shallow: PendingPayload holds a *reference* to the payload string,") + note("not a copy, so its own size doesn't include the string's bytes. The old queue.Queue held") + note("that same string directly with nothing wrapping it, so the real extra cost per item is") + note("the wrapper object itself, plus (for non-replay-safe items) a float object for enqueued_at.") + + payload_str = PACKET + wrapper_size = sys.getsizeof(PendingPayload(payload_str, monotonic(), False)) + + note("payload str (shared either way): {} bytes".format(sys.getsizeof(payload_str))) + note("PendingPayload wrapper itself (__slots__, no __dict__): {} bytes".format(wrapper_size)) + print() + + note("replay_safe=True payloads (gauge_with_timestamp, etc.) never have their enqueued_at read") + note("(SenderQueue._expired() short-circuits on replay_safe first), so base.py passes None") + note("instead of a fresh timestamp -- no float allocation at all for this class of payload:") + replay_safe_wrapped = PendingPayload(payload_str, None, True) + note(" PendingPayload(..., enqueued_at=None, replay_safe=True): {} bytes total, +0 for the timestamp".format( + sys.getsizeof(replay_safe_wrapped) + )) + print() + + note("Non-replay-safe payloads DO need a real enqueued_at, but base.py uses coalesce_enqueue_time()") + note("instead of a bare monotonic() call: many payloads enqueued within the same ~0.1s window share") + note("ONE float object instead of each allocating their own. Demonstrating with {:,} back-to-back".format(2000)) + note("puts (a burst, which is exactly when memory pressure from a growing queue matters most):") + n = 2000 + timestamps = [coalesce_enqueue_time() for _ in range(n)] + distinct = len(set(id(t) for t in timestamps)) + note(" {:,} enqueues -> {} distinct float objects allocated ({:.2f}% of naive per-item allocation)".format( + n, distinct, 100.0 * distinct / n + )) + + worst_case_extra = wrapper_size + sys.getsizeof(monotonic()) + print() + note("Worst case (every timestamp lands in a different coalesce bucket, i.e. low, spread-out") + note("traffic): extra overhead per item vs the old bare-string queue is still just ~{} bytes".format(worst_case_extra)) + for n in (100, 10000, 100000): + note(" at sender_queue_size={:<7d} that's ~{:.1f}KB worst-case additional resident overhead".format( + n, worst_case_extra * n / 1024.0 + )) + + +# -------------------------------------------------------------------------- +def main(): + print("SenderQueue performance microbenchmark") + print("Python {}.{}.{} {}".format(sys.version_info[0], sys.version_info[1], sys.version_info[2], sys.platform)) + if QUICK: + print("(--quick mode: reduced iteration counts)") + + scenario_1_unbounded_single_threaded() + scenario_2_sustained_overflow() + scenario_3_large_expired_backlog() + scenario_4_concurrency() + scenario_5_memory_footprint() + + section("DONE") + print(" This script can't literally run against the pre-SenderQueue commit (SenderQueue") + print(" didn't exist), so the queue.Queue lines above ARE the 'before' baseline: they") + print(" faithfully reproduce the old put_nowait()/get()/task_done() contract SenderQueue") + print(" replaced. Compare SenderQueue's numbers against the queue.Queue numbers *in the") + print(" same run* (same machine, same moment, same load) rather than against an absolute") + print(" number, and re-run a few times to see how much that ratio itself varies with noise.") + print(" For an end-to-end (with real socket I/O) before/after comparison instead, run") + print(" tests/performance/test_statsd_throughput.py with disable_background_sender=False") + print(" against both the current commit and the one before this queue was introduced.") + + +if __name__ == "__main__": + main() diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index 3045f3286..f6756d2b5 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -30,7 +30,8 @@ # Datadog libraries from datadog import initialize, statsd from datadog import __version__ as version -from datadog.dogstatsd.base import DEFAULT_BUFFERING_FLUSH_INTERVAL, DEFAULT_HOST, DEFAULT_PORT, DogStatsd, MIN_SEND_BUFFER_SIZE, UDP_OPTIMAL_PAYLOAD_LENGTH, UDS_CONNECT_RETRY_INITIAL_BACKOFF, UDS_OPTIMAL_PAYLOAD_LENGTH +from datadog.dogstatsd.base import DEFAULT_BUFFERING_FLUSH_INTERVAL, DEFAULT_HOST, DEFAULT_PORT, DogStatsd, MIN_SEND_BUFFER_SIZE, PendingPayload, SenderQueue, Stop, UDP_OPTIMAL_PAYLOAD_LENGTH, UDS_CONNECT_RETRY_INITIAL_BACKOFF, UDS_OPTIMAL_PAYLOAD_LENGTH +from datadog.dogstatsd.sender_queue import coalesce_enqueue_time, monotonic as sender_queue_clock from datadog.dogstatsd.context import TimedContextManagerDecorator from datadog.util.compat import is_higher_py35, is_p3k from tests.util.contextmanagers import preserve_environment_variable, EnvVars @@ -2621,26 +2622,269 @@ def test_sender_queue_no_timeout(self): statsd = DogStatsd(disable_background_sender=False, sender_queue_timeout=None) def test_bytes_dropped_queue_counts_actual_bytes(self): - # Use a queue of size 1 and a non-blocking timeout so packets are dropped - # when the queue is full, then verify bytes_dropped_queue reflects the real - # byte length of the dropped packet (including the appended newline). + # Use a queue of size 1 so the second packet forces the first (oldest) + # one out, then verify bytes_dropped_queue reflects the real byte + # length of the dropped packet (including the appended newline), and + # that the newer payload is the one that survives in the queue. statsd = DogStatsd( disable_background_sender=False, sender_queue_size=1, - sender_queue_timeout=0, ) statsd.socket = FakeSocket() # Build a packet whose serialised form we know, then compute its length. metric_name = "test.metric" - # Send two packets: the first fills the queue, the second is dropped. - statsd._send_to_server(metric_name) + # Send two packets: the first is evicted (dropped) to make room for the second. statsd._send_to_server(metric_name) + statsd._send_to_server(metric_name + ".second") expected_bytes = len((metric_name + '\n').encode("utf-8")) self.assertEqual(statsd.bytes_dropped_queue, expected_bytes) self.assertEqual(statsd.packets_dropped_queue, 1) + self.assertEqual(statsd.bytes_dropped_expired, 0) + self.assertEqual(statsd.packets_dropped_expired, 0) + + # The surviving (newest) payload is the one the sender thread will send. + statsd.wait_for_pending() + self.assertEqual(statsd.socket.payloads[0].decode("utf-8"), metric_name + ".second\n") + + statsd.stop() + + def test_sender_queue_drops_oldest_and_stale_entries_on_overflow(self): + dropped_queue_full = [] + dropped_expired = [] + + pending_queue = SenderQueue( + maxsize=2, + expiry_seconds=20.0, + on_drop_queue_full=dropped_queue_full.append, + on_drop_expired=dropped_expired.append, + ) + + now = sender_queue_clock() + fresh = PendingPayload("fresh\n", now, False) + stale = PendingPayload("stale\n", now - 100, False) + newest = PendingPayload("newest\n", now, False) + + # Fill the queue: [fresh, stale] (stale is already expired, but that + # doesn't matter until something tries to make room or pull it off). + pending_queue.put(fresh) + pending_queue.put(stale) + self.assertEqual(pending_queue.qsize(), 2) + + # Queue is full: the oldest entry (fresh) is evicted to make room, and + # since the next entry at the front (stale) is also expired, it gets + # opportunistically cleared out too. + pending_queue.put(newest) + + self.assertEqual([p.payload for p in dropped_queue_full], ["fresh\n"]) + self.assertEqual([p.payload for p in dropped_expired], ["stale\n"]) + self.assertEqual(pending_queue.qsize(), 1) + self.assertEqual(pending_queue.get().payload, "newest\n") + + def test_sender_queue_overflow_attributes_stale_oldest_entry_to_expiry(self): + dropped_queue_full = [] + dropped_expired = [] + + pending_queue = SenderQueue( + maxsize=1, + expiry_seconds=20.0, + on_drop_queue_full=dropped_queue_full.append, + on_drop_expired=dropped_expired.append, + ) + + stale = PendingPayload("stale\n", sender_queue_clock() - 100, False) + pending_queue.put(stale) + + # The oldest (and only) entry being evicted is itself already + # expired: that's a staleness drop, not a queue-full drop. + pending_queue.put(PendingPayload("newest\n", sender_queue_clock(), False)) + + self.assertEqual(dropped_queue_full, []) + self.assertEqual([p.payload for p in dropped_expired], ["stale\n"]) + + def test_sender_queue_get_drops_expired_entries(self): + dropped_expired = [] + + pending_queue = SenderQueue( + maxsize=0, + expiry_seconds=20.0, + on_drop_queue_full=lambda item: self.fail("unexpected queue-full drop"), + on_drop_expired=dropped_expired.append, + ) + + now = sender_queue_clock() + pending_queue.put(PendingPayload("stale-1\n", now - 100, False)) + pending_queue.put(PendingPayload("stale-2\n", now - 100, False)) + pending_queue.put(PendingPayload("fresh\n", now, False)) + + # get() lazily drains every stale entry at the front before handing + # back the next payload actually worth sending. + item = pending_queue.get() + self.assertEqual(item.payload, "fresh\n") + self.assertEqual([p.payload for p in dropped_expired], ["stale-1\n", "stale-2\n"]) + + def test_sender_queue_replay_safe_payload_never_expires(self): + pending_queue = SenderQueue( + maxsize=0, + expiry_seconds=20.0, + on_drop_queue_full=lambda item: self.fail("unexpected queue-full drop"), + on_drop_expired=lambda item: self.fail("replay-safe payload should not expire"), + ) + + # Far older than the expiry window, but replay_safe=True: never dropped for staleness. + old_but_replay_safe = PendingPayload("timestamped\n", sender_queue_clock() - 10000, True) + pending_queue.put(old_but_replay_safe) + + self.assertIs(pending_queue.get(), old_but_replay_safe) + + def test_coalesce_enqueue_time_reuses_the_same_object_within_a_window(self): + # Back-to-back calls (well within the coalescing window) should + # return the exact same float object, not just an equal value -- + # that's the whole point: fewer allocations under a burst. + a = coalesce_enqueue_time() + b = coalesce_enqueue_time() + self.assertIs(a, b) + + def test_coalesce_enqueue_time_advances_across_windows(self): + first = coalesce_enqueue_time() + time.sleep(0.15) # comfortably past the 0.1s coalescing granularity + second = coalesce_enqueue_time() + self.assertGreater(second, first) + + def test_sender_queue_requeue_front_when_room_available(self): + pending_queue = SenderQueue( + maxsize=2, + expiry_seconds=20.0, + on_drop_queue_full=lambda item: self.fail("unexpected queue-full drop"), + on_drop_expired=lambda item: self.fail("unexpected expiry drop"), + ) + + in_flight = PendingPayload("in-flight\n", sender_queue_clock(), False) + pending_queue.put(in_flight) + + # Simulate the sender thread picking it up and failing to send it. + got = pending_queue.get() + self.assertIs(got, in_flight) + pending_queue.requeue_front(got) + + # There was room for it: it's retried first, ahead of anything newer. + pending_queue.put(PendingPayload("new\n", sender_queue_clock(), False)) + self.assertEqual(pending_queue.get().payload, "in-flight\n") + self.assertEqual(pending_queue.get().payload, "new\n") + + def test_sender_queue_requeue_front_drops_when_queue_is_full(self): + dropped_queue_full = [] + + pending_queue = SenderQueue( + maxsize=1, + expiry_seconds=20.0, + on_drop_queue_full=dropped_queue_full.append, + on_drop_expired=lambda item: self.fail("unexpected expiry drop"), + ) + + in_flight = PendingPayload("in-flight\n", sender_queue_clock(), False) + pending_queue.put(in_flight) + + # Simulate the sender thread picking it up, failing to send it, and a + # fresh payload filling the now-empty slot in the meantime. + got = pending_queue.get() + self.assertIs(got, in_flight) + pending_queue.put(PendingPayload("new\n", sender_queue_clock(), False)) + + # The queue is already at maxsize: the requeue is dropped rather than + # growing the queue past its limit or evicting the newer entry. + pending_queue.requeue_front(got) + + self.assertEqual([p.payload for p in dropped_queue_full], ["in-flight\n"]) + self.assertEqual(pending_queue.qsize(), 1) + self.assertEqual(pending_queue.get().payload, "new\n") + + def test_sender_queue_requeue_front_drops_when_expired(self): + dropped_expired = [] + + pending_queue = SenderQueue( + maxsize=0, + expiry_seconds=0.01, + on_drop_queue_full=lambda item: self.fail("unexpected queue-full drop"), + on_drop_expired=dropped_expired.append, + ) + + # Simulate the sender thread picking up a payload and failing to + # send it, with enough time passing in between that it's now stale. + # Unbounded queue (so it's never "full") isolates the expiry check. + in_flight = PendingPayload("stale\n", sender_queue_clock(), False) + pending_queue.put(in_flight) + got = pending_queue.get() + time.sleep(0.02) + + pending_queue.requeue_front(got) + + self.assertEqual([p.payload for p in dropped_expired], ["stale\n"]) + self.assertEqual(pending_queue.qsize(), 0) + + def test_replay_safe_flows_through_to_pending_payload(self): + statsd = DogStatsd(disable_background_sender=False) + statsd.socket = FakeSocket() + + captured = [] + original_put = statsd._queue.put + + def capture_put(item): + if item is not Stop: + captured.append(item) + return original_put(item) + + statsd._queue.put = capture_put + + statsd.increment("no.timestamp") + statsd.gauge_with_timestamp("with.timestamp", 1, int(time.time())) + statsd.wait_for_pending() + + self.assertEqual(len(captured), 2) + self.assertFalse(captured[0].replay_safe) + self.assertIsNotNone(captured[0].enqueued_at, "non-replay-safe payloads need a real timestamp to expire against") + self.assertTrue(captured[1].replay_safe) + self.assertIsNone( + captured[1].enqueued_at, + "replay_safe payloads never have enqueued_at read (see SenderQueue._expired()), " + "so it should be skipped entirely rather than allocated for nothing", + ) + + statsd.stop() + + def test_connection_failure_requeues_and_resends_once_reconnected(self): + # A UDS client whose socket is broken, with a small connect budget so + # the internal reconnect-and-retry loop inside _xmit_packet gives up + # quickly and hands off to the sender queue's own retry-by-requeuing. + working_socket = FakeSocket() + attempts = {"count": 0} + + def flaky_get_uds_socket(cls, socket_path, timeout, connect_timeout): + attempts["count"] += 1 + if attempts["count"] < 4: + raise socket.error(errno.ECONNREFUSED, "still refused") + return working_socket + + with mock.patch.object(DogStatsd, "_get_uds_socket", classmethod(flaky_get_uds_socket)): + statsd = DogStatsd( + socket_path="/tmp/dogstatsd-test-requeue.sock", + disable_telemetry=True, + disable_background_sender=False, + ) + statsd.socket_connect_timeout = 0.05 + + statsd.gauge("eventually.sent", 1) + statsd.wait_for_pending() + + # The payload survived every failed reconnect attempt and was sent + # once a working socket was finally available -- it was never + # dropped as a writer failure or expired out of the queue. + self.assertGreaterEqual(attempts["count"], 4) + self.assertEqual(statsd.packets_dropped_writer, 0) + self.assertEqual(statsd.packets_dropped_expired, 0) + self.assertTrue(working_socket.payloads[0].decode("utf-8").startswith("eventually.sent:1|g")) statsd.stop() From cf0e26ef344ec072f6c10ccbca86aae0f02c3f1d Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Wed, 9 Sep 2026 13:19:29 +0100 Subject: [PATCH 02/30] Remove coalesce time optimisation. --- datadog/dogstatsd/base.py | 4 +- datadog/dogstatsd/sender_queue.py | 33 ---------------- .../test_sender_queue_benchmark.py | 38 ++++++++----------- tests/unit/dogstatsd/test_statsd.py | 16 +------- 4 files changed, 19 insertions(+), 72 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index fc969d953..69424ede9 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -48,7 +48,7 @@ PendingPayload, Stop, PENDING_PAYLOAD_EXPIRY_SECONDS, - coalesce_enqueue_time, + monotonic, ) from datadog.util.compat import text, urlparse from datadog.util.format import normalize_tags, validate_cardinality @@ -1663,7 +1663,7 @@ def _send_to_server(self, packet, replay_safe=False): # replay_safe payloads never have their enqueued_at read # (see SenderQueue._expired()'s short-circuit), so skip # both the clock read and the float allocation for them. - enqueued_at = None if replay_safe else coalesce_enqueue_time() + enqueued_at = None if replay_safe else monotonic() self._queue.put(PendingPayload(packet_with_newline, enqueued_at, replay_safe)) return diff --git a/datadog/dogstatsd/sender_queue.py b/datadog/dogstatsd/sender_queue.py index 8cf202ade..caf3e0205 100644 --- a/datadog/dogstatsd/sender_queue.py +++ b/datadog/dogstatsd/sender_queue.py @@ -23,39 +23,6 @@ # around until they can actually be sent. PENDING_PAYLOAD_EXPIRY_SECONDS = 10.0 -# Granularity for coalesce_enqueue_time() below. Deliberately far below -# PENDING_PAYLOAD_EXPIRY_SECONDS (by two orders of magnitude with the default -# above), so it has no meaningful effect on expiry accuracy, but lets many -# payloads enqueued within the same short window share one float object -# instead of each allocating their own -- which is exactly when it matters: -# under sustained load or a backlog, not when the queue is lightly used. -_TIMESTAMP_COALESCE_SECONDS = 0.1 - -# Bucket + cached value for coalesce_enqueue_time(). Plain module globals, -# not a lock: under a race between threads, the worst outcome is a -# redundant allocation (two threads each compute a fresh reading for the -# same bucket), never an incorrect timestamp. -_coalesce_bucket = None # type: Optional[int] -_coalesce_value = 0.0 # type: float - - -def coalesce_enqueue_time(): - # type: () -> float - """A monotonic() reading coalesced to _TIMESTAMP_COALESCE_SECONDS granularity. - - Only meant for stamping payloads that DO need expiry tracking (see - PendingPayload.enqueued_at). The slop this introduces (at most one - bucket width, 0.1s by default) is negligible next to the multi-second - expiry window it feeds into. - """ - global _coalesce_bucket, _coalesce_value - raw = monotonic() - bucket = int(raw / _TIMESTAMP_COALESCE_SECONDS) - if bucket != _coalesce_bucket: - _coalesce_bucket = bucket - _coalesce_value = raw - return _coalesce_value - class PendingPayload(object): """A single packet queued for the background sender. diff --git a/tests/performance/test_sender_queue_benchmark.py b/tests/performance/test_sender_queue_benchmark.py index b755c569f..c904cceb4 100644 --- a/tests/performance/test_sender_queue_benchmark.py +++ b/tests/performance/test_sender_queue_benchmark.py @@ -58,7 +58,6 @@ from datadog.dogstatsd.sender_queue import ( # noqa: E402 PendingPayload, SenderQueue, - coalesce_enqueue_time, monotonic, ) @@ -172,7 +171,7 @@ def scenario_1_unbounded_single_threaded(): old.task_done() new = make_sender_queue(maxsize=0) - total, samples = time_puts(lambda: new.put(PendingPayload(PACKET, coalesce_enqueue_time(), False)), n) + total, samples = time_puts(lambda: new.put(PendingPayload(PACKET, monotonic(), False)), n) new_p = report("SenderQueue", n, total, samples) for _ in range(n): new.get() @@ -198,8 +197,8 @@ def scenario_2_sustained_overflow(): new = make_sender_queue(maxsize=maxsize) for _ in range(maxsize): - new.put(PendingPayload(PACKET, coalesce_enqueue_time(), False)) - total, samples = time_puts(lambda: new.put(PendingPayload(PACKET, coalesce_enqueue_time(), False)), n) + new.put(PendingPayload(PACKET, monotonic(), False)) + total, samples = time_puts(lambda: new.put(PendingPayload(PACKET, monotonic(), False)), n) new_p = report("SenderQueue", n, total, samples) ratio = new_p["p99"] / old_p["p99"] if old_p["p99"] else float("inf") @@ -227,7 +226,7 @@ def scenario_3_large_expired_backlog(): q.put(PendingPayload(PACKET, stale_at, False)) t0 = time.perf_counter() - q.put(PendingPayload(PACKET, coalesce_enqueue_time(), False)) + q.put(PendingPayload(PACKET, monotonic(), False)) elapsed_us = (time.perf_counter() - t0) * 1e6 note("backlog={:>5d} stale entries -> single put() took {:>9.3f}us, evicted {:d}".format( @@ -316,7 +315,7 @@ def scenario_4_concurrency(): new = make_sender_queue(maxsize=1000) elapsed, total_ops, samples = _run_concurrent( - lambda: new.put(PendingPayload(PACKET, coalesce_enqueue_time(), False)), + lambda: new.put(PendingPayload(PACKET, monotonic(), False)), lambda: (new.get(), new.task_done()), n_producers, n_per_producer, @@ -356,25 +355,20 @@ def scenario_5_memory_footprint(): )) print() - note("Non-replay-safe payloads DO need a real enqueued_at, but base.py uses coalesce_enqueue_time()") - note("instead of a bare monotonic() call: many payloads enqueued within the same ~0.1s window share") - note("ONE float object instead of each allocating their own. Demonstrating with {:,} back-to-back".format(2000)) - note("puts (a burst, which is exactly when memory pressure from a growing queue matters most):") - n = 2000 - timestamps = [coalesce_enqueue_time() for _ in range(n)] - distinct = len(set(id(t) for t in timestamps)) - note(" {:,} enqueues -> {} distinct float objects allocated ({:.2f}% of naive per-item allocation)".format( - n, distinct, 100.0 * distinct / n + non_replay_safe_extra = wrapper_size + sys.getsizeof(monotonic()) + note("Non-replay-safe payloads DO need a real enqueued_at -- one monotonic() reading per item,") + note("same as any other Python object holding a fresh timestamp. Extra overhead per item vs the") + note("old bare-string queue: ~{} bytes ({} wrapper + {} float).".format( + non_replay_safe_extra, wrapper_size, sys.getsizeof(monotonic()) )) - - worst_case_extra = wrapper_size + sys.getsizeof(monotonic()) - print() - note("Worst case (every timestamp lands in a different coalesce bucket, i.e. low, spread-out") - note("traffic): extra overhead per item vs the old bare-string queue is still just ~{} bytes".format(worst_case_extra)) for n in (100, 10000, 100000): - note(" at sender_queue_size={:<7d} that's ~{:.1f}KB worst-case additional resident overhead".format( - n, worst_case_extra * n / 1024.0 + note(" at sender_queue_size={:<7d} that's ~{:.1f}KB of additional resident overhead".format( + n, non_replay_safe_extra * n / 1024.0 )) + note("(An earlier version of this code coalesced timestamps to a shared per-100ms-bucket float") + note("to cut this under bursty load -- best case ~234KB saved at sender_queue_size=10,000, i.e.") + note("~0.09% of a typical 256MB container's RSS. Reverted: not worth the added global mutable") + note("state, cross-instance coupling, and dedicated concurrency tests for savings that small.)") # -------------------------------------------------------------------------- diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index f6756d2b5..ed4731daf 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -31,7 +31,7 @@ from datadog import initialize, statsd from datadog import __version__ as version from datadog.dogstatsd.base import DEFAULT_BUFFERING_FLUSH_INTERVAL, DEFAULT_HOST, DEFAULT_PORT, DogStatsd, MIN_SEND_BUFFER_SIZE, PendingPayload, SenderQueue, Stop, UDP_OPTIMAL_PAYLOAD_LENGTH, UDS_CONNECT_RETRY_INITIAL_BACKOFF, UDS_OPTIMAL_PAYLOAD_LENGTH -from datadog.dogstatsd.sender_queue import coalesce_enqueue_time, monotonic as sender_queue_clock +from datadog.dogstatsd.sender_queue import monotonic as sender_queue_clock from datadog.dogstatsd.context import TimedContextManagerDecorator from datadog.util.compat import is_higher_py35, is_p3k from tests.util.contextmanagers import preserve_environment_variable, EnvVars @@ -2739,20 +2739,6 @@ def test_sender_queue_replay_safe_payload_never_expires(self): self.assertIs(pending_queue.get(), old_but_replay_safe) - def test_coalesce_enqueue_time_reuses_the_same_object_within_a_window(self): - # Back-to-back calls (well within the coalescing window) should - # return the exact same float object, not just an equal value -- - # that's the whole point: fewer allocations under a burst. - a = coalesce_enqueue_time() - b = coalesce_enqueue_time() - self.assertIs(a, b) - - def test_coalesce_enqueue_time_advances_across_windows(self): - first = coalesce_enqueue_time() - time.sleep(0.15) # comfortably past the 0.1s coalescing granularity - second = coalesce_enqueue_time() - self.assertGreater(second, first) - def test_sender_queue_requeue_front_when_room_available(self): pending_queue = SenderQueue( maxsize=2, From 08cebfc0ca8c7bf7bd6376c3b86a328a153e3a1a Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 10 Sep 2026 09:44:11 +0100 Subject: [PATCH 03/30] Include expired drops --- datadog/dogstatsd/base.py | 19 +++++++-- tests/unit/dogstatsd/test_statsd.py | 61 ++++++++++++++++++++++++++--- 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index 69424ede9..d7dd36260 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -1622,6 +1622,17 @@ def _flush_telemetry(self): tags.extend(self.constant_tags) telemetry_tags = ",".join(tags) + # There's no dedicated wire-protocol metric for expired drops (see + # bytes_dropped_expired/packets_dropped_expired for that level of + # detail in-process): they're folded into the *_dropped_queue lines + # reported to the Agent, since both categories share the same root + # cause from the Agent's point of view -- the payload never reached + # a socket write attempt, dropped by the queue itself rather than by + # the writer. Without this, they'd silently vanish even from the + # combined dropped total sent to the Agent. + bytes_dropped_queue = self.bytes_dropped_queue + self.bytes_dropped_expired + packets_dropped_queue = self.packets_dropped_queue + self.packets_dropped_expired + return TELEMETRY_FORMATTING_STR % ( self.metrics_count, telemetry_tags, @@ -1631,17 +1642,17 @@ def _flush_telemetry(self): telemetry_tags, self.bytes_sent, telemetry_tags, - self.bytes_dropped_queue + self.bytes_dropped_writer, + bytes_dropped_queue + self.bytes_dropped_writer, telemetry_tags, - self.bytes_dropped_queue, + bytes_dropped_queue, telemetry_tags, self.bytes_dropped_writer, telemetry_tags, self.packets_sent, telemetry_tags, - self.packets_dropped_queue + self.packets_dropped_writer, + packets_dropped_queue + self.packets_dropped_writer, telemetry_tags, - self.packets_dropped_queue, + packets_dropped_queue, telemetry_tags, self.packets_dropped_writer, telemetry_tags, diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index ed4731daf..ada17bd10 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -124,20 +124,26 @@ def __init__(self): super(OverflownSocket, self).__init__(errno.EAGAIN) -def telemetry_metrics(metrics=1, events=0, service_checks=0, bytes_sent=0, bytes_dropped_writer=0, packets_sent=1, packets_dropped_writer=0, transport="udp", tags="", bytes_dropped_queue=0, packets_dropped_queue=0): +def telemetry_metrics(metrics=1, events=0, service_checks=0, bytes_sent=0, bytes_dropped_writer=0, packets_sent=1, packets_dropped_writer=0, transport="udp", tags="", bytes_dropped_queue=0, packets_dropped_queue=0, bytes_dropped_expired=0, packets_dropped_expired=0): tags = "," + tags if tags else "" + # Expired drops have no dedicated wire metric: they're folded into the + # *_dropped_queue lines (and totals) reported to the Agent. See + # DogStatsd._flush_telemetry(). + reported_bytes_dropped_queue = bytes_dropped_queue + bytes_dropped_expired + reported_packets_dropped_queue = packets_dropped_queue + packets_dropped_expired + return "\n".join([ "datadog.dogstatsd.client.metrics:{}|c|#client:py,client_version:{},client_transport:{}{}".format(metrics, version, transport, tags), "datadog.dogstatsd.client.events:{}|c|#client:py,client_version:{},client_transport:{}{}".format(events, version, transport, tags), "datadog.dogstatsd.client.service_checks:{}|c|#client:py,client_version:{},client_transport:{}{}".format(service_checks, version, transport, tags), "datadog.dogstatsd.client.bytes_sent:{}|c|#client:py,client_version:{},client_transport:{}{}".format(bytes_sent, version, transport, tags), - "datadog.dogstatsd.client.bytes_dropped:{}|c|#client:py,client_version:{},client_transport:{}{}".format(bytes_dropped_queue + bytes_dropped_writer, version, transport, tags), - "datadog.dogstatsd.client.bytes_dropped_queue:{}|c|#client:py,client_version:{},client_transport:{}{}".format(bytes_dropped_queue, version, transport, tags), + "datadog.dogstatsd.client.bytes_dropped:{}|c|#client:py,client_version:{},client_transport:{}{}".format(reported_bytes_dropped_queue + bytes_dropped_writer, version, transport, tags), + "datadog.dogstatsd.client.bytes_dropped_queue:{}|c|#client:py,client_version:{},client_transport:{}{}".format(reported_bytes_dropped_queue, version, transport, tags), "datadog.dogstatsd.client.bytes_dropped_writer:{}|c|#client:py,client_version:{},client_transport:{}{}".format(bytes_dropped_writer, version, transport, tags), "datadog.dogstatsd.client.packets_sent:{}|c|#client:py,client_version:{},client_transport:{}{}".format(packets_sent, version, transport, tags), - "datadog.dogstatsd.client.packets_dropped:{}|c|#client:py,client_version:{},client_transport:{}{}".format(packets_dropped_queue + packets_dropped_writer, version, transport, tags), - "datadog.dogstatsd.client.packets_dropped_queue:{}|c|#client:py,client_version:{},client_transport:{}{}".format(packets_dropped_queue, version, transport, tags), + "datadog.dogstatsd.client.packets_dropped:{}|c|#client:py,client_version:{},client_transport:{}{}".format(reported_packets_dropped_queue + packets_dropped_writer, version, transport, tags), + "datadog.dogstatsd.client.packets_dropped_queue:{}|c|#client:py,client_version:{},client_transport:{}{}".format(reported_packets_dropped_queue, version, transport, tags), "datadog.dogstatsd.client.packets_dropped_writer:{}|c|#client:py,client_version:{},client_transport:{}{}".format(packets_dropped_writer, version, transport, tags), ]) + "\n" @@ -1955,6 +1961,51 @@ def test_telemetry(self): self.assertEqual(0, self.statsd.bytes_dropped_queue) self.assertEqual(0, self.statsd.packets_dropped_queue) + def test_telemetry_folds_expired_drops_into_dropped_queue(self): + # There's no dedicated wire metric for expired drops: they're + # reported to the Agent as part of *_dropped_queue (and the combined + # *_dropped total), alongside capacity-based queue drops, since both + # never reach a socket write attempt. The distinction is still + # available in-process via bytes_dropped_expired/packets_dropped_expired. + # Avoid any real container-id auto-detected from the host/sandbox + # cgroup leaking into the expected payload below -- this test is + # about the telemetry counters, not the container-id field. + self.statsd._container_id = None + + self.statsd.bytes_dropped_queue = 8 + self.statsd.packets_dropped_queue = 9 + self.statsd.bytes_dropped_expired = 10 + self.statsd.packets_dropped_expired = 11 + self.statsd.bytes_dropped_writer = 5 + self.statsd.packets_dropped_writer = 7 + + self.statsd.open_buffer() + self.statsd.gauge('page.views', 123) + self.statsd.close_buffer() + + payload = 'page.views:123|g\n' + telemetry = telemetry_metrics( + metrics=1, + bytes_sent=len(payload), + packets_sent=1, + bytes_dropped_queue=8, + packets_dropped_queue=9, + bytes_dropped_expired=10, + packets_dropped_expired=11, + bytes_dropped_writer=5, + packets_dropped_writer=7, + ) + + self.assert_equal_telemetry(payload, self.recv(2), telemetry=telemetry) + + # The in-process counters stay separate even after the flush resets + # them -- confirming the fold happens only in the wire output, not + # by merging the underlying attributes. + self.assertEqual(0, self.statsd.bytes_dropped_queue) + self.assertEqual(0, self.statsd.packets_dropped_queue) + self.assertEqual(0, self.statsd.bytes_dropped_expired) + self.assertEqual(0, self.statsd.packets_dropped_expired) + def test_telemetry_flush_interval(self): dogstatsd = DogStatsd(disable_buffering=False) fake_socket = FakeSocket() From c335ae8167be162953056abba3933dd6914708b1 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 10 Sep 2026 17:09:02 +0100 Subject: [PATCH 04/30] Implement sender queue timeout --- datadog/dogstatsd/base.py | 37 +++++--- datadog/dogstatsd/sender_queue.py | 45 ++++++++-- tests/unit/dogstatsd/test_statsd.py | 130 ++++++++++++++++++++++++++++ 3 files changed, 189 insertions(+), 23 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index d7dd36260..e6d4ab5bf 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -490,18 +490,22 @@ def __init__( :type disable_background_sender: boolean :param sender_queue_size: Set the maximum number of packets to queue for the sender. Optional. - Once the queue is full, adding a new packet drops the oldest queued packet (and any additional - expired packets at the front of the queue) to make room, instead of blocking or dropping the new - packet. Packets aren't held indefinitely either: a queued packet that hasn't been sent within - PENDING_PAYLOAD_EXPIRY_SECONDS is dropped when it's pulled off the queue, unless it carries its own - explicit timestamp (e.g. gauge_with_timestamp, or count/service_check/event with an explicit - timestamp), in which case it's kept until it can actually be sent. + Once the queue is full, adding a new packet waits (see sender_queue_timeout) and then, if still + full, drops the oldest queued packet (and any additional expired packets at the front of the + queue) to make room, instead of dropping the new packet. Packets aren't held indefinitely either: + a queued packet that hasn't been sent within PENDING_PAYLOAD_EXPIRY_SECONDS is dropped when it's + pulled off the queue, unless it carries its own explicit timestamp (e.g. gauge_with_timestamp, or + count/service_check/event with an explicit timestamp), in which case it's kept until it can + actually be sent. Default: 0 (unlimited). :type sender_queue_size: integer - :param sender_queue_timeout: Deprecated and ignored. The sender queue no longer blocks: it always - makes room for a new packet by dropping older or expired entries instead. Kept only for backwards - compatibility with existing call sites. + :param sender_queue_timeout: Set how long, in seconds, adding a packet to a full sender queue + will wait for the background sender to free up a slot before falling back to dropping the oldest + queued packet to make room. If set to zero or None (the default), no waiting happens: a full + queue makes room immediately by dropping the oldest packet. Note this blocks the calling thread + (the one emitting the metric), not just the background sender -- pick a value that fits how long + you're willing to let application code stall during a backlog. :type sender_queue_timeout: float :param track_instance: Keep track of this instance and automatically handle cleanup when os.fork() is called, @@ -718,19 +722,23 @@ def enable_background_sender(self, sender_queue_size=0, sender_queue_timeout=0): to os.fork(). :param sender_queue_size: Set the maximum number of packets to queue for the sender. - Once the queue is full, adding a new packet drops the oldest queued packet (and any additional - expired packets at the front of the queue) to make room, instead of blocking or dropping the new - packet. + Once the queue is full, adding a new packet waits (see sender_queue_timeout) and then, if + still full, drops the oldest queued packet (and any additional expired packets at the front + of the queue) to make room, instead of dropping the new packet. Default: 0 (unlimited). :type sender_queue_size: integer, optional - :param sender_queue_timeout: Deprecated and ignored: the sender queue no longer blocks. Kept only - for backwards compatibility with existing call sites. + :param sender_queue_timeout: Set how long, in seconds, adding a packet to a full sender queue + will wait for the background sender to free up a slot before falling back to dropping the + oldest queued packet to make room. If set to zero or None (the default), no waiting happens. + Note this blocks the calling thread (the one emitting the metric), not just the background + sender. :type sender_queue_timeout: float, optional """ with self._config_lock: self._sender_enabled = True self._sender_queue_size = sender_queue_size + self._sender_queue_timeout = sender_queue_timeout self._start_sender_thread() @@ -2131,6 +2139,7 @@ def _start_sender_thread(self): PENDING_PAYLOAD_EXPIRY_SECONDS, self._account_dropped_queue_full, self._account_dropped_expired, + put_timeout=self._sender_queue_timeout, ) log.debug("Starting background sender thread") diff --git a/datadog/dogstatsd/sender_queue.py b/datadog/dogstatsd/sender_queue.py index caf3e0205..8a5922154 100644 --- a/datadog/dogstatsd/sender_queue.py +++ b/datadog/dogstatsd/sender_queue.py @@ -53,10 +53,14 @@ def __init__(self, payload, enqueued_at, replay_safe): class SenderQueue(object): """Bounded hand-off queue between application threads and the background sender thread. - Unlike queue.Queue, put() never blocks and never rejects a payload. When - the queue is already at its maximum size, the oldest entry is dropped to - make room, along with any additional expired entries left at the front, - so a backlog of stale payloads can't shut out fresh metrics indefinitely. + put() never rejects a payload outright. When the queue is already at its + maximum size and put_timeout is falsy (the default), the oldest entry is dropped + immediately to make room, along with any additional expired entries left + at the front. When put_timeout is a positive number, put() + instead blocks the calling thread for up to that many seconds waiting + for the sender thread to drain a slot; only once that wait times out + (or immediately, if put_timeout is falsy) does it fall back to the same + drop-oldest eviction. get() drops expired entries lazily too, from the front, before returning the next payload actually worth handing to the sender. @@ -65,18 +69,21 @@ class SenderQueue(object): handed back with requeue_front() so it's retried first. That still respects both the expiry check and the size limit though: the queue must never grow past maxsize, and a payload that's gone stale while it - was being (re)tried is dropped rather than requeued. + was being (re)tried is dropped rather than requeued. requeue_front() + never blocks on put_timeout. """ - def __init__(self, maxsize, expiry_seconds, on_drop_queue_full, on_drop_expired): - # type: (int, float, Callable[[PendingPayload], None], Callable[[PendingPayload], None]) -> None + def __init__(self, maxsize, expiry_seconds, on_drop_queue_full, on_drop_expired, put_timeout=None): + # type: (int, float, Callable[[PendingPayload], None], Callable[[PendingPayload], None], Optional[float]) -> None self._maxsize = maxsize self._expiry_seconds = expiry_seconds self._on_drop_queue_full = on_drop_queue_full self._on_drop_expired = on_drop_expired + self._put_timeout = put_timeout self._deque = collections.deque() # type: collections.deque self._lock = threading.Lock() self._not_empty = threading.Condition(self._lock) + self._not_full = threading.Condition(self._lock) self._all_tasks_done = threading.Condition(self._lock) # Keep track of the tasks that are being processed. A task pulled from the queue may @@ -126,10 +133,27 @@ def _make_room_locked(self): def put(self, item): # type: (Union[PendingPayload, object]) -> None - """Queue a payload (or the Stop sentinel), evicting old entries if needed.""" + """Queue a payload (or the Stop sentinel). + + If the queue is full: waits for room for up to put_timeout seconds + (if put_timeout is a positive number), then falls back to evicting + the oldest entry (see _make_room_locked()) if the wait timed out + without room opening up -- or immediately, with no wait at all, if + put_timeout is falsy. Either way, put() never rejects the payload + outright. + """ with self._not_empty: if item is not Stop and self._maxsize > 0 and len(self._deque) >= self._maxsize: - self._make_room_locked() + if self._put_timeout: + deadline = monotonic() + self._put_timeout + while len(self._deque) >= self._maxsize: + remaining = deadline - monotonic() + if remaining <= 0: + break + self._not_full.wait(remaining) + + if len(self._deque) >= self._maxsize: + self._make_room_locked() self._deque.append(item) self._unfinished_tasks += 1 @@ -170,6 +194,9 @@ def get(self): while not self._deque: self._not_empty.wait() item = self._deque.popleft() + # A slot just opened up: wake one thread blocked in put()'s + # wait-for-room loop, if any (harmless no-op otherwise). + self._not_full.notify() if item is Stop: return item diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index ada17bd10..03fb32546 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -2671,6 +2671,33 @@ def test_sender_calls_task_done(self): def test_sender_queue_no_timeout(self): statsd = DogStatsd(disable_background_sender=False, sender_queue_timeout=None) + statsd.stop() + + def test_sender_queue_timeout_blocks_the_calling_thread_through_the_client(self): + # End-to-end: sender_queue_timeout configured on the real client + # actually makes statsd.increment() (the calling/application thread) + # block waiting for room, not just an internal SenderQueue detail. + statsd = DogStatsd( + disable_background_sender=False, + sender_queue_size=1, + sender_queue_timeout=5.0, + ) + # No socket assigned: the sender thread can never drain anything by + # actually sending, so the only way room opens up is via get() + # pulling an item off (which happens immediately, since nothing can + # succeed in sending it -- it gets hard-dropped as a writer failure + # and the sender loop moves on to the next get()). + statsd.socket = FakeSocket() + + statsd.increment("first") + + t0 = time.time() + statsd.increment("second") + elapsed = time.time() - t0 + + self.assertLess(elapsed, 5.0, "should not have waited out the full 5s timeout") + statsd.wait_for_pending() + statsd.stop() def test_bytes_dropped_queue_counts_actual_bytes(self): # Use a queue of size 1 so the second packet forces the first (oldest) @@ -2702,6 +2729,109 @@ def test_bytes_dropped_queue_counts_actual_bytes(self): statsd.stop() + def test_sender_queue_put_timeout_none_evicts_immediately(self): + # Default behaviour (put_timeout falsy): no waiting at all, same as + # before this feature existed. + dropped_queue_full = [] + pending_queue = SenderQueue( + maxsize=1, + expiry_seconds=100.0, + on_drop_queue_full=dropped_queue_full.append, + on_drop_expired=lambda item: self.fail("unexpected expiry drop"), + ) + + pending_queue.put(PendingPayload("first\n", sender_queue_clock(), False)) + + t0 = time.time() + pending_queue.put(PendingPayload("second\n", sender_queue_clock(), False)) + elapsed = time.time() - t0 + + self.assertLess(elapsed, 0.05, "put() should not have waited at all") + self.assertEqual([p.payload for p in dropped_queue_full], ["first\n"]) + self.assertEqual(pending_queue.get().payload, "second\n") + + def test_sender_queue_put_timeout_wakes_up_when_room_opens(self): + # A slot freed by get() (well within put_timeout) should wake a + # blocked put() immediately rather than making it wait out the full + # timeout, and nothing should be dropped. + dropped_queue_full = [] + pending_queue = SenderQueue( + maxsize=1, + expiry_seconds=100.0, + on_drop_queue_full=dropped_queue_full.append, + on_drop_expired=lambda item: self.fail("unexpected expiry drop"), + put_timeout=5.0, + ) + pending_queue.put(PendingPayload("first\n", sender_queue_clock(), False)) + + result = {} + + def blocked_put(): + t0 = time.time() + pending_queue.put(PendingPayload("second\n", sender_queue_clock(), False)) + result["elapsed"] = time.time() - t0 + + t = threading.Thread(target=blocked_put) + t.start() + time.sleep(0.2) + self.assertTrue(t.is_alive(), "put() should still be waiting for room") + + # Drain the one slot: the blocked put() should wake up promptly. + self.assertEqual(pending_queue.get().payload, "first\n") + pending_queue.task_done() + + t.join(timeout=5.0) + self.assertFalse(t.is_alive()) + self.assertLess(result["elapsed"], 5.0, "should have woken up well before the 5s timeout") + self.assertEqual(dropped_queue_full, [], "nothing should have been dropped: room opened up in time") + self.assertEqual(pending_queue.get().payload, "second\n") + + def test_sender_queue_put_timeout_falls_back_to_eviction(self): + # If room never opens up within put_timeout, put() falls back to + # the same drop-oldest eviction as the immediate (no-wait) case. + dropped_queue_full = [] + pending_queue = SenderQueue( + maxsize=1, + expiry_seconds=100.0, + on_drop_queue_full=dropped_queue_full.append, + on_drop_expired=lambda item: self.fail("unexpected expiry drop"), + put_timeout=0.2, + ) + pending_queue.put(PendingPayload("first\n", sender_queue_clock(), False)) + + t0 = time.time() + pending_queue.put(PendingPayload("second\n", sender_queue_clock(), False)) + elapsed = time.time() - t0 + + self.assertGreaterEqual(elapsed, 0.2) + self.assertEqual([p.payload for p in dropped_queue_full], ["first\n"]) + self.assertEqual(pending_queue.get().payload, "second\n") + + def test_sender_queue_requeue_front_never_blocks_on_put_timeout(self): + # requeue_front() runs on the background sender thread; it must + # never wait on put_timeout, or one stuck retry would stall every + # other queued payload behind it. + dropped_queue_full = [] + pending_queue = SenderQueue( + maxsize=1, + expiry_seconds=100.0, + on_drop_queue_full=dropped_queue_full.append, + on_drop_expired=lambda item: self.fail("unexpected expiry drop"), + put_timeout=5.0, + ) + in_flight = PendingPayload("in-flight\n", sender_queue_clock(), False) + pending_queue.put(in_flight) + got = pending_queue.get() + pending_queue.put(PendingPayload("new\n", sender_queue_clock(), False)) # fills the one slot again + + t0 = time.time() + pending_queue.requeue_front(got) + elapsed = time.time() - t0 + + self.assertLess(elapsed, 0.05, "requeue_front() must not block on put_timeout") + self.assertEqual([p.payload for p in dropped_queue_full], ["in-flight\n"]) + self.assertEqual(pending_queue.get().payload, "new\n") + def test_sender_queue_drops_oldest_and_stale_entries_on_overflow(self): dropped_queue_full = [] dropped_expired = [] From 0515f198e8bcba605061372aadd204731a5dd749 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Fri, 11 Sep 2026 16:04:41 +0100 Subject: [PATCH 05/30] Allow waiting indefinitely --- datadog/dogstatsd/base.py | 29 ++++++------ datadog/dogstatsd/sender_queue.py | 47 ++++++++++++------- tests/unit/dogstatsd/test_statsd.py | 73 +++++++++++++++++++++++++++-- 3 files changed, 113 insertions(+), 36 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index e6d4ab5bf..e8cb61aff 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -490,22 +490,18 @@ def __init__( :type disable_background_sender: boolean :param sender_queue_size: Set the maximum number of packets to queue for the sender. Optional. - Once the queue is full, adding a new packet waits (see sender_queue_timeout) and then, if still - full, drops the oldest queued packet (and any additional expired packets at the front of the - queue) to make room, instead of dropping the new packet. Packets aren't held indefinitely either: - a queued packet that hasn't been sent within PENDING_PAYLOAD_EXPIRY_SECONDS is dropped when it's - pulled off the queue, unless it carries its own explicit timestamp (e.g. gauge_with_timestamp, or - count/service_check/event with an explicit timestamp), in which case it's kept until it can - actually be sent. + How many packets to queue before blocking or dropping the packet if the packet queue is already full. Default: 0 (unlimited). :type sender_queue_size: integer :param sender_queue_timeout: Set how long, in seconds, adding a packet to a full sender queue will wait for the background sender to free up a slot before falling back to dropping the oldest - queued packet to make room. If set to zero or None (the default), no waiting happens: a full - queue makes room immediately by dropping the oldest packet. Note this blocks the calling thread - (the one emitting the metric), not just the background sender -- pick a value that fits how long - you're willing to let application code stall during a backlog. + queued packet to make room. Default: 0, meaning no waiting happens at all: a full queue makes + room immediately by dropping the oldest packet. If set to None, waits forever for room instead + of ever falling back to dropping the oldest packet -- an explicit opt-in to unbounded + backpressure; nothing bounds how long this can block if the sender can never catch up. Note this + blocks the calling thread (the one emitting the metric), not just the background sender -- pick a + value that fits how long you're willing to let application code stall during a backlog. :type sender_queue_timeout: float :param track_instance: Keep track of this instance and automatically handle cleanup when os.fork() is called, @@ -729,9 +725,10 @@ def enable_background_sender(self, sender_queue_size=0, sender_queue_timeout=0): :type sender_queue_size: integer, optional :param sender_queue_timeout: Set how long, in seconds, adding a packet to a full sender queue will wait for the background sender to free up a slot before falling back to dropping the - oldest queued packet to make room. If set to zero or None (the default), no waiting happens. - Note this blocks the calling thread (the one emitting the metric), not just the background - sender. + oldest queued packet to make room. Default: 0, meaning no waiting happens at all. If set to + None, waits forever for room instead of ever falling back to dropping the oldest packet -- + an explicit opt-in to unbounded backpressure. Note this blocks the calling thread (the one + emitting the metric), not just the background sender. :type sender_queue_timeout: float, optional """ @@ -2176,7 +2173,9 @@ def _sender_main_loop(self, pending_queue): # next line has type ignore because the type checker cannot # know that 'if item is Stop' is the only case where item is # of object type. - sent = self._xmit_packet_with_telemetry(item.payload, queue_mode=True) # type: ignore[attr-defined] # noqa: F821 + sent = self._xmit_packet_with_telemetry( + item.payload, queue_mode=True # type: ignore[attr-defined] + ) if sent is None: # Connection trouble: keep the payload for the next attempt diff --git a/datadog/dogstatsd/sender_queue.py b/datadog/dogstatsd/sender_queue.py index 8a5922154..f7db9a181 100644 --- a/datadog/dogstatsd/sender_queue.py +++ b/datadog/dogstatsd/sender_queue.py @@ -13,7 +13,7 @@ from typing import Callable, Optional, Union # noqa: F401 -# Sentinel telling the background sender thread to shut down. +# Sentinel telling the background sender thread to shut down. Stop = object() # How long (in seconds) a non-replay-safe payload may sit in the background @@ -54,13 +54,18 @@ class SenderQueue(object): """Bounded hand-off queue between application threads and the background sender thread. put() never rejects a payload outright. When the queue is already at its - maximum size and put_timeout is falsy (the default), the oldest entry is dropped - immediately to make room, along with any additional expired entries left - at the front. When put_timeout is a positive number, put() - instead blocks the calling thread for up to that many seconds waiting - for the sender thread to drain a slot; only once that wait times out - (or immediately, if put_timeout is falsy) does it fall back to the same - drop-oldest eviction. + maximum size, what happens depends on put_timeout: + - 0 (the default): no waiting at all -- the oldest entry is dropped + immediately to make room, along with any additional expired entries + left at the front. + - None: put() blocks the calling thread indefinitely, waiting for the + sender thread to drain a slot. It will wait forever if nothing ever + does -- this is an explicit opt-in to unbounded backpressure on the + calling thread. + - a positive number: put() blocks the calling thread for up to that + many seconds waiting for a slot; if the wait times out without one + opening up, it falls back to the same drop-oldest eviction as the + 0 case. get() drops expired entries lazily too, from the front, before returning the next payload actually worth handing to the sender. @@ -73,8 +78,8 @@ class SenderQueue(object): never blocks on put_timeout. """ - def __init__(self, maxsize, expiry_seconds, on_drop_queue_full, on_drop_expired, put_timeout=None): - # type: (int, float, Callable[[PendingPayload], None], Callable[[PendingPayload], None], Optional[float]) -> None + def __init__(self, maxsize, expiry_seconds, on_drop_queue_full, on_drop_expired, put_timeout=0): + # type: (int, float, Callable[[PendingPayload], None], Callable[[PendingPayload], None], Optional[float]) -> None # noqa: E501 self._maxsize = maxsize self._expiry_seconds = expiry_seconds self._on_drop_queue_full = on_drop_queue_full @@ -135,22 +140,29 @@ def put(self, item): # type: (Union[PendingPayload, object]) -> None """Queue a payload (or the Stop sentinel). - If the queue is full: waits for room for up to put_timeout seconds - (if put_timeout is a positive number), then falls back to evicting - the oldest entry (see _make_room_locked()) if the wait timed out - without room opening up -- or immediately, with no wait at all, if - put_timeout is falsy. Either way, put() never rejects the payload - outright. + If the queue is full: waits for room according to put_timeout -- + forever if it's None, up to put_timeout seconds if it's a positive + number, or not at all if it's 0 (the default) -- then falls back to + evicting the oldest entry (see _make_room_locked()) if the queue is + still full once the wait is over. Either way, put() never rejects + the payload outright. """ with self._not_empty: if item is not Stop and self._maxsize > 0 and len(self._deque) >= self._maxsize: - if self._put_timeout: + if self._put_timeout is None: + # Wait forever: an explicit opt-in to unbounded + # backpressure on the calling thread. + while len(self._deque) >= self._maxsize: + self._not_full.wait() + elif self._put_timeout > 0: deadline = monotonic() + self._put_timeout while len(self._deque) >= self._maxsize: remaining = deadline - monotonic() if remaining <= 0: break self._not_full.wait(remaining) + # else: put_timeout is 0 (or negative) -- no wait at all, + # straight to eviction below. if len(self._deque) >= self._maxsize: self._make_room_locked() @@ -238,4 +250,3 @@ def empty(self): # type: () -> bool with self._lock: return not self._deque - diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index 03fb32546..5fac674cc 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -2729,9 +2729,13 @@ def test_bytes_dropped_queue_counts_actual_bytes(self): statsd.stop() - def test_sender_queue_put_timeout_none_evicts_immediately(self): - # Default behaviour (put_timeout falsy): no waiting at all, same as - # before this feature existed. + def test_sender_queue_put_timeout_default_evicts_immediately(self): + # Default put_timeout (0, whether omitted or explicit): no waiting + # at all, same as before this feature existed. Deliberately omits + # put_timeout here to prove the *default* -- not just 0 -- means + # "don't wait", since None means something very different (wait + # forever) and must not be the implicit default for anyone who + # constructs a SenderQueue without thinking about put_timeout at all. dropped_queue_full = [] pending_queue = SenderQueue( maxsize=1, @@ -2750,6 +2754,69 @@ def test_sender_queue_put_timeout_none_evicts_immediately(self): self.assertEqual([p.payload for p in dropped_queue_full], ["first\n"]) self.assertEqual(pending_queue.get().payload, "second\n") + def test_sender_queue_put_timeout_zero_evicts_immediately(self): + # Same as the default, but with put_timeout=0 passed explicitly. + dropped_queue_full = [] + pending_queue = SenderQueue( + maxsize=1, + expiry_seconds=100.0, + on_drop_queue_full=dropped_queue_full.append, + on_drop_expired=lambda item: self.fail("unexpected expiry drop"), + put_timeout=0, + ) + + pending_queue.put(PendingPayload("first\n", sender_queue_clock(), False)) + + t0 = time.time() + pending_queue.put(PendingPayload("second\n", sender_queue_clock(), False)) + elapsed = time.time() - t0 + + self.assertLess(elapsed, 0.05, "put() should not have waited at all") + self.assertEqual([p.payload for p in dropped_queue_full], ["first\n"]) + self.assertEqual(pending_queue.get().payload, "second\n") + + def test_sender_queue_put_timeout_none_waits_forever_and_never_evicts(self): + # put_timeout=None is an explicit opt-in to unbounded blocking: put() + # must keep waiting indefinitely -- not fall back to eviction after + # some internal default -- until room actually opens up. + dropped_queue_full = [] + pending_queue = SenderQueue( + maxsize=1, + expiry_seconds=100.0, + on_drop_queue_full=lambda item: dropped_queue_full.append(item), + on_drop_expired=lambda item: self.fail("unexpected expiry drop"), + put_timeout=None, + ) + pending_queue.put(PendingPayload("first\n", sender_queue_clock(), False)) + + result = {} + + def blocked_put(): + t0 = time.time() + pending_queue.put(PendingPayload("second\n", sender_queue_clock(), False)) + result["elapsed"] = time.time() - t0 + + t = threading.Thread(target=blocked_put) + t.start() + try: + # Nothing is draining the queue: with a real timeout this would + # have already fired and evicted "first" well before 1s. With + # None it must still be waiting. + time.sleep(1.0) + self.assertTrue(t.is_alive(), "put(timeout=None) must keep waiting, never fall back to eviction on its own") + self.assertEqual(dropped_queue_full, []) + + # Now free up room: the blocked put() should wake up and + # succeed without ever having dropped anything. + self.assertEqual(pending_queue.get().payload, "first\n") + pending_queue.task_done() + finally: + t.join(timeout=5.0) + + self.assertFalse(t.is_alive()) + self.assertEqual(dropped_queue_full, [], "put_timeout=None must never fall back to eviction") + self.assertEqual(pending_queue.get().payload, "second\n") + def test_sender_queue_put_timeout_wakes_up_when_room_opens(self): # A slot freed by get() (well within put_timeout) should wake a # blocked put() immediately rather than making it wait out the full From 9a7aaa139fd2a797b2d42db420dd7860e4d8d82d Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Mon, 14 Sep 2026 12:02:52 +0100 Subject: [PATCH 06/30] Maintain replay and non-replay safe buffers --- datadog/dogstatsd/base.py | 107 ++++++------- datadog/dogstatsd/sender_queue.py | 15 +- .../dogstatsd/test_statsd_sender.py | 3 +- tests/unit/dogstatsd/test_statsd.py | 140 ++++++++++++++++++ 4 files changed, 208 insertions(+), 57 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index e8cb61aff..246b53bf2 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -26,7 +26,9 @@ # pylint: disable=unused-import if sys.version_info[:2] >= (3, 5): - from typing import Any, Optional, List, Text, Tuple, Type, Union, Iterable, Callable, overload # noqa: F401 + from typing import ( # noqa: F401 + Any, Callable, Dict, Iterable, List, Optional, Text, Tuple, Type, Union, overload, + ) try: from typing import SupportsIndex @@ -494,14 +496,11 @@ def __init__( Default: 0 (unlimited). :type sender_queue_size: integer - :param sender_queue_timeout: Set how long, in seconds, adding a packet to a full sender queue - will wait for the background sender to free up a slot before falling back to dropping the oldest - queued packet to make room. Default: 0, meaning no waiting happens at all: a full queue makes - room immediately by dropping the oldest packet. If set to None, waits forever for room instead - of ever falling back to dropping the oldest packet -- an explicit opt-in to unbounded - backpressure; nothing bounds how long this can block if the sender can never catch up. Note this - blocks the calling thread (the one emitting the metric), not just the background sender -- pick a - value that fits how long you're willing to let application code stall during a backlog. + :param sender_queue_timeout: Set timeout for packet queue operations, in seconds. Optional. + How long the application thread is willing to wait for the queue clear up before dropping the metric packet. + If set to None, wait forever. + If set to zero drop the packet immediately if the queue is full. + Default: 0 (no wait) :type sender_queue_timeout: float :param track_instance: Keep track of this instance and automatically handle cleanup when os.fork() is called, @@ -610,8 +609,6 @@ def __init__( self._telemetry = not disable_telemetry self._last_flush_time = time.time() - self._current_buffer_total_size = 0 - self._buffer = [] # type: List[Text] self._buffer_lock = RLock() self._reset_buffer() @@ -718,17 +715,13 @@ def enable_background_sender(self, sender_queue_size=0, sender_queue_timeout=0): to os.fork(). :param sender_queue_size: Set the maximum number of packets to queue for the sender. - Once the queue is full, adding a new packet waits (see sender_queue_timeout) and then, if - still full, drops the oldest queued packet (and any additional expired packets at the front - of the queue) to make room, instead of dropping the new packet. + How many packets to queue before blocking or dropping the packet if the packet queue is already full. Default: 0 (unlimited). :type sender_queue_size: integer, optional - :param sender_queue_timeout: Set how long, in seconds, adding a packet to a full sender queue - will wait for the background sender to free up a slot before falling back to dropping the - oldest queued packet to make room. Default: 0, meaning no waiting happens at all. If set to - None, waits forever for room instead of ever falling back to dropping the oldest packet -- - an explicit opt-in to unbounded backpressure. Note this blocks the calling thread (the one - emitting the metric), not just the background sender. + :param sender_queue_timeout: Set timeout for packet queue operations, in seconds. + How long the application thread is willing to wait for the queue clear up before dropping the metric packet. + If set to None, wait forever. If set to zero drop the packet immediately if the queue is full. + Default: 0 (no wait). :type sender_queue_timeout: float, optional """ @@ -1168,27 +1161,51 @@ def close_buffer(self): def _reset_buffer(self): # type: () -> None with self._buffer_lock: - self._current_buffer_total_size = 0 - self._buffer = [] - # A freshly (re)started buffer starts out replay-safe; it's - # downgraded to False as soon as anything not-replay-safe is - # appended to it. See _send_to_buffer(). - self._buffer_replay_safe = True + # Buffered lines are kept in two separate batches, keyed by + # whether they're replay-safe. Replay safe metrics are posted with + # the timestamp. + self._buffers = {False: [], True: []} # type: Dict[bool, List[Text]] + # Running packet size per buffer, each including the newline that + # will join its lines, so both stay under _max_payload_size + # independently. + self._buffer_sizes = {False: 0, True: 0} # type: Dict[bool, int] def flush(self): # type: () -> None self.flush_buffered_metrics() + def _flush_one_buffer(self, replay_safe): + # type: (bool) -> None + """Flush just the batch holding lines with the given expiry policy. + + Caller must hold self._buffer_lock (an RLock, so a re-entrant flush + from _send_to_buffer() is fine). Only the named batch is touched: the + other one keeps accumulating, which is the whole point of splitting + them. + """ + lines = self._buffers[replay_safe] + if not lines: + return + self._buffers[replay_safe] = [] + self._buffer_sizes[replay_safe] = 0 + self._send_to_server("\n".join(lines), replay_safe) + def flush_buffered_metrics(self): # type: () -> None """ Flush the metrics buffer by sending the data to the server. + + Emits up to two packets, one per expiry policy. Lines keep their + relative order within each packet, but ordering *between* the two is + not preserved: replay-safe payloads carry their own explicit + timestamp, and non-replay-safe ones are timestamped on receipt, so + neither one's meaning depends on where the other lands. """ with self._buffer_lock: - # Only send packets if there are packets to send - if self._buffer: - self._send_to_server("\n".join(self._buffer), self._buffer_replay_safe) - self._reset_buffer() + # Non-replay-safe first: it's the only batch subject to staleness + # expiry, so give it the earliest queue position. + self._flush_one_buffer(False) + self._flush_one_buffer(True) def flush_aggregated_metrics(self): # type: () -> None @@ -1627,14 +1644,6 @@ def _flush_telemetry(self): tags.extend(self.constant_tags) telemetry_tags = ",".join(tags) - # There's no dedicated wire-protocol metric for expired drops (see - # bytes_dropped_expired/packets_dropped_expired for that level of - # detail in-process): they're folded into the *_dropped_queue lines - # reported to the Agent, since both categories share the same root - # cause from the Agent's point of view -- the payload never reached - # a socket write attempt, dropped by the queue itself rather than by - # the writer. Without this, they'd silently vanish even from the - # combined dropped total sent to the Agent. bytes_dropped_queue = self.bytes_dropped_queue + self.bytes_dropped_expired packets_dropped_queue = self.packets_dropped_queue + self.packets_dropped_expired @@ -1900,21 +1909,19 @@ def _xmit_packet_attempt(self, packet, is_telemetry, retry_eligible, retry_deadl def _send_to_buffer(self, packet, replay_safe=False): # type: (str, bool) -> None with self._buffer_lock: - if self._should_flush(len(packet)): - self.flush_buffered_metrics() + replay_safe = bool(replay_safe) - self._buffer.append(packet) + if self._should_flush(len(packet), replay_safe): + self._flush_one_buffer(replay_safe) + + self._buffers[replay_safe].append(packet) # Update the current buffer length, including line break to anticipate # the final packet size - self._current_buffer_total_size += len(packet) + 1 - # The flushed batch is only as replay-safe as its least safe - # member: if anything in it needs to be treated as time-sensitive, - # treat the whole batch that way. - self._buffer_replay_safe = self._buffer_replay_safe and replay_safe - - def _should_flush(self, length_to_be_added): - # type: (int) -> bool - if self._current_buffer_total_size + length_to_be_added + 1 > self._max_payload_size: + self._buffer_sizes[replay_safe] += len(packet) + 1 + + def _should_flush(self, length_to_be_added, replay_safe=False): + # type: (int, bool) -> bool + if self._buffer_sizes[bool(replay_safe)] + length_to_be_added + 1 > self._max_payload_size: return True return False diff --git a/datadog/dogstatsd/sender_queue.py b/datadog/dogstatsd/sender_queue.py index f7db9a181..2b8a92536 100644 --- a/datadog/dogstatsd/sender_queue.py +++ b/datadog/dogstatsd/sender_queue.py @@ -120,21 +120,24 @@ def _make_room_locked(self): now = monotonic() oldest = self._deque.popleft() - # The oldest entry is always dropped to make room. If it happens to - # also be expired, attribute it to staleness rather than to the - # queue being full, since that's the more useful signal. + # The oldest entry is always dropped to make room. if self._expired(oldest, now): self._on_drop_expired(oldest) else: self._on_drop_queue_full(oldest) self._finish_task_locked() - # Keep clearing out additional stale entries left at the front: they - # would otherwise just sit there consuming a slot until they're - # eventually popped. + # Keep clearing out additional stale entries left at the front. + # If any additional entries were cleared out notify not_full as the + # queue will now have available space for additional entries. + reclaimed = 0 while self._deque and self._deque[0] is not Stop and self._expired(self._deque[0], now): self._on_drop_expired(self._deque.popleft()) self._finish_task_locked() + reclaimed += 1 + + if reclaimed: + self._not_full.notify(reclaimed) def put(self, item): # type: (Union[PendingPayload, object]) -> None diff --git a/tests/integration/dogstatsd/test_statsd_sender.py b/tests/integration/dogstatsd/test_statsd_sender.py index 1cd5a50ed..4fc4e735f 100644 --- a/tests/integration/dogstatsd/test_statsd_sender.py +++ b/tests/integration/dogstatsd/test_statsd_sender.py @@ -103,7 +103,8 @@ def test_fork_hooks(disable_background_sender, disable_buffering): assert statsd._flush_thread is None assert statsd._sender_thread is None assert statsd._queue is None or statsd._queue.empty() - assert len(statsd._buffer) == 0 + # Buffered lines are split by expiry policy, so check every batch. + assert not any(statsd._buffers.values()) statsd.post_fork_parent() diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index 5fac674cc..8d6dbab03 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -1749,6 +1749,79 @@ def test_manual_buffer_ops_deprecation(self, mock_warn): self.statsd.close_buffer() self.assertEqual(mock_warn.call_count, 2) + def test_mixed_batch_splits_by_replay_safety(self): + # A queued packet expires as a single unit, so every line in it has to + # share one expiry policy. Batching timestamped lines together with + # plain ones would make the whole packet non-replay-safe and strip the + # timestamped lines of the staleness exemption they're supposed to + # have, dropping them with the batch after ~10s of backlog. The buffer + # must split by policy instead. + sent = [] + self.statsd._send_to_server = lambda packet, replay_safe=False: sent.append((packet, replay_safe)) + + self.statsd.open_buffer() + self.statsd.gauge_with_timestamp("ts.one", 1, timestamp=1700000000) + self.statsd.gauge("plain.one", 2) + self.statsd.gauge_with_timestamp("ts.two", 3, timestamp=1700000001) + self.statsd.gauge("plain.two", 4) + self.statsd.close_buffer() + + # One packet per expiry policy, not one per metric: interleaving must + # not defeat batching. + self.assertEqual(len(sent), 2, "expected exactly one packet per expiry policy, got: {!r}".format(sent)) + + by_policy = dict((replay_safe, packet) for packet, replay_safe in sent) + self.assertEqual(sorted(by_policy.keys()), [False, True]) + + # Assert on structure rather than exact packet text: constant/origin + # tags vary by environment, but which lines land in which packet, and + # in what order, does not. + def names(packet): + return [line.split(":")[0] for line in packet.split("\n")] + + self.assertEqual(names(by_policy[True]), ["ts.one", "ts.two"]) + self.assertEqual(names(by_policy[False]), ["plain.one", "plain.two"]) + + # The invariant that actually matters: no packet mixes the two, and no + # timestamped line ever rides in an expiring packet. + for packet, replay_safe in sent: + lines = packet.split("\n") + timestamped = [line for line in lines if "|T" in line] + if replay_safe: + self.assertEqual(timestamped, lines, "replay-safe packet must be entirely timestamped lines") + else: + self.assertEqual(timestamped, [], "timestamped line leaked into an expiring packet") + + def test_mixed_batch_respects_max_payload_size_per_buffer(self): + # Each buffer has to stay under _max_payload_size on its own, and one + # buffer overflowing must not drag the other one out with it. + sent = [] + self.statsd._send_to_server = lambda packet, replay_safe=False: sent.append((packet, replay_safe)) + self.statsd._max_payload_size = 250 + + self.statsd.open_buffer() + # One small replay-safe line that should still be buffered while the + # plain buffer churns through several flushes. + self.statsd.gauge_with_timestamp("ts.keep", 1, timestamp=1700000000) + for i in range(12): + self.statsd.gauge("plain.filler.{}".format(i), i) + flushes_before_close = len(sent) + self.statsd.close_buffer() + + self.assertGreater(flushes_before_close, 0, "the plain buffer should have overflowed at least once") + self.assertTrue( + all(not replay_safe for _, replay_safe in sent[:flushes_before_close]), + "overflow of the plain buffer must not flush the replay-safe buffer", + ) + for packet, _ in sent: + self.assertLessEqual(len(packet) + 1, self.statsd._max_payload_size) + + # The replay-safe line survived to the final flush, intact and alone. + final_packet, final_replay_safe = sent[-1] + self.assertTrue(final_replay_safe) + self.assertEqual([line.split(":")[0] for line in final_packet.split("\n")], ["ts.keep"]) + self.assertIn("|T1700000000", final_packet) + def test_batching_sequential(self): self.statsd.open_buffer() self.statsd.gauge('discarded.data', 123) @@ -2874,6 +2947,73 @@ def test_sender_queue_put_timeout_falls_back_to_eviction(self): self.assertEqual([p.payload for p in dropped_queue_full], ["first\n"]) self.assertEqual(pending_queue.get().payload, "second\n") + def test_sender_queue_bulk_expired_reclaim_wakes_blocked_producers(self): + # When the eviction path's cleanup loop reclaims *more* than the one + # slot its caller needs, the surplus is real free capacity. Producers + # already parked in put()'s wait-for-room loop have to be told about + # it, otherwise they sleep out their full put_timeout while the queue + # sits half empty. + put_timeout = 1.0 + maxsize = 4 + dropped_expired = [] + pending_queue = SenderQueue( + maxsize=maxsize, + expiry_seconds=100.0, + on_drop_queue_full=lambda item: self.fail("entries are stale: expect expiry drops, not full drops"), + on_drop_expired=dropped_expired.append, + put_timeout=put_timeout, + ) + # Fill to capacity with entries that are already stale, so the + # cleanup loop has something to reclaim beyond the mandatory one. + stale_clock = sender_queue_clock() - 1000.0 + for i in range(maxsize): + pending_queue.put(PendingPayload("stale-{}\n".format(i), stale_clock, False)) + + result = {} + + def evictor(): + # Queue is full and nothing drains it, so this waits out + # put_timeout and then falls back to eviction, whose cleanup loop + # reclaims all remaining stale entries in one go. + pending_queue.put(PendingPayload("evictor\n", sender_queue_clock(), False)) + + def late_waiter(): + t0 = time.time() + pending_queue.put(PendingPayload("late\n", sender_queue_clock(), False)) + result["elapsed"] = time.time() - t0 + + t_evictor = threading.Thread(target=evictor) + t_evictor.start() + # Start the second producer halfway through the first one's timeout so + # its own deadline is strictly later: it must be woken by the bulk + # reclaim, not by its own timeout firing. + time.sleep(put_timeout / 2.0) + t_late = threading.Thread(target=late_waiter) + t_late.start() + + t_evictor.join(timeout=5.0) + t_late.join(timeout=5.0) + self.assertFalse(t_evictor.is_alive()) + self.assertFalse(t_late.is_alive()) + + # All four stale entries went out through the cleanup path. + self.assertEqual( + [p.payload for p in dropped_expired], + ["stale-0\n", "stale-1\n", "stale-2\n", "stale-3\n"], + ) + # Both live payloads made it, and the queue is well under maxsize. + self.assertEqual(pending_queue.qsize(), 2) + + # The heart of it: the late producer had roughly put_timeout/2 left on + # its own clock when capacity opened up. Waking on the reclaim means + # ~put_timeout/2 elapsed; sleeping through it means the full + # put_timeout. Assert it beat its own deadline by a clear margin. + self.assertLess( + result["elapsed"], + put_timeout * 0.9, + "blocked producer slept through its put_timeout despite the bulk reclaim freeing capacity", + ) + def test_sender_queue_requeue_front_never_blocks_on_put_timeout(self): # requeue_front() runs on the background sender thread; it must # never wait on put_timeout, or one stuck retry would stall every From 2571b80d8f66bfcf5583161e62fef5b89713177d Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Mon, 14 Sep 2026 13:14:32 +0100 Subject: [PATCH 07/30] Simplify test --- tests/unit/dogstatsd/test_statsd.py | 42 +++++++++++++++-------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index 8d6dbab03..97b5cf7d7 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -30,7 +30,7 @@ # Datadog libraries from datadog import initialize, statsd from datadog import __version__ as version -from datadog.dogstatsd.base import DEFAULT_BUFFERING_FLUSH_INTERVAL, DEFAULT_HOST, DEFAULT_PORT, DogStatsd, MIN_SEND_BUFFER_SIZE, PendingPayload, SenderQueue, Stop, UDP_OPTIMAL_PAYLOAD_LENGTH, UDS_CONNECT_RETRY_INITIAL_BACKOFF, UDS_OPTIMAL_PAYLOAD_LENGTH +from datadog.dogstatsd.base import DEFAULT_BUFFERING_FLUSH_INTERVAL, DEFAULT_HOST, DEFAULT_PORT, DogStatsd, MIN_SEND_BUFFER_SIZE, PENDING_PAYLOAD_EXPIRY_SECONDS, PendingPayload, SenderQueue, Stop, UDP_OPTIMAL_PAYLOAD_LENGTH, UDS_CONNECT_RETRY_INITIAL_BACKOFF, UDS_OPTIMAL_PAYLOAD_LENGTH from datadog.dogstatsd.sender_queue import monotonic as sender_queue_clock from datadog.dogstatsd.context import TimedContextManagerDecorator from datadog.util.compat import is_higher_py35, is_p3k @@ -2773,32 +2773,34 @@ def test_sender_queue_timeout_blocks_the_calling_thread_through_the_client(self) statsd.stop() def test_bytes_dropped_queue_counts_actual_bytes(self): - # Use a queue of size 1 so the second packet forces the first (oldest) - # one out, then verify bytes_dropped_queue reflects the real byte - # length of the dropped packet (including the appended newline), and - # that the newer payload is the one that survives in the queue. - statsd = DogStatsd( - disable_background_sender=False, - sender_queue_size=1, + # No sender thread: a live one could drain the first payload before the + # third is queued, so nothing would be evicted and the counters below + # would describe a schedule that never happened. Size 2 rather than 1 + # so the eviction order is observable -- with a single slot the evicted + # entry is both the oldest and the newest. + statsd = DogStatsd(disable_background_sender=True) + statsd._queue = SenderQueue( + 2, + PENDING_PAYLOAD_EXPIRY_SECONDS, + statsd._account_dropped_queue_full, + statsd._account_dropped_expired, ) - statsd.socket = FakeSocket() - - # Build a packet whose serialised form we know, then compute its length. - metric_name = "test.metric" - # Send two packets: the first is evicted (dropped) to make room for the second. - statsd._send_to_server(metric_name) - statsd._send_to_server(metric_name + ".second") + first, second, third = "test.metric.first", "test.metric.second", "test.metric.third" + statsd._send_to_server(first) + statsd._send_to_server(second) + statsd._send_to_server(third) # evicts the oldest (first) to make room - expected_bytes = len((metric_name + '\n').encode("utf-8")) - self.assertEqual(statsd.bytes_dropped_queue, expected_bytes) + # bytes_dropped_queue is the real byte length, including the newline + # _send_to_server() appends. + self.assertEqual(statsd.bytes_dropped_queue, len((first + "\n").encode("utf-8"))) self.assertEqual(statsd.packets_dropped_queue, 1) self.assertEqual(statsd.bytes_dropped_expired, 0) self.assertEqual(statsd.packets_dropped_expired, 0) - # The surviving (newest) payload is the one the sender thread will send. - statsd.wait_for_pending() - self.assertEqual(statsd.socket.payloads[0].decode("utf-8"), metric_name + ".second\n") + # Dropping the oldest leaves the two newest queued, in order. + survivors = [statsd._queue.get().payload, statsd._queue.get().payload] + self.assertEqual(survivors, [second + "\n", third + "\n"]) statsd.stop() From 7762b14c2dabe6385dc15aea9c28133ada5540b9 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Mon, 14 Sep 2026 13:49:42 +0100 Subject: [PATCH 08/30] Fix test --- tests/unit/dogstatsd/test_statsd.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index 97b5cf7d7..41f620bfc 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -1797,7 +1797,19 @@ def test_mixed_batch_respects_max_payload_size_per_buffer(self): # buffer overflowing must not drag the other one out with it. sent = [] self.statsd._send_to_server = lambda packet, replay_safe=False: sent.append((packet, replay_safe)) - self.statsd._max_payload_size = 250 + + # Measure a real serialised line and size the cap from it. Hard-coding + # a byte count would make the test depend on how long constant/origin + # tags happen to make each line in this environment: too small and a + # single line breaches the cap, too large and nothing ever overflows. + self.statsd.open_buffer() + self.statsd.gauge("plain.filler.0", 0) + line_size = self.statsd._buffer_sizes[False] + self.statsd.close_buffer() + del sent[:] + + # Room for two lines, so every third one forces a flush. + self.statsd._max_payload_size = line_size * 2 + 1 self.statsd.open_buffer() # One small replay-safe line that should still be buffered while the From dce1a3b4e7dfd30fe311a9b0da9182e834b725e0 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Mon, 14 Sep 2026 14:24:36 +0100 Subject: [PATCH 09/30] Move monotonic to compat --- datadog/dogstatsd/base.py | 3 +-- datadog/dogstatsd/context.py | 7 +------ datadog/dogstatsd/context_async.py | 6 ++---- datadog/dogstatsd/sender_queue.py | 7 +------ datadog/threadstats/base.py | 6 +----- datadog/util/compat.py | 10 ++++++++++ tests/performance/test_sender_queue_benchmark.py | 2 +- tests/unit/dogstatsd/test_statsd.py | 2 +- 8 files changed, 18 insertions(+), 25 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index 246b53bf2..2580b6dcc 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -50,9 +50,8 @@ PendingPayload, Stop, PENDING_PAYLOAD_EXPIRY_SECONDS, - monotonic, ) -from datadog.util.compat import text, urlparse +from datadog.util.compat import monotonic, text, urlparse from datadog.util.format import normalize_tags, validate_cardinality from datadog.version import __version__ diff --git a/datadog/dogstatsd/context.py b/datadog/dogstatsd/context.py index ec27b30dd..3e2df8d86 100644 --- a/datadog/dogstatsd/context.py +++ b/datadog/dogstatsd/context.py @@ -6,14 +6,9 @@ import sys -try: - from time import monotonic # type: ignore[attr-defined] -except ImportError: - from time import time as monotonic - # datadog from datadog.dogstatsd.context_async import _get_wrapped_co -from datadog.util.compat import iscoroutinefunction +from datadog.util.compat import iscoroutinefunction, monotonic if sys.version_info[:2] >= (3, 5): diff --git a/datadog/dogstatsd/context_async.py b/datadog/dogstatsd/context_async.py index 7cb0f826c..af976f3d3 100644 --- a/datadog/dogstatsd/context_async.py +++ b/datadog/dogstatsd/context_async.py @@ -23,10 +23,8 @@ # https://github.com/python/mypy/issues/6897 ASYNC_SOURCE = r''' from functools import wraps -try: - from time import monotonic -except ImportError: - from time import time as monotonic + +from datadog.util.compat import monotonic def _get_wrapped_co(self, func): diff --git a/datadog/dogstatsd/sender_queue.py b/datadog/dogstatsd/sender_queue.py index 2b8a92536..c419ef4c5 100644 --- a/datadog/dogstatsd/sender_queue.py +++ b/datadog/dogstatsd/sender_queue.py @@ -2,12 +2,7 @@ import sys import threading -try: - # Python 3.3+ - from time import monotonic -except ImportError: - # Python 2: no monotonic clock available, fall back to wall clock. - from time import time as monotonic +from datadog.util.compat import monotonic if sys.version_info[:2] >= (3, 5): from typing import Callable, Optional, Union # noqa: F401 diff --git a/datadog/threadstats/base.py b/datadog/threadstats/base.py index 9dbbdb9be..69f474383 100644 --- a/datadog/threadstats/base.py +++ b/datadog/threadstats/base.py @@ -16,17 +16,13 @@ from functools import wraps from time import time -try: - from time import monotonic # type: ignore[attr-defined] -except ImportError: - from time import time as monotonic - # datadog from datadog.api.exceptions import ApiNotInitialized from datadog.threadstats.constants import MetricType from datadog.threadstats.events import EventsAggregator from datadog.threadstats.metrics import MetricsAggregator, Counter, Gauge, Histogram, Timing, Distribution, Set from datadog.threadstats.reporters import HttpReporter +from datadog.util.compat import monotonic # Loggers log = logging.getLogger("datadog.threadstats") diff --git a/datadog/util/compat.py b/datadog/util/compat.py index febb804b9..4b863e8e7 100644 --- a/datadog/util/compat.py +++ b/datadog/util/compat.py @@ -109,6 +109,16 @@ def emit(self, record): pass +# Python >= 3.3 +if sys.version_info >= (3, 3): + from time import monotonic +# Python 2.x: there is no monotonic clock, so fall back to the wall clock. +# Callers that compare two readings (elapsed time, queue entry age) are +# therefore sensitive to the clock being stepped backwards on Python 2. +else: + from time import time as monotonic + + def _is_py_version_higher_than(major, minor=0): # type: (int, int) -> bool """ diff --git a/tests/performance/test_sender_queue_benchmark.py b/tests/performance/test_sender_queue_benchmark.py index c904cceb4..c77b6c99c 100644 --- a/tests/performance/test_sender_queue_benchmark.py +++ b/tests/performance/test_sender_queue_benchmark.py @@ -58,8 +58,8 @@ from datadog.dogstatsd.sender_queue import ( # noqa: E402 PendingPayload, SenderQueue, - monotonic, ) +from datadog.util.compat import monotonic # noqa: E402 QUICK = "--quick" in sys.argv diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index 41f620bfc..80a5d39fe 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -31,7 +31,7 @@ from datadog import initialize, statsd from datadog import __version__ as version from datadog.dogstatsd.base import DEFAULT_BUFFERING_FLUSH_INTERVAL, DEFAULT_HOST, DEFAULT_PORT, DogStatsd, MIN_SEND_BUFFER_SIZE, PENDING_PAYLOAD_EXPIRY_SECONDS, PendingPayload, SenderQueue, Stop, UDP_OPTIMAL_PAYLOAD_LENGTH, UDS_CONNECT_RETRY_INITIAL_BACKOFF, UDS_OPTIMAL_PAYLOAD_LENGTH -from datadog.dogstatsd.sender_queue import monotonic as sender_queue_clock +from datadog.util.compat import monotonic as sender_queue_clock from datadog.dogstatsd.context import TimedContextManagerDecorator from datadog.util.compat import is_higher_py35, is_p3k from tests.util.contextmanagers import preserve_environment_variable, EnvVars From 6df5d2d971092cef6e3bc06c1855b9f2f4844ab6 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Mon, 14 Sep 2026 17:36:10 +0100 Subject: [PATCH 10/30] Add timeout to stop and wait_for_pending --- datadog/dogstatsd/base.py | 90 +++++++++--- datadog/dogstatsd/sender_queue.py | 26 +++- tests/unit/dogstatsd/test_statsd.py | 215 ++++++++++++++++++++++++++++ 3 files changed, 310 insertions(+), 21 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index 2580b6dcc..06a06f0d4 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -731,15 +731,21 @@ def enable_background_sender(self, sender_queue_size=0, sender_queue_timeout=0): self._start_sender_thread() - def disable_background_sender(self): - # type: () -> None + def disable_background_sender(self, timeout=None): + # type: (Optional[float]) -> bool """Disable background sender mode. This call will block until all previously queued payloads are sent. + + :param timeout: Maximum number of seconds to wait for the sender thread + to drain the queue and exit. None (the default) waits indefinitely. + :type timeout: float, optional + :return: True if the sender thread finished, False if timeout elapsed + while it was still running. """ with self._config_lock: self._sender_enabled = False - self._stop_sender_thread() + return self._stop_sender_thread(timeout) def disable_telemetry(self): # type: () -> None @@ -2154,18 +2160,28 @@ def _start_sender_thread(self): self._sender_thread.daemon = True self._sender_thread.start() - def _stop_sender_thread(self): - # type: () -> None + def _stop_sender_thread(self, timeout=None): + # type: (Optional[float]) -> bool # Lock ensures that nothing gets added to the queue after we disable it. with self._buffer_lock: - if not self._queue: - return - self._queue.put(Stop) - self._queue = None + if self._queue is not None: + # put() lets the Stop sentinel past the size limit, so this + # never blocks even when the queue is full. + self._queue.put(Stop) + self._queue = None + + thread = self._sender_thread + if thread is None: + return True + + thread.join(timeout) + if thread.is_alive(): + # Timed out. Keep the handle so a later call can wait for it again + # rather than losing track of a still-running thread. + return False - if self._sender_thread is not None: - self._sender_thread.join() self._sender_thread = None + return True def _sender_main_loop(self, pending_queue): # type: (SenderQueue) -> None @@ -2199,10 +2215,16 @@ def _sender_main_loop(self, pending_queue): pending_queue.task_done() backoff = UDS_CONNECT_RETRY_INITIAL_BACKOFF - def wait_for_pending(self): - # type: () -> None + def wait_for_pending(self, timeout=None): + # type: (Optional[float]) -> bool """ Flush the buffer and wait for all queued payloads to be written to the server. + + :param timeout: Maximum number of seconds to wait for the queue to + drain. None (the default) waits indefinitely. + :type timeout: float, optional + :return: True if every queued payload has been sent, dropped or + expired, False if timeout elapsed with payloads still outstanding. """ self.flush_buffered_metrics() @@ -2212,8 +2234,10 @@ def wait_for_pending(self): # check and join later. queue = self._queue - if queue is not None: - queue.join() + if queue is None: + return True + + return queue.join(timeout) def pre_fork(self): # type: () -> None @@ -2266,21 +2290,51 @@ def post_fork_child(self): self._start_flush_thread() self._start_sender_thread() - def stop(self): - # type: () -> None + def stop(self, timeout=None): + # type: (Optional[float]) -> bool """Stop the client. Disable buffering, aggregation, background sender and flush any pending payloads to the server. Client remains usable after this method, but sending metrics may block if socket_timeout is enabled. + + :param timeout: Maximum number of seconds to wait for the background + sender to drain its queue and exit. None (the default) waits + indefinitely, however long that takes. + :type timeout: float, optional + :return: True if the background sender drained and stopped, and the + final flush and socket close ran. False if timeout elapsed first, + in which case the sender thread is still running, the final flush + and close were skipped (see below), and a later stop() call can + wait for the thread again. """ - self.disable_background_sender() + stopped = self.disable_background_sender(timeout) self._disable_buffering = True self._disable_aggregation = True + + if not stopped: + # We gave up waiting, so the sender thread is still running and + # still owns the socket -- it can be parked inside a send() with + # _socket_lock held. Flushing or closing here would block on that + # same lock for as long as the sender stays wedged, which would + # make timeout meaningless: the caller asked for a bounded stop(). + # Pushing more data through that socket could not succeed anyway, + # and closing it from under a thread mid-write is not safe. Leave + # it open; the OS reclaims the fd when the process exits, and the + # sender thread is a daemon so it never holds up interpreter + # shutdown. + log.warning( + "stop() timed out after %ss with the background sender still running; " + "skipping the final flush and socket close", + timeout, + ) + return False + self.flush_aggregated_metrics() self.flush_buffered_metrics() self.close_socket() + return True statsd = DogStatsd() diff --git a/datadog/dogstatsd/sender_queue.py b/datadog/dogstatsd/sender_queue.py index c419ef4c5..0116dcd4f 100644 --- a/datadog/dogstatsd/sender_queue.py +++ b/datadog/dogstatsd/sender_queue.py @@ -233,11 +233,31 @@ def task_done(self): with self._all_tasks_done: self._finish_task_locked() - def join(self): - # type: () -> None + def join(self, timeout=None): + # type: (Optional[float]) -> bool + """Wait until every queued payload has been sent, dropped or expired. + + :param timeout: Maximum number of seconds to wait. None (the default) + waits indefinitely. + :return: True if nothing is outstanding any more, False if timeout + elapsed while payloads were still in flight. + """ with self._all_tasks_done: + if timeout is None: + while self._unfinished_tasks: + self._all_tasks_done.wait() + return True + + # Condition.wait()'s return value can't be used to detect a + # timeout: on Python 2 it is always None. Track the deadline + # ourselves instead, the same way put() does for put_timeout. + deadline = monotonic() + timeout while self._unfinished_tasks: - self._all_tasks_done.wait() + remaining = deadline - monotonic() + if remaining <= 0: + return False + self._all_tasks_done.wait(remaining) + return True def qsize(self): # type: () -> int diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index 80a5d39fe..18882f88d 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -2758,6 +2758,221 @@ def test_sender_queue_no_timeout(self): statsd = DogStatsd(disable_background_sender=False, sender_queue_timeout=None) statsd.stop() + def _call_bounded(self, func, args=(), limit=5.0): + """Call func in a worker thread, failing if it doesn't return in time. + + Everything exercised below exists to *bound* a wait, so a regression + that reintroduces an unbounded wait should surface as a clear failure + rather than hanging the whole suite until CI kills the job. + """ + result = {} + + def run(): + result["value"] = func(*args) + + t = threading.Thread(target=run) + t.daemon = True + t.start() + t.join(limit) + self.assertFalse( + t.is_alive(), + "{} did not return within {}s: timeout not honoured".format(getattr(func, "__name__", func), limit), + ) + return result["value"] + + def test_queue_join_timeout(self): + # join(timeout) must report whether the queue actually drained, and must + # not rely on Condition.wait()'s return value (always None on Python 2). + pending_queue = SenderQueue( + maxsize=0, + expiry_seconds=100.0, + on_drop_queue_full=lambda item: self.fail("unexpected full drop"), + on_drop_expired=lambda item: self.fail("unexpected expiry drop"), + ) + self.assertIs(pending_queue.join(0), True) + self.assertIs(pending_queue.join(), True) + + pending_queue.put(PendingPayload("first\n", sender_queue_clock(), False)) + + # Nothing is draining it, so a bounded join must give up and say so + # rather than blocking forever or claiming success. + t0 = time.time() + self.assertIs(self._call_bounded(pending_queue.join, (0.2,)), False) + self.assertGreaterEqual(time.time() - t0, 0.2) + + # timeout=0 is a non-blocking poll. + t0 = time.time() + self.assertIs(self._call_bounded(pending_queue.join, (0,)), False) + self.assertLess(time.time() - t0, 0.2) + + # Once the payload is accounted for, join() succeeds. + pending_queue.get() + pending_queue.task_done() + self.assertIs(pending_queue.join(0), True) + + def test_queue_join_timeout_returns_as_soon_as_the_queue_drains(self): + # A generous timeout must not be waited out: join() returns as soon as + # the last task is done. + pending_queue = SenderQueue( + maxsize=0, + expiry_seconds=100.0, + on_drop_queue_full=lambda item: None, + on_drop_expired=lambda item: None, + ) + pending_queue.put(PendingPayload("first\n", sender_queue_clock(), False)) + + def drain(): + time.sleep(0.2) + pending_queue.get() + pending_queue.task_done() + + t = threading.Thread(target=drain) + t.start() + try: + t0 = time.time() + self.assertIs(pending_queue.join(10.0), True) + elapsed = time.time() - t0 + finally: + t.join(timeout=5.0) + self.assertLess(elapsed, 5.0, "join() should return on drain, not wait out the whole timeout") + + def test_wait_for_pending_timeout(self): + # A queue with no sender thread draining it: wait_for_pending() must + # give up and report False rather than blocking forever. Done without a + # thread on purpose -- the default transport is UDP, where a send + # succeeds even with nothing listening, so "assign no socket" would not + # reliably keep a payload pending. + statsd = DogStatsd(disable_background_sender=True, disable_telemetry=True) + statsd._queue = SenderQueue( + 0, + PENDING_PAYLOAD_EXPIRY_SECONDS, + lambda item: None, + lambda item: None, + ) + statsd._send_to_server("test.metric:1|c") + + t0 = time.time() + self.assertIs(self._call_bounded(statsd.wait_for_pending, (0.2,)), False) + self.assertGreaterEqual(time.time() - t0, 0.2) + + # timeout=0 is a non-blocking poll. + self.assertIs(self._call_bounded(statsd.wait_for_pending, (0,)), False) + + def test_wait_for_pending_returns_true_with_no_queue(self): + # Nothing queued (background sender disabled) is trivially "drained". + statsd = DogStatsd(disable_background_sender=True, disable_telemetry=True) + self.assertIsNone(statsd._queue) + self.assertIs(statsd.wait_for_pending(), True) + self.assertIs(statsd.wait_for_pending(0), True) + + def test_stop_timeout_reports_failure_and_keeps_the_thread_joinable(self): + # A wedged sender must not make stop() hang forever when a timeout is + # given, and stop() must say it didn't finish. + statsd = DogStatsd(disable_background_sender=False, disable_telemetry=True) + release = threading.Event() + wedged = statsd._sender_thread + + # Wedge the sender inside a send so it can't observe Stop. + def blocking_xmit(packet, queue_mode=False): + release.wait(10.0) + return True + + statsd._xmit_packet_with_telemetry = blocking_xmit + statsd._send_to_server("test.metric:1|c") + time.sleep(0.1) # let the sender pick it up and wedge + + try: + t0 = time.time() + self.assertIs(self._call_bounded(statsd.stop, (0.2,)), False) + self.assertGreaterEqual(time.time() - t0, 0.2) + # The handle is retained so the thread isn't lost. + self.assertIs(statsd._sender_thread, wedged) + self.assertTrue(wedged.is_alive()) + + # Unwedge: a second stop() now succeeds and clears the handle. + release.set() + self.assertIs(statsd.stop(5.0), True) + self.assertIsNone(statsd._sender_thread) + finally: + release.set() + wedged.join(timeout=5.0) + + def test_stop_timeout_is_bounded_while_the_sender_holds_the_socket_lock(self): + # The wedge that matters in practice: the sender is parked inside a + # blocking send() and therefore owns _socket_lock. stop()'s own + # close_socket()/flush calls want that same lock, so without care they + # block for as long as the sender stays stuck and the timeout means + # nothing. stop() must still return within its timeout. + statsd = DogStatsd(disable_background_sender=False, disable_telemetry=True) + release = threading.Event() + entered_send = threading.Event() + wedged = statsd._sender_thread + + class BlockingSocket(object): + def send(self, data): + entered_send.set() + release.wait(30.0) + return len(data) + + def sendall(self, data): + return self.send(data) + + def close(self): + pass + + def setblocking(self, *args): + pass + + def settimeout(self, *args): + pass + + def getsockopt(self, *args): + return MIN_SEND_BUFFER_SIZE + + def setsockopt(self, *args): + pass + + statsd.socket = BlockingSocket() + for i in range(5): + statsd._send_to_server("test.metric.{}:1|c".format(i)) + self.assertTrue(entered_send.wait(5.0), "sender never reached send()") + + try: + t0 = time.time() + self.assertIs(self._call_bounded(statsd.stop, (0.2,)), False) + elapsed = time.time() - t0 + self.assertGreaterEqual(elapsed, 0.2) + self.assertLess(elapsed, 5.0, "stop() blocked well past its timeout") + + # The socket was deliberately left alone: closing it under a thread + # that is mid-send is both unsafe and the thing that would block. + self.assertIsNotNone(statsd.socket) + self.assertTrue(wedged.is_alive()) + self.assertIs(statsd._sender_thread, wedged) + finally: + release.set() + wedged.join(timeout=5.0) + + def test_stop_and_wait_for_pending_default_to_waiting_forever(self): + # The default must stay unbounded: a slow-but-progressing sender is + # waited out completely, with nothing left pending. + statsd = DogStatsd(disable_background_sender=False, disable_telemetry=True) + sent = [] + + def slow_xmit(packet, queue_mode=False): + time.sleep(0.05) + sent.append(packet) + return True + + statsd._xmit_packet_with_telemetry = slow_xmit + for i in range(5): + statsd._send_to_server("test.metric.{}:1|c".format(i)) + + self.assertIs(statsd.wait_for_pending(), True) + self.assertEqual(len(sent), 5, "unbounded wait_for_pending() must drain everything") + self.assertIs(statsd.stop(), True) + self.assertIsNone(statsd._sender_thread) + def test_sender_queue_timeout_blocks_the_calling_thread_through_the_client(self): # End-to-end: sender_queue_timeout configured on the real client # actually makes statsd.increment() (the calling/application thread) From 53083df353cd9f17754a19d21fc682bce5e5970b Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 15 Sep 2026 11:14:14 +0100 Subject: [PATCH 11/30] Fix leaving the sender thread and queue in an incoherent state after timeout --- datadog/dogstatsd/base.py | 32 ++++- tests/unit/dogstatsd/test_statsd.py | 191 ++++++++++++++++++++++++++++ 2 files changed, 217 insertions(+), 6 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index 06a06f0d4..abc1ecb2e 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -2168,18 +2168,29 @@ def _stop_sender_thread(self, timeout=None): # put() lets the Stop sentinel past the size limit, so this # never blocks even when the queue is full. self._queue.put(Stop) - self._queue = None thread = self._sender_thread if thread is None: + # Nothing left to stop: no thread ever runs, so clear any residual + # queue. + with self._buffer_lock: + self._queue = None return True thread.join(timeout) if thread.is_alive(): - # Timed out. Keep the handle so a later call can wait for it again - # rather than losing track of a still-running thread. + # Timed out. Leave _queue in place: it is what stops + # _start_sender_thread() from minting a second sender, and it keeps + # wait_for_pending()/_send_to_server() targeting the real queue. + # The sender thread clears this state itself when it eventually + # exits (see _sender_main_loop). return False + # _sender_main_loop clears this state on its way out when the thread + # has drained the queue, so this may already be a no-op; it also covers + # a thread that exited without draining (e.g. never actually started). + with self._buffer_lock: + self._queue = None self._sender_thread = None return True @@ -2190,6 +2201,11 @@ def _sender_main_loop(self, pending_queue): item = pending_queue.get() if item is Stop: pending_queue.task_done() + with self._buffer_lock: + if self._queue is pending_queue: + self._queue = None + if self._sender_thread is threading.current_thread(): + self._sender_thread = None return # next line has type ignore because the type checker cannot @@ -2304,9 +2320,13 @@ def stop(self, timeout=None): :type timeout: float, optional :return: True if the background sender drained and stopped, and the final flush and socket close ran. False if timeout elapsed first, - in which case the sender thread is still running, the final flush - and close were skipped (see below), and a later stop() call can - wait for the thread again. + in which case the sender thread is still running and neither the + final flush nor the socket close ran (see below). Do not call + stop() again while that sender is still running: it queues a + second internal shutdown signal that is never drained, which can + make wait_for_pending() block forever on the abandoned queue. Use + wait_for_pending() to wait for the sender instead, then call + stop() again once it has actually stopped. """ stopped = self.disable_background_sender(timeout) diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index 18882f88d..a4bb3ec3d 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -2953,6 +2953,197 @@ def setsockopt(self, *args): release.set() wedged.join(timeout=5.0) + def test_stop_timeout_then_restart_does_not_orphan_the_sender(self): + # After a timed-out stop() the sender is still running and the queue is + # intentionally retained. Re-enabling the background sender must not + # mint a SECOND thread alongside the first (which is what happened when + # the queue was dropped on timeout): the client would leak a thread and + # end up with two senders sharing one socket/lock. + statsd = DogStatsd(disable_background_sender=False, disable_telemetry=True) + release = threading.Event() + entered_send = threading.Event() + wedged = statsd._sender_thread + + def blocking_send(self, data): + entered_send.set() + release.wait(30.0) + return len(data) + + statsd.socket = type("W", (object,), { + "send": blocking_send, + "sendall": blocking_send, + "close": lambda self: None, + "setblocking": lambda self, *a: None, + "settimeout": lambda self, *a: None, + "getsockopt": lambda self, *a: MIN_SEND_BUFFER_SIZE, + "setsockopt": lambda self, *a: None, + })() + statsd._send_to_server("test.metric:1|c") + self.assertTrue(entered_send.wait(5.0)) + + try: + self.assertIs(self._call_bounded(statsd.stop, (0.2,)), False) + self.assertTrue(wedged.is_alive()) + self.assertIsNotNone(statsd._queue, "queue must be retained so state stays coherent") + + statsd.enable_background_sender() + # This identity check IS the proof there's no orphan: if a second + # thread had been minted, _sender_thread would now point at it + # instead of at wedged. (Deliberately not also scanning + # threading.enumerate() for same-named threads process-wide: this + # file has other, unrelated tests that spin up daemon + # "DogStatsd_sender_thread"s of their own, so a global count is not + # a property this test can own.) + self.assertIs( + statsd._sender_thread, wedged, + "re-enabling after a timed-out stop must not start a second sender thread", + ) + self.assertTrue(wedged.is_alive()) + finally: + release.set() + wedged.join(timeout=5.0) + + def test_stop_timeout_then_unwedge_then_restart_starts_fresh(self): + # Once the timed-out thread finally drains and exits, the client + # self-heals: the next stop() reports success and cleans up, and a + # re-enabled background sender is a brand new thread over a new queue. + statsd = DogStatsd(disable_background_sender=False, disable_telemetry=True) + release = threading.Event() + entered_send = threading.Event() + wedged = statsd._sender_thread + + def blocking_send(self, data): + entered_send.set() + release.wait(30.0) + return len(data) + + statsd.socket = type("W", (object,), { + "send": blocking_send, + "sendall": blocking_send, + "close": lambda self: None, + "setblocking": lambda self, *a: None, + "settimeout": lambda self, *a: None, + "getsockopt": lambda self, *a: MIN_SEND_BUFFER_SIZE, + "setsockopt": lambda self, *a: None, + })() + statsd._send_to_server("test.metric:1|c") + self.assertTrue(entered_send.wait(5.0)) + + try: + self.assertIs(self._call_bounded(statsd.stop, (0.2,)), False) + release.set() # let the wedged send finish; the thread drains and exits + self.assertIs(self._call_bounded(statsd.stop, (5.0,)), True) + self.assertIsNone(statsd._queue) + self.assertIsNone(statsd._sender_thread) + + statsd.enable_background_sender() + self.assertIsNotNone(statsd._sender_thread) + self.assertIsNot(statsd._sender_thread, wedged, "a fresh thread should start") + self.assertTrue(statsd._sender_thread.is_alive()) + self.assertFalse(wedged.is_alive(), "the old thread must have actually exited, not just been forgotten") + finally: + release.set() + # The fresh thread's queue is empty and has never been sent a Stop + # sentinel, so joining statsd._sender_thread directly would just + # time out waiting on a get() that never returns -- leaking the + # thread into later tests. stop() sends Stop and joins correctly + # regardless of which thread/queue is current. + statsd.stop(5.0) + + def test_sender_self_heals_when_the_timed_out_thread_finishes_on_its_own(self): + # The timed-out thread can exit without a second stop() ever being + # called. The client must detect that and allow a fresh sender, rather + # than keeping the dead thread / stale queue around and silently + # refusing to start a new one (metrics would then queue up and drop). + statsd = DogStatsd(disable_background_sender=False, disable_telemetry=True) + release = threading.Event() + entered_send = threading.Event() + wedged = statsd._sender_thread + + def blocking_send(self, data): + entered_send.set() + release.wait(30.0) + return len(data) + + statsd.socket = type("W", (object,), { + "send": blocking_send, + "sendall": blocking_send, + "close": lambda self: None, + "setblocking": lambda self, *a: None, + "settimeout": lambda self, *a: None, + "getsockopt": lambda self, *a: MIN_SEND_BUFFER_SIZE, + "setsockopt": lambda self, *a: None, + })() + statsd._send_to_server("test.metric:1|c") + self.assertTrue(entered_send.wait(5.0)) + + try: + self.assertIs(self._call_bounded(statsd.stop, (0.2,)), False) + self.assertIsNotNone(statsd._queue) + self.assertIs(statsd._sender_thread, wedged) + + # Caller moved on and never called stop() again; the wedged send + # eventually completes and the thread drains and exits on its own. + release.set() + wedged.join(timeout=5.0) + self.assertFalse(wedged.is_alive()) + + # The exit must have healed the client state so a re-enable works. + self.assertIsNone(statsd._queue) + self.assertIsNone(statsd._sender_thread) + statsd.enable_background_sender() + self.assertIsNot( + statsd._sender_thread, wedged, + "a timed-out thread that finished on its own must not block a restart", + ) + self.assertTrue(statsd._sender_thread.is_alive()) + finally: + release.set() + # Same reason as the sibling test above: the fresh thread started + # by enable_background_sender() has an empty queue and was never + # sent Stop, so joining it directly would time out and leak it. + statsd.stop(5.0) + + def test_wait_for_pending_is_honest_after_a_stop_timeout(self): + # After a timed-out stop(), the queue is retained (not shown as empty), + # so wait_for_pending() still reports the truth -- False while the + # sender is draining, True once it has actually drained -- rather than + # claiming "drained" while a live thread is still sending. + statsd = DogStatsd(disable_background_sender=False, disable_telemetry=True) + release = threading.Event() + entered_send = threading.Event() + wedged = statsd._sender_thread + + def blocking_send(self, data): + entered_send.set() + release.wait(30.0) + return len(data) + + statsd.socket = type("W", (object,), { + "send": blocking_send, + "sendall": blocking_send, + "close": lambda self: None, + "setblocking": lambda self, *a: None, + "settimeout": lambda self, *a: None, + "getsockopt": lambda self, *a: MIN_SEND_BUFFER_SIZE, + "setsockopt": lambda self, *a: None, + })() + statsd._send_to_server("test.metric:1|c") + self.assertTrue(entered_send.wait(5.0)) + + try: + self.assertIs(self._call_bounded(statsd.stop, (0.2,)), False) + self.assertIs( + self._call_bounded(statsd.wait_for_pending, (0.2,)), False, + "wait_for_pending() must not claim drained while the sender is still running", + ) + + release.set() # sender finishes the send, hits Stop, drains, exits + self.assertIs(self._call_bounded(statsd.wait_for_pending, (5.0,)), True) + finally: + release.set() + wedged.join(timeout=5.0) + def test_stop_and_wait_for_pending_default_to_waiting_forever(self): # The default must stay unbounded: a slow-but-progressing sender is # waited out completely, with nothing left pending. From 4e5e2d173b085e4df67f74f5ccc08aa531fcc73e Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 15 Sep 2026 12:29:59 +0100 Subject: [PATCH 12/30] Socket_connect_retry is now a boolean option --- datadog/dogstatsd/base.py | 221 ++++++-------------- tests/unit/dogstatsd/test_statsd.py | 310 ++++++++++------------------ 2 files changed, 171 insertions(+), 360 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index abc1ecb2e..af74fac45 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -181,10 +181,11 @@ def reverse(self): # Socket options MIN_SEND_BUFFER_SIZE = 32 * 1024 -DEFAULT_SOCKET_CONNECT_TIMEOUT = 0 +# Backoff for the background sender's own retry-by-requeuing loop (see +# _sender_main_loop). Not used for direct/synchronous sends, which never +# retry a connection failure regardless of socket_connect_retry. UDS_CONNECT_RETRY_INITIAL_BACKOFF = 0.025 -UDS_CONNECT_RETRY_MAX_BACKOFF = 1.0 -UDS_TRANSIENT_CONNECT_ERRORS = set([errno.ENOENT, errno.ECONNREFUSED]) +UDS_CONNECT_RETRY_MAX_BACKOFF = 60.0 # Errors seen while sending on an already-connected socket that indicate the # peer went away (e.g. the agent crashed/restarted). These are worth a single # reconnect-and-resend attempt instead of dropping the packet outright. @@ -307,7 +308,7 @@ def __init__( sender_queue_size=0, # type: int sender_queue_timeout=0, # type: Optional[float] track_instance=True, # type: bool - socket_connect_timeout=DEFAULT_SOCKET_CONNECT_TIMEOUT, # type: Optional[float] + socket_connect_retry=False, # type: bool ): # type: (...) -> None """ Initialize a DogStatsd object. @@ -473,10 +474,15 @@ def __init__( This option does not affect hostname resolution when using UDP. :type socket_timeout: float - :param socket_connect_timeout: Set the timeout for connecting to a UNIX socket, in seconds. Optional. - Transient connection failures are retried within this timeout. If set to zero or None, do not retry. - Default: 0 (no retries). - :type socket_connect_timeout: float + :param socket_connect_retry: Only affects the background sender (disable_background_sender=False). + If True, a connection failure while sending a queued payload to a UNIX socket is retried + indefinitely, backing off up to once a minute between attempts, instead of dropping the payload + immediately. This is safe to enable because a payload stuck retrying is still subject to the + sender queue's own expiry: it is eventually dropped as stale rather than retried forever if the + Agent never comes back. Direct/synchronous sends (the default mode) always fail fast on a + connection error and are unaffected by this setting. + Default: False (fail fast, matching the previous socket_connect_timeout=0 default). + :type socket_connect_retry: bool :param telemetry_socket_timeout: Set timeout for the telemetry socket operations. Optional. Effective only if either telemetry_host or telemetry_socket_path are set. @@ -540,7 +546,7 @@ def __init__( # Connection self._max_buffer_len = max_buffer_len self.socket_timeout = socket_timeout - self.socket_connect_timeout = socket_connect_timeout + self.socket_connect_retry = socket_connect_retry if socket_path is not None: self.socket_path = socket_path # type: Optional[text] self.host = None @@ -958,23 +964,14 @@ def resolve_host(host, use_default_route): return get_default_route() - def get_socket(self, telemetry=False, connect_timeout=None): - # type: (bool, Optional[float]) -> _Socket + def get_socket(self, telemetry=False): + # type: (bool) -> _Socket """ Return a connected socket. Note: connect the socket before assigning it to the class instance to avoid bad thread race conditions. - - :param connect_timeout: Optional override for the UDS connect-retry - budget passed to _get_uds_socket, in place of - self.socket_connect_timeout. The send-retry loop in _xmit_packet - uses this to pass down how much of its overall deadline is left, - so a retried attempt doesn't get a brand-new full budget. """ - if connect_timeout is None: - connect_timeout = self.socket_connect_timeout - with self._socket_lock: if telemetry and self._dedicated_telemetry_destination(): if not self.telemetry_socket: @@ -982,7 +979,6 @@ def get_socket(self, telemetry=False, connect_timeout=None): self.telemetry_socket = self._get_uds_socket( self.telemetry_socket_path, self.telemetry_socket_timeout, - connect_timeout, ) else: self.telemetry_socket = self._get_udp_socket( @@ -998,7 +994,6 @@ def get_socket(self, telemetry=False, connect_timeout=None): self.socket = self._get_uds_socket( self.socket_path, self.socket_timeout, - connect_timeout, ) else: self.socket = self._get_udp_socket( @@ -1034,8 +1029,10 @@ def _ensure_min_send_buffer_size(cls, sock, min_size=MIN_SEND_BUFFER_SIZE): log.debug("Socket send buffer increased to %dkb", min_size / 1024) @classmethod - def _get_uds_socket(cls, socket_path, timeout, connect_timeout): - # type: (Text, Optional[float], Optional[float]) -> _Socket + def _get_uds_socket(cls, socket_path, timeout): + # type: (Text, Optional[float]) -> _Socket + """Make one connect attempt per candidate socket kind. + """ valid_socket_kinds = [socket.SOCK_DGRAM, socket.SOCK_STREAM] if socket_path.startswith(UNIX_ADDRESS_DATAGRAM_SCHEME): valid_socket_kinds = [socket.SOCK_DGRAM] @@ -1047,54 +1044,27 @@ def _get_uds_socket(cls, socket_path, timeout, connect_timeout): socket_path = socket_path[len(UNIX_ADDRESS_SCHEME):] last_error = socket.timeout("timed out connecting to UDS socket") # type: Exception - deadline = None - if connect_timeout and connect_timeout > 0: - deadline = time.time() + connect_timeout - for socket_kind in valid_socket_kinds: # py2 stores socket kinds differently than py3, determine the name independently from version sk_name = {socket.SOCK_STREAM: "stream", socket.SOCK_DGRAM: "datagram"}[socket_kind] - - backoff = UDS_CONNECT_RETRY_INITIAL_BACKOFF - - while deadline is None or time.time() < deadline: - sock = None - try: - connect_attempt_timeout = timeout - if deadline is not None: - connect_attempt_timeout = deadline - time.time() - if connect_attempt_timeout <= 0: - break - - sock = socket.socket(socket.AF_UNIX, socket_kind) - sock.settimeout(connect_attempt_timeout) - cls._ensure_min_send_buffer_size(sock) - sock.connect(socket_path) - sock.settimeout(timeout) - log.debug("Connected to socket %s with kind %s", socket_path, sk_name) - return sock - except Exception as e: - if sock is not None: - sock.close() - log.debug("Failed to connect to %s with kind %s: %s", socket_path, sk_name, e) - if getattr(e, "errno", None) == errno.EPROTOTYPE: - last_error = e - break - if ( - deadline is not None - and getattr(e, "errno", None) in UDS_TRANSIENT_CONNECT_ERRORS - ): - last_error = e - remaining_time = max(0, deadline - time.time()) - sleep_time = min(backoff, remaining_time) - if sleep_time > 0: - time.sleep(sleep_time) - backoff = min(backoff * 2, UDS_CONNECT_RETRY_MAX_BACKOFF) - continue - raise e - if getattr(last_error, "errno", None) == errno.EPROTOTYPE: - continue - raise last_error + sock = None + try: + sock = socket.socket(socket.AF_UNIX, socket_kind) + sock.settimeout(timeout) + cls._ensure_min_send_buffer_size(sock) + sock.connect(socket_path) + log.debug("Connected to socket %s with kind %s", socket_path, sk_name) + return sock + except Exception as e: + if sock is not None: + sock.close() + log.debug("Failed to connect to %s with kind %s: %s", socket_path, sk_name, e) + last_error = e + if getattr(e, "errno", None) == errno.EPROTOTYPE: + # Wrong socket kind for this address -- try the other one. + continue + raise e + # Only reachable if every candidate kind failed with EPROTOTYPE. raise last_error @classmethod @@ -1725,29 +1695,17 @@ def _xmit_packet_with_telemetry(self, packet, queue_mode=False): return sent - def _installed_socket(self, is_telemetry): - # type: (bool) -> Optional[_Socket] - """ - The socket a send for this packet would use, if one is already installed. - - Returns None when a fresh connection would have to be established, which - is the only situation where the socket_connect_timeout budget applies. - Callers that mean to gate on "would we have to connect?" must use this - rather than the deadline alone. - """ - if is_telemetry and self._dedicated_telemetry_destination(): - return self.telemetry_socket - return self.socket - def _xmit_packet(self, packet, is_telemetry, queue_mode=False): # type: (str, bool, bool) -> Optional[bool] - """Attempt to send packet, retrying a reconnect within this call as budget allows. + """Attempt to send a packet, once. Returns True if sent. Otherwise returns False for a definitive, non-retryable failure (already accounted for as a dropped packet), - or -- only when queue_mode is True -- None for a connection failure - that the sender queue should retry by requeuing the payload rather - than have accounted for here as a drop. + or -- only when queue_mode is True, the transport is UDS, and + socket_connect_retry is enabled -- None for a connection failure that + the sender queue should retry by requeuing the payload (with its own + backoff, capped at UDS_CONNECT_RETRY_MAX_BACKOFF) rather than have + accounted for here as a drop. """ if is_telemetry and self._dedicated_telemetry_destination(): @@ -1755,51 +1713,18 @@ def _xmit_packet(self, packet, is_telemetry, queue_mode=False): else: uses_uds = self.socket_path is not None - retry_deadline = None - if uses_uds and self.socket_connect_timeout and self.socket_connect_timeout > 0: - retry_deadline = time.time() + self.socket_connect_timeout - - backoff = UDS_CONNECT_RETRY_INITIAL_BACKOFF - sent = None # type: Optional[bool] - while True: - # Cheap fast-path check before even trying to acquire _socket_lock. - if ( - retry_deadline is not None - and retry_deadline - time.time() <= 0 - and not self._installed_socket(is_telemetry) - ): - log.warning( - "Gave up reconnecting after socket_connect_timeout (%ss), dropping the packet", - self.socket_connect_timeout, - ) - sent = None - break + # Direct/synchronous sends (queue_mode=False) always fail fast on a + # connection error: there is no queue expiry to protect a calling + # thread from retrying indefinitely, so socket_connect_retry does not + # apply to them. Reconnect-and-retry also stays UDS-only, matching + # UDP's different (connectionless) failure semantics. + retry_eligible = queue_mode and uses_uds and self.socket_connect_retry - sent = self._xmit_packet_attempt( - packet, is_telemetry, retry_eligible=retry_deadline is not None, retry_deadline=retry_deadline - ) - if sent: - return True - # `sent` is False for a definitive failure (already logged/dropped - # above), or None for a transient one that's worth reconnecting - # and retrying, bounded by socket_connect_timeout. - # - # A None result implies retry_eligible, which implies a deadline was - # set; the explicit check keeps that invariant locally provable - # (for readers and for the type checker) instead of implicit. - if sent is not None or retry_deadline is None: - break - remaining = retry_deadline - time.time() - if remaining <= 0: - log.warning( - "Gave up reconnecting after socket_connect_timeout (%ss), dropping the packet", - self.socket_connect_timeout, - ) - break - time.sleep(min(backoff, remaining)) - backoff = min(backoff * 2, UDS_CONNECT_RETRY_MAX_BACKOFF) + sent = self._xmit_packet_attempt(packet, is_telemetry, retry_eligible) + if sent: + return True - if sent is None and queue_mode: + if sent is None: # Connection trouble, and the caller is the background sender # queue: let it requeue the payload and retry once reconnected, # instead of dropping it here. @@ -1810,48 +1735,24 @@ def _xmit_packet(self, packet, is_telemetry, queue_mode=False): self.packets_dropped_writer += 1 return False - def _xmit_packet_attempt(self, packet, is_telemetry, retry_eligible, retry_deadline=None): - # type: (str, bool, bool, Optional[float]) -> Optional[bool] + def _xmit_packet_attempt(self, packet, is_telemetry, retry_eligible): + # type: (str, bool, bool) -> Optional[bool] """ Attempt to send a single packet. Returns True if the packet was sent, False if it should be dropped without retrying, or None if the failure is transient (the peer went - away), `retry_eligible` is set, and it's worth a reconnect-and-retry. - - :param retry_deadline: Optional absolute time.time()-based deadline - for the overall _xmit_packet retry operation. The connect_timeout - handed to get_socket() is computed from this only after - _socket_lock is actually acquired (not before), so time spent - waiting on a contended lock counts against the budget instead of - silently extending it. + away, which includes a failed reconnect), `retry_eligible` is set, + and it's worth a reconnect-and-retry by the caller. """ socket_kind = None with self._socket_lock: try: - # Captured under the lock, before the deadline check: whether a - # socket already exists decides whether that deadline is even - # relevant to this attempt. - existing_socket = self._installed_socket(is_telemetry) - - connect_timeout = self.socket_connect_timeout - if retry_deadline is not None: - connect_timeout = retry_deadline - time.time() - if connect_timeout <= 0 and not existing_socket: - log.warning( - "Gave up reconnecting after socket_connect_timeout (%ss), dropping the packet", - self.socket_connect_timeout, - ) - return False - if is_telemetry and self._dedicated_telemetry_destination(): - mysocket = existing_socket or self.get_socket( - telemetry=True, connect_timeout=connect_timeout - ) + mysocket = self.get_socket(telemetry=True) socket_kind = self._telemetry_socket_kind else: - # If set, use socket directly - mysocket = existing_socket or self.get_socket(connect_timeout=connect_timeout) + mysocket = self.get_socket() socket_kind = self._socket_kind encoded_packet = packet.encode(self.encoding) diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index a4bb3ec3d..c45bda142 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -30,7 +30,7 @@ # Datadog libraries from datadog import initialize, statsd from datadog import __version__ as version -from datadog.dogstatsd.base import DEFAULT_BUFFERING_FLUSH_INTERVAL, DEFAULT_HOST, DEFAULT_PORT, DogStatsd, MIN_SEND_BUFFER_SIZE, PENDING_PAYLOAD_EXPIRY_SECONDS, PendingPayload, SenderQueue, Stop, UDP_OPTIMAL_PAYLOAD_LENGTH, UDS_CONNECT_RETRY_INITIAL_BACKOFF, UDS_OPTIMAL_PAYLOAD_LENGTH +from datadog.dogstatsd.base import DEFAULT_BUFFERING_FLUSH_INTERVAL, DEFAULT_HOST, DEFAULT_PORT, DogStatsd, MIN_SEND_BUFFER_SIZE, PENDING_PAYLOAD_EXPIRY_SECONDS, PendingPayload, SenderQueue, Stop, UDP_OPTIMAL_PAYLOAD_LENGTH, UDS_CONNECT_RETRY_MAX_BACKOFF, UDS_OPTIMAL_PAYLOAD_LENGTH from datadog.util.compat import monotonic as sender_queue_clock from datadog.dogstatsd.context import TimedContextManagerDecorator from datadog.util.compat import is_higher_py35, is_p3k @@ -927,79 +927,50 @@ def test_socket_error(self): mock.ANY, ) - def _uds_statsd(self, connect_timeout): + def _uds_statsd(self, socket_connect_retry=False): """ A UDS-backed client whose current socket is already broken. The reconnect-and-retry path in _xmit_packet is deliberately scoped to UDS only, so these tests must not use the default UDP client. """ - statsd = DogStatsd(socket_path='/tmp/dogstatsd-test.sock', telemetry_min_flush_interval=0) - statsd.socket_connect_timeout = connect_timeout + statsd = DogStatsd( + socket_path='/tmp/dogstatsd-test.sock', + disable_telemetry=True, + socket_connect_retry=socket_connect_retry, + ) statsd.socket = BrokenSocket(error_number=errno.ECONNREFUSED) - statsd._reset_telemetry() return statsd @patch('datadog.dogstatsd.base.DogStatsd._get_uds_socket') - def test_socket_connection_error_reconnects_and_resends(self, mock_get_uds_socket): - working_socket = FakeSocket() - mock_get_uds_socket.return_value = working_socket - statsd = self._uds_statsd(connect_timeout=5) - - with mock.patch("datadog.dogstatsd.base.log") as mock_log: - statsd.gauge('reconnected', 1) - statsd.flush() - - mock_log.error.assert_not_called() - mock_log.warning.assert_not_called() - - # The packet was not dropped: it was resent once a fresh socket was obtained. - mock_get_uds_socket.assert_called_once() - self.assertEqual(statsd.packets_dropped_writer, 0) - self.assertEqual(working_socket.payloads[0].decode('utf-8'), 'reconnected:1|g\n') - - def test_socket_connection_error_drops_packet_if_reconnect_also_fails(self): - # Small deadline so the retry loop gives up quickly in the test. - statsd = self._uds_statsd(connect_timeout=0.05) - - with mock.patch.object( - DogStatsd, '_get_uds_socket', side_effect=socket.error(errno.ECONNREFUSED, "still refused") - ): + def test_direct_send_never_retries_uds_even_with_socket_connect_retry_enabled(self, mock_get_uds_socket): + # socket_connect_retry only affects the background sender: a direct/ + # synchronous send (background sender disabled, the default) has no + # queue and no expiry to protect the calling thread, so it must always + # fail fast on a connection error regardless of this setting. + mock_get_uds_socket.return_value = FakeSocket() + statsd = self._uds_statsd(socket_connect_retry=True) + + with mock.patch.object(statsd.socket, 'send', wraps=statsd.socket.send) as mock_send: with mock.patch("datadog.dogstatsd.base.log") as mock_log: - statsd.gauge('no error', 1) - statsd.flush() + statsd.gauge('not reconnected', 1) mock_log.error.assert_not_called() + mock_log.warning.assert_called_once_with( + "Error submitting packet: %s, dropping the packet and closing the socket", + mock.ANY, + ) - # Both the metric and the telemetry flush hit the same broken reconnect and get dropped - # once the retry deadline is exhausted. - self.assertEqual(statsd.packets_dropped_writer, 2) - - @patch('datadog.dogstatsd.base.DogStatsd._get_uds_socket') - def test_socket_connection_error_retries_multiple_times_before_success(self, mock_get_uds_socket): - working_socket = FakeSocket() - mock_get_uds_socket.side_effect = [ - socket.error(errno.ECONNREFUSED, "still refused"), - socket.error(errno.ECONNREFUSED, "still refused"), - working_socket, - ] - # Long enough to cover a few backoff sleeps well under a second. - statsd = self._uds_statsd(connect_timeout=5) - - with mock.patch("datadog.dogstatsd.base.log") as mock_log: - statsd.gauge('reconnected after retries', 1) - - mock_log.warning.assert_not_called() + # A single send attempt on the broken socket, then dropped -- no + # reconnect-and-resend, and no connect was ever attempted either. + mock_send.assert_called_once() - # Two failed reconnect attempts, then a third that finally succeeds. - self.assertEqual(mock_get_uds_socket.call_count, 3) - self.assertEqual(statsd.packets_dropped_writer, 0) - self.assertTrue(working_socket.payloads[0].decode('utf-8').startswith('reconnected after retries:1|g')) + mock_get_uds_socket.assert_not_called() def test_socket_connection_error_does_not_retry_for_udp(self): # Reconnect-and-retry is UDS-only: a UDP client drops the packet on a - # transient connection error even when socket_connect_timeout is set. - self.statsd.socket_connect_timeout = 5 + # transient connection error even with socket_connect_retry enabled. + self.statsd.socket_connect_retry = True broken_socket = BrokenSocket(error_number=errno.ECONNREFUSED) self.statsd.socket = broken_socket @@ -1016,67 +987,30 @@ def test_socket_connection_error_does_not_retry_for_udp(self): # No reconnect-and-resend: a single send attempt, then dropped. mock_send.assert_called_once() - @patch('datadog.dogstatsd.base.DogStatsd._get_uds_socket') - def test_expired_deadline_sends_on_socket_installed_by_another_thread(self, mock_get_uds_socket): - # socket_connect_timeout budgets *connecting*. If another thread already - # installed a healthy socket while this one waited for _socket_lock, the - # spent budget is irrelevant: sending on it does no connecting, so the - # packet must not be dropped at the moment of recovery. - statsd = self._uds_statsd(connect_timeout=5) - working_socket = FakeSocket() - statsd.socket = working_socket - - with mock.patch("datadog.dogstatsd.base.log") as mock_log: - sent = statsd._xmit_packet_attempt( - 'recovered:1|g\n', - is_telemetry=False, - retry_eligible=True, - retry_deadline=time.time() - 1, # budget consumed while waiting for the lock - ) - - self.assertTrue(sent) - mock_log.warning.assert_not_called() - - # The installed socket was used as-is; no connect was attempted. - mock_get_uds_socket.assert_not_called() - self.assertEqual(working_socket.payloads[0].decode('utf-8'), 'recovered:1|g\n') - - @patch('datadog.dogstatsd.base.DogStatsd._get_uds_socket') - def test_expired_deadline_drops_when_a_connect_would_be_needed(self, mock_get_uds_socket): - # The complement of the case above, and the reason the guard exists at - # all: with no socket installed, an expired budget must not reach - # get_socket(), which treats a <= 0 connect_timeout as "unbounded". - statsd = self._uds_statsd(connect_timeout=5) - statsd.socket = None - - with mock.patch("datadog.dogstatsd.base.log") as mock_log: - sent = statsd._xmit_packet_attempt( - 'dropped:1|g\n', - is_telemetry=False, - retry_eligible=True, - retry_deadline=time.time() - 1, - ) - - self.assertFalse(sent) - mock_log.warning.assert_called_once_with( - "Gave up reconnecting after socket_connect_timeout (%ss), dropping the packet", - 5, - ) - - mock_get_uds_socket.assert_not_called() + def test_socket_connect_retry_defaults_to_false(self): + self.assertFalse(self.statsd.socket_connect_retry) def test_concurrent_reconnect_does_not_drop_backlog_after_recovery(self): - # End-to-end ordering: thread A reconnects slowly while holding - # _socket_lock; B queues behind it and has its entire connect budget - # consumed by the wait. Once A installs a healthy socket, B must send on - # it rather than discard its packet. - statsd = self._uds_statsd(connect_timeout=0.2) + # End-to-end ordering: thread A connects slowly while holding + # _socket_lock (no socket installed yet); B queues right behind it on + # the same lock. Once A installs a healthy socket, B must send on it + # rather than also trying to connect -- get_socket() always prefers an + # already-installed socket over connecting again. Direct sends never + # retry now, so unlike the old version of this test, neither packet + # can survive a *failed* connect; this exercises the still-relevant + # part, concurrent first-time connects sharing one outcome. + statsd = DogStatsd( + socket_path='/tmp/dogstatsd-test-concurrent.sock', + disable_telemetry=True, + ) working_socket = FakeSocket() a_is_connecting = threading.Event() release_connect = threading.Event() + connect_calls = [] def slow_connect(*args, **kwargs): + connect_calls.append(1) a_is_connecting.set() release_connect.wait(5) return working_socket @@ -1089,14 +1023,17 @@ def slow_connect(*args, **kwargs): thread_b = Thread(target=lambda: statsd.gauge('from-b', 1)) thread_b.start() - # Let B's whole 0.2s budget elapse while it blocks on _socket_lock, - # then let A's connect succeed and install the socket. - time.sleep(0.5) + # Give B a moment to reach and block on _socket_lock (held by A's + # in-progress connect), then let A's connect succeed and install + # the socket. + time.sleep(0.2) release_connect.set() thread_a.join(5) thread_b.join(5) self.assertFalse(thread_a.is_alive()) + # B reused the socket A installed rather than connecting itself. + self.assertEqual(connect_calls, [1]) self.assertFalse(thread_b.is_alive()) payloads = [payload.decode('utf-8') for payload in working_socket.payloads] @@ -1135,25 +1072,6 @@ def send(self, payload): closer_thread.join(2) self.assertFalse(closer_thread.is_alive()) - def test_socket_connection_error_does_not_retry_without_connect_timeout(self): - # socket_connect_timeout defaults to 0 (unset): no reconnect attempt should be made - # for this packet, it should be dropped immediately instead. - self.assertEqual(self.statsd.socket_connect_timeout, 0) - broken_socket = BrokenSocket(error_number=errno.ECONNREFUSED) - self.statsd.socket = broken_socket - - with mock.patch.object(broken_socket, 'send', wraps=broken_socket.send) as mock_send: - with mock.patch("datadog.dogstatsd.base.log") as mock_log: - self.statsd.gauge('not reconnected', 1) - - mock_log.warning.assert_called_once_with( - "Error submitting packet: %s, dropping the packet and closing the socket", - mock.ANY, - ) - - # Only one send attempt was made on the broken socket: no reconnect-and-resend. - mock_send.assert_called_once() - def test_socket_overflown(self): self.statsd.socket = OverflownSocket() with mock.patch("datadog.dogstatsd.base.log") as mock_log: @@ -1210,82 +1128,46 @@ def test_uds_socket_ensures_min_receive_buffer(self, mock_socket_create): MIN_SEND_BUFFER_SIZE, ) - @patch('datadog.dogstatsd.base.time.sleep') - @patch('socket.socket') - def test_uds_socket_retries_missing_socket_until_timeout(self, mock_socket_create, mock_sleep): - missing_socket_error = socket.error(errno.ENOENT, os.strerror(errno.ENOENT)) - first_socket = Mock() - first_socket.connect.side_effect = missing_socket_error - first_socket.getsockopt.return_value = MIN_SEND_BUFFER_SIZE - second_socket = Mock() - second_socket.connect.return_value = None - second_socket.getsockopt.return_value = MIN_SEND_BUFFER_SIZE - mock_socket_create.side_effect = [first_socket, second_socket] - - datadog = DogStatsd( - socket_path="/fake/uds/socket/path", - socket_timeout=0.1, - socket_connect_timeout=1, - ) - datadog.gauge('some value', 1) - datadog.flush() - - self.assertEqual(mock_socket_create.call_count, 2) - first_socket.close.assert_called_once() - second_socket.close.assert_not_called() - second_socket.settimeout.assert_called_with(0.1) - mock_sleep.assert_any_call(UDS_CONNECT_RETRY_INITIAL_BACKOFF) - - @patch('datadog.dogstatsd.base.time.sleep') @patch('socket.socket') - def test_uds_socket_retries_refused_socket_until_timeout(self, mock_socket_create, mock_sleep): - refused_socket_error = socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED)) - first_socket = Mock() - first_socket.connect.side_effect = refused_socket_error - first_socket.getsockopt.return_value = MIN_SEND_BUFFER_SIZE - second_socket = Mock() - second_socket.connect.return_value = None - second_socket.getsockopt.return_value = MIN_SEND_BUFFER_SIZE - mock_socket_create.side_effect = [first_socket, second_socket] - - datadog = DogStatsd( - socket_path="unixstream:///fake/uds/socket/path", - socket_timeout=0.1, - socket_connect_timeout=1, - ) - datadog.gauge('some value', 1) - datadog.flush() - - self.assertEqual(mock_socket_create.call_count, 2) - first_socket.close.assert_called_once() - second_socket.close.assert_not_called() - second_socket.settimeout.assert_called_with(0.1) - mock_sleep.assert_any_call(UDS_CONNECT_RETRY_INITIAL_BACKOFF) - - @patch('datadog.dogstatsd.base.time.sleep') - @patch('datadog.dogstatsd.base.time.time', side_effect=[0, 0, 0, 0.5, 0.9, 1.1]) - @patch('socket.socket') - def test_uds_socket_never_sets_expired_deadline(self, mock_socket_create, mock_time, mock_sleep): + def test_uds_socket_missing_socket_fails_immediately_no_retry(self, mock_socket_create): + # _get_uds_socket makes exactly one connect attempt now: retrying a + # connect failure, if at all, is the caller's job (socket_connect_retry + # for the background sender; never for a direct send). missing_socket_error = socket.error(errno.ENOENT, os.strerror(errno.ENOENT)) mock_socket = mock_socket_create.return_value mock_socket.connect.side_effect = missing_socket_error mock_socket.getsockopt.return_value = MIN_SEND_BUFFER_SIZE with self.assertRaises(socket.error) as raised: - DogStatsd._get_uds_socket("unixgram:///fake/uds/socket/path", 0.1, 1) + DogStatsd._get_uds_socket("unixgram:///fake/uds/socket/path", 0.1) self.assertEqual(raised.exception.errno, errno.ENOENT) mock_socket_create.assert_called_once_with(socket.AF_UNIX, socket.SOCK_DGRAM) - mock_socket.settimeout.assert_called_once_with(1) + mock_socket.settimeout.assert_called_once_with(0.1) + mock_socket.close.assert_called_once() - @patch('datadog.dogstatsd.base.time.time', side_effect=[0, 0.9, 1.1]) @patch('socket.socket') - def test_uds_socket_raises_timeout_before_first_attempt(self, mock_socket_create, mock_time): - with self.assertRaises(socket.timeout) as raised: - DogStatsd._get_uds_socket("unixgram:///fake/uds/socket/path", 0.1, 1) - - self.assertEqual(str(raised.exception), "timed out connecting to UDS socket") - mock_socket_create.assert_not_called() + def test_uds_socket_falls_back_to_the_other_kind_on_eprototype(self, mock_socket_create): + # EPROTOTYPE means "wrong socket kind for this address", not a + # transient failure: _get_uds_socket falls back to the other + # candidate kind for it (still one connect attempt per kind), rather + # than retrying the same kind or giving up. + wrong_kind_socket = Mock() + wrong_kind_socket.connect.side_effect = socket.error(errno.EPROTOTYPE, os.strerror(errno.EPROTOTYPE)) + wrong_kind_socket.getsockopt.return_value = MIN_SEND_BUFFER_SIZE + right_kind_socket = Mock() + right_kind_socket.connect.return_value = None + right_kind_socket.getsockopt.return_value = MIN_SEND_BUFFER_SIZE + mock_socket_create.side_effect = [wrong_kind_socket, right_kind_socket] + + sock = DogStatsd._get_uds_socket("/fake/uds/socket/path", 0.1) + + self.assertIs(sock, right_kind_socket) + mock_socket_create.assert_has_calls([ + call(socket.AF_UNIX, socket.SOCK_DGRAM), + call(socket.AF_UNIX, socket.SOCK_STREAM), + ]) + wrong_kind_socket.close.assert_called_once() @patch('socket.socket') def test_udp_socket_ensures_min_receive_buffer(self, mock_socket_create): @@ -3649,13 +3531,14 @@ def capture_put(item): statsd.stop() def test_connection_failure_requeues_and_resends_once_reconnected(self): - # A UDS client whose socket is broken, with a small connect budget so - # the internal reconnect-and-retry loop inside _xmit_packet gives up - # quickly and hands off to the sender queue's own retry-by-requeuing. + # A UDS client whose socket is broken. socket_connect_retry=True is + # what makes queue-mode sends hand off a connection failure to the + # sender queue's own retry-by-requeuing instead of dropping the + # payload on the first attempt -- direct sends never do this. working_socket = FakeSocket() attempts = {"count": 0} - def flaky_get_uds_socket(cls, socket_path, timeout, connect_timeout): + def flaky_get_uds_socket(cls, socket_path, timeout): attempts["count"] += 1 if attempts["count"] < 4: raise socket.error(errno.ECONNREFUSED, "still refused") @@ -3666,8 +3549,8 @@ def flaky_get_uds_socket(cls, socket_path, timeout, connect_timeout): socket_path="/tmp/dogstatsd-test-requeue.sock", disable_telemetry=True, disable_background_sender=False, + socket_connect_retry=True, ) - statsd.socket_connect_timeout = 0.05 statsd.gauge("eventually.sent", 1) statsd.wait_for_pending() @@ -3682,6 +3565,33 @@ def flaky_get_uds_socket(cls, socket_path, timeout, connect_timeout): statsd.stop() + def test_queue_mode_drops_immediately_by_default(self): + # socket_connect_retry defaults to False for the background sender + # too: without it, a connection failure on a queued payload is + # accounted as a writer drop on the very first attempt, exactly like + # a direct send, rather than requeued and retried. + statsd = DogStatsd( + socket_path="/tmp/dogstatsd-test-no-retry-queue.sock", + disable_background_sender=False, + ) + self.assertFalse(statsd.socket_connect_retry) + statsd.socket = BrokenSocket(error_number=errno.ECONNREFUSED) + + statsd.gauge("dropped.immediately", 1) + statsd.wait_for_pending() + + self.assertGreaterEqual(statsd.packets_dropped_writer, 1) + self.assertEqual(statsd.packets_dropped_expired, 0) + statsd.stop() + + def test_queue_mode_retry_backoff_caps_at_one_minute(self): + # "Backoff...to a maximum of once per minute": the doubling algorithm + # in _sender_main_loop is unchanged (already exercised end-to-end by + # test_connection_failure_requeues_and_resends_once_reconnected, + # which drives it through several real doublings below the cap); what + # changed is this constant, from the old 1-second cap to 60. + self.assertEqual(UDS_CONNECT_RETRY_MAX_BACKOFF, 60.0) + def test_set_socket_timeout(self): statsd = DogStatsd(disable_background_sender=False) statsd.socket = FakeSocket() From 88c46721ac3ea1e06eff351a64b725e3ebcc935c Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 15 Sep 2026 13:39:41 +0100 Subject: [PATCH 13/30] Allow backoff to be interrupted --- datadog/dogstatsd/base.py | 101 ++++++++++++++++++++-------- tests/unit/dogstatsd/test_statsd.py | 98 ++++++++++++++++++++++++++- 2 files changed, 170 insertions(+), 29 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index af74fac45..18d7136f9 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -182,10 +182,12 @@ def reverse(self): # Socket options MIN_SEND_BUFFER_SIZE = 32 * 1024 # Backoff for the background sender's own retry-by-requeuing loop (see -# _sender_main_loop). Not used for direct/synchronous sends, which never -# retry a connection failure regardless of socket_connect_retry. -UDS_CONNECT_RETRY_INITIAL_BACKOFF = 0.025 -UDS_CONNECT_RETRY_MAX_BACKOFF = 60.0 +# _sender_main_loop). Covers every retryable failure it sees, connect and +# mid-send alike -- not just connects, hence the transport-neutral name. Not +# used for direct/synchronous sends, which never retry a connection failure +# regardless of socket_connect_retry. +SENDER_RETRY_INITIAL_BACKOFF = 0.025 +SENDER_RETRY_MAX_BACKOFF = 60.0 # Errors seen while sending on an already-connected socket that indicate the # peer went away (e.g. the agent crashed/restarted). These are worth a single # reconnect-and-resend attempt instead of dropping the packet outright. @@ -477,10 +479,10 @@ def __init__( :param socket_connect_retry: Only affects the background sender (disable_background_sender=False). If True, a connection failure while sending a queued payload to a UNIX socket is retried indefinitely, backing off up to once a minute between attempts, instead of dropping the payload - immediately. This is safe to enable because a payload stuck retrying is still subject to the - sender queue's own expiry: it is eventually dropped as stale rather than retried forever if the - Agent never comes back. Direct/synchronous sends (the default mode) always fail fast on a - connection error and are unaffected by this setting. + immediately. Ordinary payloads stuck retrying are eventually dropped as stale by the sender + queue's own expiry stop() and pre_fork() interrupt the retrying rather than waiting for it. + Direct/synchronous sends (the default mode) always fail fast on a connection error and are + unaffected by this setting. Default: False (fail fast, matching the previous socket_connect_timeout=0 default). :type socket_connect_retry: bool @@ -640,6 +642,9 @@ def __init__( self._queue = None # type: Optional[SenderQueue] self._sender_thread = None # type: Optional[threading.Thread] + # Set to ask a running sender thread to stop. Also what makes its + # retry backoff interruptible -- see _sender_main_loop. + self._sender_stopping = threading.Event() self._sender_enabled = False if not disable_background_sender: @@ -883,6 +888,15 @@ def _dedicated_telemetry_destination(self): # type: () -> bool return bool(self.telemetry_socket_path or self.telemetry_host) + def _uses_dedicated_telemetry(self, is_telemetry): + # type: (bool) -> bool + """True when this packet goes to a separate telemetry destination. + + Decides which socket/socket_path pair applies to a packet, so the + send path and its transport check can't disagree about it. + """ + return is_telemetry and self._dedicated_telemetry_destination() + # Context manager helper def __enter__(self): # type: () -> DogStatsd @@ -1043,8 +1057,8 @@ def _get_uds_socket(cls, socket_path, timeout): elif socket_path.startswith(UNIX_ADDRESS_SCHEME): socket_path = socket_path[len(UNIX_ADDRESS_SCHEME):] - last_error = socket.timeout("timed out connecting to UDS socket") # type: Exception - for socket_kind in valid_socket_kinds: + for index, socket_kind in enumerate(valid_socket_kinds): + is_last_kind = index == len(valid_socket_kinds) - 1 # py2 stores socket kinds differently than py3, determine the name independently from version sk_name = {socket.SOCK_STREAM: "stream", socket.SOCK_DGRAM: "datagram"}[socket_kind] sock = None @@ -1059,13 +1073,14 @@ def _get_uds_socket(cls, socket_path, timeout): if sock is not None: sock.close() log.debug("Failed to connect to %s with kind %s: %s", socket_path, sk_name, e) - last_error = e - if getattr(e, "errno", None) == errno.EPROTOTYPE: + if getattr(e, "errno", None) == errno.EPROTOTYPE and not is_last_kind: # Wrong socket kind for this address -- try the other one. continue - raise e - # Only reachable if every candidate kind failed with EPROTOTYPE. - raise last_error + raise + # Unreachable: valid_socket_kinds is never empty, and every path above + # either returns or raises. Present only so this always has a return + # type of _Socket. + raise socket.error("no usable socket kind for {}".format(socket_path)) @classmethod def _get_udp_socket(cls, host, port, timeout): @@ -1704,11 +1719,11 @@ def _xmit_packet(self, packet, is_telemetry, queue_mode=False): or -- only when queue_mode is True, the transport is UDS, and socket_connect_retry is enabled -- None for a connection failure that the sender queue should retry by requeuing the payload (with its own - backoff, capped at UDS_CONNECT_RETRY_MAX_BACKOFF) rather than have + backoff, capped at SENDER_RETRY_MAX_BACKOFF) rather than have accounted for here as a drop. """ - if is_telemetry and self._dedicated_telemetry_destination(): + if self._uses_dedicated_telemetry(is_telemetry): uses_uds = self.telemetry_socket_path is not None else: uses_uds = self.socket_path is not None @@ -1748,7 +1763,7 @@ def _xmit_packet_attempt(self, packet, is_telemetry, retry_eligible): socket_kind = None with self._socket_lock: try: - if is_telemetry and self._dedicated_telemetry_destination(): + if self._uses_dedicated_telemetry(is_telemetry): mysocket = self.get_socket(telemetry=True) socket_kind = self._telemetry_socket_kind else: @@ -2044,6 +2059,10 @@ def _start_sender_thread(self): if self._queue is not None: return + # A previous _stop_sender_thread() leaves this set; clear it before the + # new sender starts so it doesn't immediately think it's shutting down. + self._sender_stopping.clear() + self._queue = SenderQueue( self._sender_queue_size, PENDING_PAYLOAD_EXPIRY_SECONDS, @@ -2063,6 +2082,13 @@ def _start_sender_thread(self): def _stop_sender_thread(self, timeout=None): # type: (Optional[float]) -> bool + # Ask the sender to stop before anything else: this is what breaks it + # out of a retry backoff (which can be as long as + # SENDER_RETRY_MAX_BACKOFF) instead of having to wait that out, and + # what lets it give up on a payload that would otherwise starve the + # Stop sentinel forever (see _sender_main_loop). + self._sender_stopping.set() + # Lock ensures that nothing gets added to the queue after we disable it. with self._buffer_lock: if self._queue is not None: @@ -2095,18 +2121,30 @@ def _stop_sender_thread(self, timeout=None): self._sender_thread = None return True + def _release_sender_state(self, pending_queue): + # type: (SenderQueue) -> None + """Drop the client's pointers to this sender's queue and thread. + + Called by the sender thread on its way out so the client self-heals + and a later enable_background_sender() can start fresh -- even when + this exit wasn't awaited, e.g. a stop(timeout) that timed out and the + caller moved on. The guards check this thread still owns the fields: a + newer sender may have already taken over. + """ + with self._buffer_lock: + if self._queue is pending_queue: + self._queue = None + if self._sender_thread is threading.current_thread(): + self._sender_thread = None + def _sender_main_loop(self, pending_queue): # type: (SenderQueue) -> None - backoff = UDS_CONNECT_RETRY_INITIAL_BACKOFF + backoff = SENDER_RETRY_INITIAL_BACKOFF while True: item = pending_queue.get() if item is Stop: pending_queue.task_done() - with self._buffer_lock: - if self._queue is pending_queue: - self._queue = None - if self._sender_thread is threading.current_thread(): - self._sender_thread = None + self._release_sender_state(pending_queue) return # next line has type ignore because the type checker cannot @@ -2122,15 +2160,24 @@ def _sender_main_loop(self, pending_queue): # future get()) is what eventually gives up on a payload # that's been stuck for too long, unless it's replay-safe. pending_queue.requeue_front(item) # type: ignore[arg-type] - time.sleep(backoff) - backoff = min(backoff * 2, UDS_CONNECT_RETRY_MAX_BACKOFF) + + # Interruptible backoff. + self._sender_stopping.wait(backoff) + if self._sender_stopping.is_set(): + # Checked rather than using wait()'s return value, which + # is only meaningful on Python 2.7+ -- and this module + # still supports 2.7, where several other wait() APIs + # return None. + self._release_sender_state(pending_queue) + return + backoff = min(backoff * 2, SENDER_RETRY_MAX_BACKOFF) continue # Sent, or a definitive failure that _xmit_packet already # accounted for as a dropped packet -- either way, this # payload's story is over. pending_queue.task_done() - backoff = UDS_CONNECT_RETRY_INITIAL_BACKOFF + backoff = SENDER_RETRY_INITIAL_BACKOFF def wait_for_pending(self, timeout=None): # type: (Optional[float]) -> bool diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index c45bda142..e51e66017 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -30,7 +30,7 @@ # Datadog libraries from datadog import initialize, statsd from datadog import __version__ as version -from datadog.dogstatsd.base import DEFAULT_BUFFERING_FLUSH_INTERVAL, DEFAULT_HOST, DEFAULT_PORT, DogStatsd, MIN_SEND_BUFFER_SIZE, PENDING_PAYLOAD_EXPIRY_SECONDS, PendingPayload, SenderQueue, Stop, UDP_OPTIMAL_PAYLOAD_LENGTH, UDS_CONNECT_RETRY_MAX_BACKOFF, UDS_OPTIMAL_PAYLOAD_LENGTH +from datadog.dogstatsd.base import DEFAULT_BUFFERING_FLUSH_INTERVAL, DEFAULT_HOST, DEFAULT_PORT, DogStatsd, MIN_SEND_BUFFER_SIZE, PENDING_PAYLOAD_EXPIRY_SECONDS, PendingPayload, SenderQueue, Stop, UDP_OPTIMAL_PAYLOAD_LENGTH, SENDER_RETRY_MAX_BACKOFF, UDS_OPTIMAL_PAYLOAD_LENGTH from datadog.util.compat import monotonic as sender_queue_clock from datadog.dogstatsd.context import TimedContextManagerDecorator from datadog.util.compat import is_higher_py35, is_p3k @@ -3590,7 +3590,101 @@ def test_queue_mode_retry_backoff_caps_at_one_minute(self): # test_connection_failure_requeues_and_resends_once_reconnected, # which drives it through several real doublings below the cap); what # changed is this constant, from the old 1-second cap to 60. - self.assertEqual(UDS_CONNECT_RETRY_MAX_BACKOFF, 60.0) + self.assertEqual(SENDER_RETRY_MAX_BACKOFF, 60.0) + + def _unreachable_retrying_client(self): + """Background-sender client whose UDS path will never exist. + + socket_connect_retry=True, so every queued payload fails to connect + and gets requeued at the FRONT of the queue -- ahead of any Stop + sentinel. + """ + sock_path = os.path.join(tempfile.mkdtemp(), "never-created.sock") + return DogStatsd( + socket_path="unix://" + sock_path, + socket_connect_retry=True, + disable_background_sender=False, + disable_telemetry=True, + # Keep these out of the module-level _instances WeakSet: the + # pre_fork test below abandons its instance with _config_lock + # still held, and the global os.register_at_fork hooks iterate + # _instances -- a later real fork() would deadlock on it. + track_instance=False, + ) + + def test_stop_is_bounded_with_a_stuck_replay_safe_payload(self): + # requeue_front() puts a failed payload back ahead of the Stop + # sentinel, and replay-safe payloads are exempt from the queue's + # expiry, so such a payload never resolves while the Agent is down. + # Without an interruptible shutdown signal the sender never reaches + # Stop at all and stop() hangs forever. + statsd = self._unreachable_retrying_client() + statsd.gauge_with_timestamp("replay.safe", 1, timestamp=int(time.time())) + time.sleep(0.1) # let the sender pick it up and start retrying + + t0 = time.time() + self.assertIs(self._call_bounded(statsd.stop, ()), True) + self.assertLess(time.time() - t0, 5.0, "stop() did not interrupt the retry loop") + self.assertIsNone(statsd._queue) + + def test_pre_fork_is_bounded_with_a_stuck_replay_safe_payload(self): + # Same starvation, reached through pre_fork() -- which matters more: + # it is installed as an os.register_at_fork hook, has no timeout + # parameter, and would therefore block any fork() in the process. + statsd = self._unreachable_retrying_client() + statsd.gauge_with_timestamp("replay.safe", 1, timestamp=int(time.time())) + time.sleep(0.1) + + # Run it on a worker so a regression fails this assertion instead of + # hanging the suite. That means _config_lock -- which pre_fork() + # deliberately acquires and leaves held for post_fork_parent() to + # release -- ends up owned by a thread that then exits, so this + # instance is deliberately abandoned rather than restored. It is + # constructed with track_instance=False precisely so nothing else can + # ever try to take that lock again. + t0 = time.time() + self._call_bounded(statsd.pre_fork, ()) + self.assertLess(time.time() - t0, 5.0, "pre_fork() would have blocked os.fork()") + self.assertIsNone(statsd._sender_thread, "pre_fork() should have stopped the sender") + + def test_stop_interrupts_a_long_retry_backoff_instead_of_waiting_it_out(self): + # The backoff cap is a full minute. A plain time.sleep() would make + # stop() wait out however much of it is left; the shutdown signal must + # cut it short. + statsd = self._unreachable_retrying_client() + with patch("datadog.dogstatsd.base.SENDER_RETRY_INITIAL_BACKOFF", 30.0): + statsd.gauge("ordinary", 1) + time.sleep(0.3) # fail once, then settle into the 30s backoff + + t0 = time.time() + self.assertIs(self._call_bounded(statsd.stop, ()), True) + self.assertLess(time.time() - t0, 5.0, "stop() waited out the backoff sleep") + + def test_sender_can_restart_after_a_stop_cleared_the_stopping_signal(self): + # The shutdown signal is sticky, so it has to be cleared when a new + # sender starts or the replacement would exit immediately. + statsd = self._unreachable_retrying_client() + statsd.gauge("ordinary", 1) + time.sleep(0.1) + self.assertIs(self._call_bounded(statsd.stop, ()), True) + + statsd.enable_background_sender() + try: + self.assertIsNotNone(statsd._queue) + fresh = statsd._sender_thread + self.assertIsNotNone(fresh) + + # The stale signal only bites once the sender reaches its retry + # branch, so it has to actually fail a send here: an idle sender + # just blocks in get() and would mask the bug. With the signal + # left set, this first failure looks like a shutdown request and + # the fresh sender exits silently on it. + statsd.gauge("ordinary", 1) + time.sleep(0.3) + self.assertTrue(fresh.is_alive(), "fresh sender exited on a stale stopping signal") + self.assertIs(statsd._sender_thread, fresh) + finally: + statsd.stop(5.0) def test_set_socket_timeout(self): statsd = DogStatsd(disable_background_sender=False) From d02b09ee1f93a7f6b3659d4c50703740e9974b1f Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Wed, 16 Sep 2026 15:33:16 +0100 Subject: [PATCH 14/30] Add raw payload to queue for replay safe metrics --- datadog/dogstatsd/base.py | 34 ++++--- datadog/dogstatsd/sender_queue.py | 74 +++++++++----- .../test_sender_queue_benchmark.py | 68 +++++++------ tests/unit/dogstatsd/test_statsd.py | 99 +++++++++++-------- 4 files changed, 170 insertions(+), 105 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index 2580b6dcc..2b8d47e5e 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -50,7 +50,11 @@ PendingPayload, Stop, PENDING_PAYLOAD_EXPIRY_SECONDS, + payload_text, ) + +if sys.version_info[:2] >= (3, 5): + from datadog.dogstatsd.sender_queue import QueuedItem # noqa: F401 from datadog.util.compat import monotonic, text, urlparse from datadog.util.format import normalize_tags, validate_cardinality from datadog.version import __version__ @@ -1625,16 +1629,16 @@ def bytes_dropped(self): return self.bytes_dropped_queue + self.bytes_dropped_writer + self.bytes_dropped_expired def _account_dropped_queue_full(self, item): - # type: (PendingPayload) -> None + # type: (QueuedItem) -> None """A payload was evicted from the sender queue to make room for a new one.""" self.packets_dropped_queue += 1 - self.bytes_dropped_queue += len(item.payload.encode(self.encoding)) + self.bytes_dropped_queue += len(payload_text(item).encode(self.encoding)) def _account_dropped_expired(self, item): - # type: (PendingPayload) -> None + # type: (QueuedItem) -> None """A payload sat in the sender queue longer than PENDING_PAYLOAD_EXPIRY_SECONDS.""" self.packets_dropped_expired += 1 - self.bytes_dropped_expired += len(item.payload.encode(self.encoding)) + self.bytes_dropped_expired += len(payload_text(item).encode(self.encoding)) def _flush_telemetry(self): # type: () -> str @@ -1684,11 +1688,15 @@ def _send_to_server(self, packet, replay_safe=False): with self._buffer_lock: packet_with_newline = packet + '\n' if self._queue is not None: - # replay_safe payloads never have their enqueued_at read - # (see SenderQueue._expired()'s short-circuit), so skip - # both the clock read and the float allocation for them. - enqueued_at = None if replay_safe else monotonic() - self._queue.put(PendingPayload(packet_with_newline, enqueued_at, replay_safe)) + if replay_safe: + # Never expires, so it needs no enqueued_at and no + # wrapper at all: queue the bare string and let the + # queue infer replay-safety from the type. Saves the + # PendingPayload object (~56 bytes) per entry and + # keeps these out of the cyclic GC's traversal set. + self._queue.put(packet_with_newline) + else: + self._queue.put(PendingPayload(packet_with_newline, monotonic())) return self._xmit_packet_with_telemetry(packet + '\n') @@ -2176,11 +2184,11 @@ def _sender_main_loop(self, pending_queue): pending_queue.task_done() return - # next line has type ignore because the type checker cannot - # know that 'if item is Stop' is the only case where item is - # of object type. + # payload_text() also narrows the type: 'if item is Stop' above is + # the only case where item is the bare object sentinel, which the + # type checker can't know on its own. sent = self._xmit_packet_with_telemetry( - item.payload, queue_mode=True # type: ignore[attr-defined] + payload_text(item), queue_mode=True # type: ignore[arg-type] ) if sent is None: diff --git a/datadog/dogstatsd/sender_queue.py b/datadog/dogstatsd/sender_queue.py index c419ef4c5..debc858b2 100644 --- a/datadog/dogstatsd/sender_queue.py +++ b/datadog/dogstatsd/sender_queue.py @@ -11,38 +11,66 @@ # Sentinel telling the background sender thread to shut down. Stop = object() +# What the queue can hold. A payload is either a bare string (replay-safe, no +# expiry state needed) or a PendingPayload (subject to expiry); Stop is the +# only other thing that ever goes in, and is matched by identity. +if sys.version_info[:2] >= (3, 5): + QueuedItem = Union[str, "PendingPayload"] # noqa: F401 + QueuedItemOrStop = Union[str, "PendingPayload", object] # noqa: F401 + # How long (in seconds) a non-replay-safe payload may sit in the background # sender queue before it's considered stale and dropped instead of sent. -# Payloads that carry their own explicit timestamp (replay_safe) are exempt: +# Payloads that carry their own explicit timestamp (replay-safe) are exempt: # delivering those late doesn't change what they mean, so they're kept # around until they can actually be sent. PENDING_PAYLOAD_EXPIRY_SECONDS = 10.0 class PendingPayload(object): - """A single packet queued for the background sender. + """A packet queued for the background sender that can go stale. + + Only payloads subject to expiry are wrapped in this. A replay-safe + payload -- one carrying its own explicit timestamp, so that delivering it + late doesn't change what it means -- is queued as the bare packet string + instead, because it needs none of the state here. The queue therefore + reads replay-safety off the entry's *type* rather than a stored flag (see + SenderQueue._expired() and is_replay_safe()), which keeps ~56 bytes per + replay-safe entry out of the queue and keeps those entries out of the + cyclic GC's traversal set entirely, since str holds no references. :ivar payload: The already-serialized packet text (including its trailing newline), ready to be written to the socket. :ivar enqueued_at: A monotonic timestamp recorded when the payload became eligible for sending (i.e. when it was put on the queue). Used to decide whether it has been sitting in the queue for too - long to still be worth sending. None when replay_safe is True: it's - never read in that case (see SenderQueue._expired()'s short-circuit), - so skipping the allocation costs nothing. - :ivar replay_safe: True when delayed delivery preserves the payload's - meaning because it carries its own explicit timestamp. Such - payloads are never dropped for being stale, and never need - enqueued_at. + long to still be worth sending. """ - __slots__ = ("payload", "enqueued_at", "replay_safe") + __slots__ = ("payload", "enqueued_at") - def __init__(self, payload, enqueued_at, replay_safe): - # type: (str, Optional[float], bool) -> None + def __init__(self, payload, enqueued_at): + # type: (str, float) -> None self.payload = payload self.enqueued_at = enqueued_at - self.replay_safe = replay_safe + + +def is_replay_safe(item): + # type: (Union[str, PendingPayload]) -> bool + """True when this queue entry is exempt from expiry. + + Replay-safe entries are queued as bare strings; everything subject to + expiry is wrapped in PendingPayload. Centralised here so the type test + isn't repeated at every site that cares. + """ + return not isinstance(item, PendingPayload) + + +def payload_text(item): + # type: (Union[str, PendingPayload]) -> str + """The serialized packet text of a queue entry, whichever form it took.""" + if isinstance(item, PendingPayload): + return item.payload + return item class SenderQueue(object): @@ -74,7 +102,7 @@ class SenderQueue(object): """ def __init__(self, maxsize, expiry_seconds, on_drop_queue_full, on_drop_expired, put_timeout=0): - # type: (int, float, Callable[[PendingPayload], None], Callable[[PendingPayload], None], Optional[float]) -> None # noqa: E501 + # type: (int, float, Callable[[QueuedItem], None], Callable[[QueuedItem], None], Optional[float]) -> None # noqa: E501 self._maxsize = maxsize self._expiry_seconds = expiry_seconds self._on_drop_queue_full = on_drop_queue_full @@ -92,14 +120,12 @@ def __init__(self, maxsize, expiry_seconds, on_drop_queue_full, on_drop_expired, self._unfinished_tasks = 0 def _expired(self, item, now): - # type: (PendingPayload, float) -> bool - if item.replay_safe: + # type: (QueuedItem, float) -> bool + if not isinstance(item, PendingPayload): + # A bare string is a replay-safe payload: it carries its own + # timestamp, so it never goes stale (see PendingPayload). return False - # enqueued_at is only ever None for replay_safe items (see - # PendingPayload), which are already excluded above -- it's a plain - # float here. mypy can't correlate that invariant across the two - # attributes, hence the ignore. - return (now - item.enqueued_at) > self._expiry_seconds # type: ignore[operator] + return (now - item.enqueued_at) > self._expiry_seconds def _make_room_locked(self): # type: () -> None @@ -135,7 +161,7 @@ def _make_room_locked(self): self._not_full.notify(reclaimed) def put(self, item): - # type: (Union[PendingPayload, object]) -> None + # type: (QueuedItemOrStop) -> None """Queue a payload (or the Stop sentinel). If the queue is full: waits for room according to put_timeout -- @@ -170,7 +196,7 @@ def put(self, item): self._not_empty.notify() def requeue_front(self, item): - # type: (PendingPayload) -> None + # type: (QueuedItem) -> None """Put an in-flight payload back at the front after a failed send attempt. The payload was already accounted for by the put() that originally @@ -197,7 +223,7 @@ def requeue_front(self, item): self._not_empty.notify() def get(self): - # type: () -> Union[PendingPayload, object] + # type: () -> QueuedItemOrStop """Block for the next payload, silently dropping expired entries along the way.""" while True: with self._not_empty: diff --git a/tests/performance/test_sender_queue_benchmark.py b/tests/performance/test_sender_queue_benchmark.py index c77b6c99c..b24561781 100644 --- a/tests/performance/test_sender_queue_benchmark.py +++ b/tests/performance/test_sender_queue_benchmark.py @@ -171,7 +171,7 @@ def scenario_1_unbounded_single_threaded(): old.task_done() new = make_sender_queue(maxsize=0) - total, samples = time_puts(lambda: new.put(PendingPayload(PACKET, monotonic(), False)), n) + total, samples = time_puts(lambda: new.put(PendingPayload(PACKET, monotonic())), n) new_p = report("SenderQueue", n, total, samples) for _ in range(n): new.get() @@ -197,8 +197,8 @@ def scenario_2_sustained_overflow(): new = make_sender_queue(maxsize=maxsize) for _ in range(maxsize): - new.put(PendingPayload(PACKET, monotonic(), False)) - total, samples = time_puts(lambda: new.put(PendingPayload(PACKET, monotonic(), False)), n) + new.put(PendingPayload(PACKET, monotonic())) + total, samples = time_puts(lambda: new.put(PendingPayload(PACKET, monotonic())), n) new_p = report("SenderQueue", n, total, samples) ratio = new_p["p99"] / old_p["p99"] if old_p["p99"] else float("inf") @@ -223,10 +223,10 @@ def scenario_3_large_expired_backlog(): q = make_sender_queue(maxsize=backlog, expiry_seconds=0.0) stale_at = monotonic() - 1000.0 for _ in range(backlog): - q.put(PendingPayload(PACKET, stale_at, False)) + q.put(PendingPayload(PACKET, stale_at)) t0 = time.perf_counter() - q.put(PendingPayload(PACKET, monotonic(), False)) + q.put(PendingPayload(PACKET, monotonic())) elapsed_us = (time.perf_counter() - t0) * 1e6 note("backlog={:>5d} stale entries -> single put() took {:>9.3f}us, evicted {:d}".format( @@ -315,7 +315,7 @@ def scenario_4_concurrency(): new = make_sender_queue(maxsize=1000) elapsed, total_ops, samples = _run_concurrent( - lambda: new.put(PendingPayload(PACKET, monotonic(), False)), + lambda: new.put(PendingPayload(PACKET, monotonic())), lambda: (new.get(), new.task_done()), n_producers, n_per_producer, @@ -333,42 +333,52 @@ def scenario_4_concurrency(): # Scenario 5: per-item memory footprint. # -------------------------------------------------------------------------- def scenario_5_memory_footprint(): - section("5. Per-item memory footprint: PendingPayload wrapper vs a bare str") + section("5. Per-item memory footprint: bare str (replay-safe) vs PendingPayload (expiring)") note("sys.getsizeof() is shallow: PendingPayload holds a *reference* to the payload string,") - note("not a copy, so its own size doesn't include the string's bytes. The old queue.Queue held") - note("that same string directly with nothing wrapping it, so the real extra cost per item is") - note("the wrapper object itself, plus (for non-replay-safe items) a float object for enqueued_at.") + note("not a copy, so its own size doesn't include the string's bytes. The string is shared") + note("either way, so the real per-item cost is the wrapper object plus its enqueued_at float.") payload_str = PACKET - wrapper_size = sys.getsizeof(PendingPayload(payload_str, monotonic(), False)) + str_size = sys.getsizeof(payload_str) + wrapper_size = sys.getsizeof(PendingPayload(payload_str, monotonic())) + float_size = sys.getsizeof(monotonic()) - note("payload str (shared either way): {} bytes".format(sys.getsizeof(payload_str))) + note("payload str (shared either way): {} bytes".format(str_size)) note("PendingPayload wrapper itself (__slots__, no __dict__): {} bytes".format(wrapper_size)) print() - note("replay_safe=True payloads (gauge_with_timestamp, etc.) never have their enqueued_at read") - note("(SenderQueue._expired() short-circuits on replay_safe first), so base.py passes None") - note("instead of a fresh timestamp -- no float allocation at all for this class of payload:") - replay_safe_wrapped = PendingPayload(payload_str, None, True) - note(" PendingPayload(..., enqueued_at=None, replay_safe=True): {} bytes total, +0 for the timestamp".format( - sys.getsizeof(replay_safe_wrapped) + note("Replay-safe payloads (gauge_with_timestamp, event(date_happened=...),") + note("service_check(timestamp=...)) never expire, so they carry no enqueued_at and are queued") + note("as the BARE STRING -- SenderQueue infers replay-safety from the entry's type. That means") + note("zero wrapper overhead for them, not merely a skipped float:") + note(" replay-safe entry: {} bytes (just the shared str) -> +0 bytes overhead".format(str_size)) + note(" expiring entry: {} + {} + {} = {} bytes -> +{} bytes overhead".format( + str_size, wrapper_size, float_size, + str_size + wrapper_size + float_size, wrapper_size + float_size, )) print() - non_replay_safe_extra = wrapper_size + sys.getsizeof(monotonic()) - note("Non-replay-safe payloads DO need a real enqueued_at -- one monotonic() reading per item,") - note("same as any other Python object holding a fresh timestamp. Extra overhead per item vs the") - note("old bare-string queue: ~{} bytes ({} wrapper + {} float).".format( - non_replay_safe_extra, wrapper_size, sys.getsizeof(monotonic()) + saved_per_entry = wrapper_size + note("Saving vs wrapping replay-safe payloads too (the previous design): {} bytes/entry,".format( + saved_per_entry + )) + note("which is {:.0f}% of what such an entry used to occupy.".format( + 100.0 * saved_per_entry / (str_size + saved_per_entry) )) for n in (100, 10000, 100000): - note(" at sender_queue_size={:<7d} that's ~{:.1f}KB of additional resident overhead".format( - n, non_replay_safe_extra * n / 1024.0 + note(" at sender_queue_size={:<7d} that's ~{:.1f}KB less resident memory".format( + n, saved_per_entry * n / 1024.0 )) - note("(An earlier version of this code coalesced timestamps to a shared per-100ms-bucket float") - note("to cut this under bursty load -- best case ~234KB saved at sender_queue_size=10,000, i.e.") - note("~0.09% of a typical 256MB container's RSS. Reverted: not worth the added global mutable") - note("state, cross-instance coupling, and dedicated concurrency tests for savings that small.)") + print() + + note("The second effect is GC pressure, and it is the bigger one in practice: PendingPayload") + note("holds references, so every instance is tracked by the cyclic collector and traversed on") + note("each gen-2 pass. str is atomic and never traversed. A queue full of replay-safe payloads") + note("therefore contributes nothing to GC pause time now -- see scenario 1's max latency, which") + note("was dominated by exactly this traversal when every entry was wrapped.") + note("(An earlier version coalesced timestamps into a shared per-100ms-bucket float to cut the") + note("expiring-entry cost too. Reverted: global mutable state and cross-instance coupling for") + note("~234KB at sender_queue_size=10,000, i.e. ~0.09% of a 256MB container's RSS.)") # -------------------------------------------------------------------------- diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index 80a5d39fe..3a8a76f9c 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -31,6 +31,7 @@ from datadog import initialize, statsd from datadog import __version__ as version from datadog.dogstatsd.base import DEFAULT_BUFFERING_FLUSH_INTERVAL, DEFAULT_HOST, DEFAULT_PORT, DogStatsd, MIN_SEND_BUFFER_SIZE, PENDING_PAYLOAD_EXPIRY_SECONDS, PendingPayload, SenderQueue, Stop, UDP_OPTIMAL_PAYLOAD_LENGTH, UDS_CONNECT_RETRY_INITIAL_BACKOFF, UDS_OPTIMAL_PAYLOAD_LENGTH +from datadog.dogstatsd.sender_queue import is_replay_safe, payload_text from datadog.util.compat import monotonic as sender_queue_clock from datadog.dogstatsd.context import TimedContextManagerDecorator from datadog.util.compat import is_higher_py35, is_p3k @@ -2831,10 +2832,10 @@ def test_sender_queue_put_timeout_default_evicts_immediately(self): on_drop_expired=lambda item: self.fail("unexpected expiry drop"), ) - pending_queue.put(PendingPayload("first\n", sender_queue_clock(), False)) + pending_queue.put(PendingPayload("first\n", sender_queue_clock())) t0 = time.time() - pending_queue.put(PendingPayload("second\n", sender_queue_clock(), False)) + pending_queue.put(PendingPayload("second\n", sender_queue_clock())) elapsed = time.time() - t0 self.assertLess(elapsed, 0.05, "put() should not have waited at all") @@ -2852,10 +2853,10 @@ def test_sender_queue_put_timeout_zero_evicts_immediately(self): put_timeout=0, ) - pending_queue.put(PendingPayload("first\n", sender_queue_clock(), False)) + pending_queue.put(PendingPayload("first\n", sender_queue_clock())) t0 = time.time() - pending_queue.put(PendingPayload("second\n", sender_queue_clock(), False)) + pending_queue.put(PendingPayload("second\n", sender_queue_clock())) elapsed = time.time() - t0 self.assertLess(elapsed, 0.05, "put() should not have waited at all") @@ -2874,13 +2875,13 @@ def test_sender_queue_put_timeout_none_waits_forever_and_never_evicts(self): on_drop_expired=lambda item: self.fail("unexpected expiry drop"), put_timeout=None, ) - pending_queue.put(PendingPayload("first\n", sender_queue_clock(), False)) + pending_queue.put(PendingPayload("first\n", sender_queue_clock())) result = {} def blocked_put(): t0 = time.time() - pending_queue.put(PendingPayload("second\n", sender_queue_clock(), False)) + pending_queue.put(PendingPayload("second\n", sender_queue_clock())) result["elapsed"] = time.time() - t0 t = threading.Thread(target=blocked_put) @@ -2916,13 +2917,13 @@ def test_sender_queue_put_timeout_wakes_up_when_room_opens(self): on_drop_expired=lambda item: self.fail("unexpected expiry drop"), put_timeout=5.0, ) - pending_queue.put(PendingPayload("first\n", sender_queue_clock(), False)) + pending_queue.put(PendingPayload("first\n", sender_queue_clock())) result = {} def blocked_put(): t0 = time.time() - pending_queue.put(PendingPayload("second\n", sender_queue_clock(), False)) + pending_queue.put(PendingPayload("second\n", sender_queue_clock())) result["elapsed"] = time.time() - t0 t = threading.Thread(target=blocked_put) @@ -2951,10 +2952,10 @@ def test_sender_queue_put_timeout_falls_back_to_eviction(self): on_drop_expired=lambda item: self.fail("unexpected expiry drop"), put_timeout=0.2, ) - pending_queue.put(PendingPayload("first\n", sender_queue_clock(), False)) + pending_queue.put(PendingPayload("first\n", sender_queue_clock())) t0 = time.time() - pending_queue.put(PendingPayload("second\n", sender_queue_clock(), False)) + pending_queue.put(PendingPayload("second\n", sender_queue_clock())) elapsed = time.time() - t0 self.assertGreaterEqual(elapsed, 0.2) @@ -2981,7 +2982,7 @@ def test_sender_queue_bulk_expired_reclaim_wakes_blocked_producers(self): # cleanup loop has something to reclaim beyond the mandatory one. stale_clock = sender_queue_clock() - 1000.0 for i in range(maxsize): - pending_queue.put(PendingPayload("stale-{}\n".format(i), stale_clock, False)) + pending_queue.put(PendingPayload("stale-{}\n".format(i), stale_clock)) result = {} @@ -2989,11 +2990,11 @@ def evictor(): # Queue is full and nothing drains it, so this waits out # put_timeout and then falls back to eviction, whose cleanup loop # reclaims all remaining stale entries in one go. - pending_queue.put(PendingPayload("evictor\n", sender_queue_clock(), False)) + pending_queue.put(PendingPayload("evictor\n", sender_queue_clock())) def late_waiter(): t0 = time.time() - pending_queue.put(PendingPayload("late\n", sender_queue_clock(), False)) + pending_queue.put(PendingPayload("late\n", sender_queue_clock())) result["elapsed"] = time.time() - t0 t_evictor = threading.Thread(target=evictor) @@ -3040,10 +3041,10 @@ def test_sender_queue_requeue_front_never_blocks_on_put_timeout(self): on_drop_expired=lambda item: self.fail("unexpected expiry drop"), put_timeout=5.0, ) - in_flight = PendingPayload("in-flight\n", sender_queue_clock(), False) + in_flight = PendingPayload("in-flight\n", sender_queue_clock()) pending_queue.put(in_flight) got = pending_queue.get() - pending_queue.put(PendingPayload("new\n", sender_queue_clock(), False)) # fills the one slot again + pending_queue.put(PendingPayload("new\n", sender_queue_clock())) # fills the one slot again t0 = time.time() pending_queue.requeue_front(got) @@ -3065,9 +3066,9 @@ def test_sender_queue_drops_oldest_and_stale_entries_on_overflow(self): ) now = sender_queue_clock() - fresh = PendingPayload("fresh\n", now, False) - stale = PendingPayload("stale\n", now - 100, False) - newest = PendingPayload("newest\n", now, False) + fresh = PendingPayload("fresh\n", now) + stale = PendingPayload("stale\n", now - 100) + newest = PendingPayload("newest\n", now) # Fill the queue: [fresh, stale] (stale is already expired, but that # doesn't matter until something tries to make room or pull it off). @@ -3096,12 +3097,12 @@ def test_sender_queue_overflow_attributes_stale_oldest_entry_to_expiry(self): on_drop_expired=dropped_expired.append, ) - stale = PendingPayload("stale\n", sender_queue_clock() - 100, False) + stale = PendingPayload("stale\n", sender_queue_clock() - 100) pending_queue.put(stale) # The oldest (and only) entry being evicted is itself already # expired: that's a staleness drop, not a queue-full drop. - pending_queue.put(PendingPayload("newest\n", sender_queue_clock(), False)) + pending_queue.put(PendingPayload("newest\n", sender_queue_clock())) self.assertEqual(dropped_queue_full, []) self.assertEqual([p.payload for p in dropped_expired], ["stale\n"]) @@ -3117,9 +3118,9 @@ def test_sender_queue_get_drops_expired_entries(self): ) now = sender_queue_clock() - pending_queue.put(PendingPayload("stale-1\n", now - 100, False)) - pending_queue.put(PendingPayload("stale-2\n", now - 100, False)) - pending_queue.put(PendingPayload("fresh\n", now, False)) + pending_queue.put(PendingPayload("stale-1\n", now - 100)) + pending_queue.put(PendingPayload("stale-2\n", now - 100)) + pending_queue.put(PendingPayload("fresh\n", now)) # get() lazily drains every stale entry at the front before handing # back the next payload actually worth sending. @@ -3135,10 +3136,13 @@ def test_sender_queue_replay_safe_payload_never_expires(self): on_drop_expired=lambda item: self.fail("replay-safe payload should not expire"), ) - # Far older than the expiry window, but replay_safe=True: never dropped for staleness. - old_but_replay_safe = PendingPayload("timestamped\n", sender_queue_clock() - 10000, True) + # A replay-safe entry is the bare string, so there is no enqueued_at to + # age against at all -- it can never be dropped for staleness however + # long it sits there. + old_but_replay_safe = "timestamped\n" pending_queue.put(old_but_replay_safe) + self.assertTrue(is_replay_safe(old_but_replay_safe)) self.assertIs(pending_queue.get(), old_but_replay_safe) def test_sender_queue_requeue_front_when_room_available(self): @@ -3149,7 +3153,7 @@ def test_sender_queue_requeue_front_when_room_available(self): on_drop_expired=lambda item: self.fail("unexpected expiry drop"), ) - in_flight = PendingPayload("in-flight\n", sender_queue_clock(), False) + in_flight = PendingPayload("in-flight\n", sender_queue_clock()) pending_queue.put(in_flight) # Simulate the sender thread picking it up and failing to send it. @@ -3158,7 +3162,7 @@ def test_sender_queue_requeue_front_when_room_available(self): pending_queue.requeue_front(got) # There was room for it: it's retried first, ahead of anything newer. - pending_queue.put(PendingPayload("new\n", sender_queue_clock(), False)) + pending_queue.put(PendingPayload("new\n", sender_queue_clock())) self.assertEqual(pending_queue.get().payload, "in-flight\n") self.assertEqual(pending_queue.get().payload, "new\n") @@ -3172,14 +3176,14 @@ def test_sender_queue_requeue_front_drops_when_queue_is_full(self): on_drop_expired=lambda item: self.fail("unexpected expiry drop"), ) - in_flight = PendingPayload("in-flight\n", sender_queue_clock(), False) + in_flight = PendingPayload("in-flight\n", sender_queue_clock()) pending_queue.put(in_flight) # Simulate the sender thread picking it up, failing to send it, and a # fresh payload filling the now-empty slot in the meantime. got = pending_queue.get() self.assertIs(got, in_flight) - pending_queue.put(PendingPayload("new\n", sender_queue_clock(), False)) + pending_queue.put(PendingPayload("new\n", sender_queue_clock())) # The queue is already at maxsize: the requeue is dropped rather than # growing the queue past its limit or evicting the newer entry. @@ -3202,7 +3206,7 @@ def test_sender_queue_requeue_front_drops_when_expired(self): # Simulate the sender thread picking up a payload and failing to # send it, with enough time passing in between that it's now stale. # Unbounded queue (so it's never "full") isolates the expiry check. - in_flight = PendingPayload("stale\n", sender_queue_clock(), False) + in_flight = PendingPayload("stale\n", sender_queue_clock()) pending_queue.put(in_flight) got = pending_queue.get() time.sleep(0.02) @@ -3212,7 +3216,12 @@ def test_sender_queue_requeue_front_drops_when_expired(self): self.assertEqual([p.payload for p in dropped_expired], ["stale\n"]) self.assertEqual(pending_queue.qsize(), 0) - def test_replay_safe_flows_through_to_pending_payload(self): + def test_replay_safety_is_carried_by_the_queued_entry_type(self): + # Replay-safety is not a stored flag: an entry subject to expiry is a + # PendingPayload (carrying the enqueued_at it will be judged against), + # and a replay-safe one is the bare packet string. That keeps the + # wrapper -- and the GC traversal it implies -- off replay-safe entries + # entirely. statsd = DogStatsd(disable_background_sender=False) statsd.socket = FakeSocket() @@ -3231,14 +3240,26 @@ def capture_put(item): statsd.wait_for_pending() self.assertEqual(len(captured), 2) - self.assertFalse(captured[0].replay_safe) - self.assertIsNotNone(captured[0].enqueued_at, "non-replay-safe payloads need a real timestamp to expire against") - self.assertTrue(captured[1].replay_safe) - self.assertIsNone( - captured[1].enqueued_at, - "replay_safe payloads never have enqueued_at read (see SenderQueue._expired()), " - "so it should be skipped entirely rather than allocated for nothing", - ) + + expiring, replay_safe = captured + self.assertIsInstance(expiring, PendingPayload) + self.assertFalse(is_replay_safe(expiring)) + self.assertIsInstance( + expiring.enqueued_at, float, + "an expiring payload needs a real timestamp to be judged against", + ) + + # Deliberately not asserting a concrete string type here: on Python 2 + # the serialized packet is unicode, not str. What matters is that the + # entry is the bare payload rather than a wrapper, which is exactly + # what is_replay_safe()/payload_text() key off. + self.assertNotIsInstance(replay_safe, PendingPayload) + self.assertTrue(is_replay_safe(replay_safe)) + self.assertIs(payload_text(replay_safe), replay_safe) + + # Either form still yields its packet text the same way. + self.assertTrue(payload_text(expiring).startswith("no.timestamp")) + self.assertTrue(payload_text(replay_safe).startswith("with.timestamp")) statsd.stop() From ed38131bd0c8ab272e99579d756fb1a229767ac3 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 17 Sep 2026 14:43:38 +0100 Subject: [PATCH 15/30] Keep the split buffers off the per-metric hot path _send_to_buffer/_should_flush indexed {False:..., True:...} dicts by replay_safe, costing a bool() coercion plus a subscript on the single hottest path in the library (once per metric, ~18k/s per benchmark client). Measured in the benchmark image: _send_to_buffer 0.6388us on master -> 0.7344us on this branch (+15.0%), which was ~72% of the whole per-metric regression (+0.132us, +7.2%). Hold the two batches as four plain attributes and branch on replay_safe instead, and inline the size comparison so the hot path makes no call to _should_flush (which master also paid). _should_flush is retained for the pre-existing surface and for tests. Also stop reading the clock in SenderQueue.get() for entries that can never expire: the argument to _expired() is evaluated before the call, so every bare-string (replay-safe) get() computed a monotonic() it then discarded. Measured after, best-of-7: _send_to_buffer -16.5%, full per-metric -8.5%. Unit tests: 184 passed, 1 skipped with DD_ORIGIN_DETECTION_ENABLED=false (the 45 failures otherwise seen are a pre-existing container-id artifact of the dev host, identical before and after this change). --- datadog/dogstatsd/base.py | 83 ++- datadog/dogstatsd/sender_queue.py | 7 +- .../dogstatsd/test_statsd_sender.py | 2 +- .../emulate_reconnect_then_write_fails.py | 234 ++++++++ .../test_gauge_with_timestamp_aggregation.py | 22 + tests/manual/test_sender_queue_manual.py | 564 ++++++++++++++++++ tests/manual/test_shutdown_bound.py | 70 +++ tests/unit/dogstatsd/test_statsd.py | 2 +- 8 files changed, 958 insertions(+), 26 deletions(-) create mode 100644 tests/manual/emulate_reconnect_then_write_fails.py create mode 100644 tests/manual/test_gauge_with_timestamp_aggregation.py create mode 100644 tests/manual/test_sender_queue_manual.py create mode 100644 tests/manual/test_shutdown_bound.py diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index 2b8d47e5e..014962cca 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -27,7 +27,7 @@ # pylint: disable=unused-import if sys.version_info[:2] >= (3, 5): from typing import ( # noqa: F401 - Any, Callable, Dict, Iterable, List, Optional, Text, Tuple, Type, Union, overload, + Any, Callable, Iterable, List, Optional, Text, Tuple, Type, Union, overload, ) try: @@ -1164,14 +1164,25 @@ def close_buffer(self): def _reset_buffer(self): # type: () -> None with self._buffer_lock: - # Buffered lines are kept in two separate batches, keyed by - # whether they're replay-safe. Replay safe metrics are posted with - # the timestamp. - self._buffers = {False: [], True: []} # type: Dict[bool, List[Text]] + # Buffered lines are kept in two separate batches, one per expiry + # policy: replay-safe metrics carry their own timestamp, the rest + # are stamped on receipt. + # + # HELD AS FOUR PLAIN ATTRIBUTES, not as {False: ..., True: ...} + # dicts. This is the hottest path in the library -- touched once + # per metric, i.e. ~18k/s per client in the benchmark -- and the + # dict form cost a bool() coercion plus a subscript on every + # access. Measured: _send_to_buffer 0.6388us -> 0.7344us per metric + # (+15.0%) for the dict version, which was ~72% of this branch's + # entire per-metric CPU regression against master (+0.132us, + # +7.2%). Attribute access costs nothing extra and reads no worse. + self._buffer = [] # type: List[Text] # non-replay-safe + self._buffer_rs = [] # type: List[Text] # replay-safe # Running packet size per buffer, each including the newline that # will join its lines, so both stay under _max_payload_size # independently. - self._buffer_sizes = {False: 0, True: 0} # type: Dict[bool, int] + self._buffer_size = 0 # type: int + self._buffer_rs_size = 0 # type: int def flush(self): # type: () -> None @@ -1186,11 +1197,18 @@ def _flush_one_buffer(self, replay_safe): other one keeps accumulating, which is the whole point of splitting them. """ - lines = self._buffers[replay_safe] - if not lines: - return - self._buffers[replay_safe] = [] - self._buffer_sizes[replay_safe] = 0 + if replay_safe: + lines = self._buffer_rs + if not lines: + return + self._buffer_rs = [] + self._buffer_rs_size = 0 + else: + lines = self._buffer + if not lines: + return + self._buffer = [] + self._buffer_size = 0 self._send_to_server("\n".join(lines), replay_safe) def flush_buffered_metrics(self): @@ -1915,22 +1933,41 @@ def _xmit_packet_attempt(self, packet, is_telemetry, retry_eligible, retry_deadl def _send_to_buffer(self, packet, replay_safe=False): # type: (str, bool) -> None - with self._buffer_lock: - replay_safe = bool(replay_safe) - - if self._should_flush(len(packet), replay_safe): - self._flush_one_buffer(replay_safe) + """Append one serialized line to the batch matching its expiry policy. - self._buffers[replay_safe].append(packet) - # Update the current buffer length, including line break to anticipate - # the final packet size - self._buffer_sizes[replay_safe] += len(packet) + 1 + Deliberately written out per branch rather than indexing a dict by + replay_safe, and with the size check inlined rather than delegated to + _should_flush(): both cost real CPU at once-per-metric frequency. See + _reset_buffer() for the measurements. The bool() coercion the dict form + needed is gone too -- the branch treats any truthy value correctly. + """ + with self._buffer_lock: + # Length including the newline that will join this line to the + # next, so the running total anticipates the final packet size. + length = len(packet) + 1 + + if replay_safe: + if self._buffer_rs_size + length > self._max_payload_size: + self._flush_one_buffer(True) + self._buffer_rs.append(packet) + self._buffer_rs_size += length + else: + if self._buffer_size + length > self._max_payload_size: + self._flush_one_buffer(False) + self._buffer.append(packet) + self._buffer_size += length def _should_flush(self, length_to_be_added, replay_safe=False): # type: (int, bool) -> bool - if self._buffer_sizes[bool(replay_safe)] + length_to_be_added + 1 > self._max_payload_size: - return True - return False + """Whether adding a line of this length would overflow its batch. + + NOT used by _send_to_buffer(), which inlines the same comparison to + keep a function call off the per-metric path. Retained because it is + part of the pre-existing surface and is convenient in tests; keep the + two in step if either changes. + """ + current = self._buffer_rs_size if replay_safe else self._buffer_size + return current + length_to_be_added + 1 > self._max_payload_size @staticmethod def _escape_event_content(string): diff --git a/datadog/dogstatsd/sender_queue.py b/datadog/dogstatsd/sender_queue.py index debc858b2..95098e389 100644 --- a/datadog/dogstatsd/sender_queue.py +++ b/datadog/dogstatsd/sender_queue.py @@ -237,7 +237,12 @@ def get(self): if item is Stop: return item - if self._expired(item, monotonic()): + # Guard the monotonic() call on the type test rather than letting + # _expired() do it: the argument is evaluated BEFORE the call, so + # `self._expired(item, monotonic())` read the clock on every get() + # including for bare strings, which are replay-safe and can never + # expire, so the value was computed and immediately discarded. + if isinstance(item, PendingPayload) and self._expired(item, monotonic()): self._on_drop_expired(item) self.task_done() continue diff --git a/tests/integration/dogstatsd/test_statsd_sender.py b/tests/integration/dogstatsd/test_statsd_sender.py index 4fc4e735f..5841cd772 100644 --- a/tests/integration/dogstatsd/test_statsd_sender.py +++ b/tests/integration/dogstatsd/test_statsd_sender.py @@ -104,7 +104,7 @@ def test_fork_hooks(disable_background_sender, disable_buffering): assert statsd._sender_thread is None assert statsd._queue is None or statsd._queue.empty() # Buffered lines are split by expiry policy, so check every batch. - assert not any(statsd._buffers.values()) + assert not statsd._buffer and not statsd._buffer_rs statsd.post_fork_parent() diff --git a/tests/manual/emulate_reconnect_then_write_fails.py b/tests/manual/emulate_reconnect_then_write_fails.py new file mode 100644 index 000000000..d66695311 --- /dev/null +++ b/tests/manual/emulate_reconnect_then_write_fails.py @@ -0,0 +1,234 @@ +""" +Emulates: server crashes -> write fails -> client reconnects -> reconnect +succeeds -> the very next write fails again. + +This is a controlled, deterministic emulation (not a probabilistic race): we +puppet the "agent" side explicitly so every step happens in the exact order +described, every time you run this script. + +Two parts: + + 1. `raw_socket_narrative()` - plain UDS SOCK_DGRAM sockets, no datadogpy + involved, just to show the sequence of syscalls/return values in + isolation. + + 2. `through_dogstatsd_client()` - drives the actual + `datadog.dogstatsd.base.DogStatsd` client through the same induced + sequence (by monkeypatching `_get_uds_socket` to kill the "agent" at the + right moments) so you can see how the real retry/backoff code reacts: + does it recover, retry again, or drop the packet. + +Run: + python3 tests/manual/emulate_reconnect_then_write_fails.py +""" +import errno +import os +import socket +import sys +import time + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + +SOCK_PATH = "/tmp/emulate_reconnect_then_write_fails.sock" + + +def errname(exc): + return errno.errorcode.get(getattr(exc, "errno", None), str(exc)) + + +def _fresh_path(): + try: + os.unlink(SOCK_PATH) + except OSError: + pass + + +def _bind_server(): + """Start a UDS SOCK_DGRAM 'agent' bound at SOCK_PATH.""" + srv = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + srv.bind(SOCK_PATH) + return srv + + +def raw_socket_narrative(): + print("=" * 70) + print("PART 1: raw UDS sockets, step by step") + print("=" * 70) + + _fresh_path() + + # --- Step 0: agent v1 is up, client is connected and happily sending --- + print("\n[step 0] agent v1 starts, client connects") + agent1 = _bind_server() + client = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + client.connect(SOCK_PATH) + client.send(b"metric.a:1|c") + print(" client wrote successfully:", agent1.recv(1024)) + + # --- Step 1: agent crashes --- + print("\n[step 1] agent v1 CRASHES (socket closed, file left on disk)") + agent1.close() + + # --- Step 2: client's next write fails --- + print("[step 2] client writes on the now-dead connection ...") + try: + client.send(b"metric.b:1|c") + print(" unexpectedly succeeded") + except OSError as e: + print(f" write FAILED as expected: {errname(e)} ({e})") + + # --- Step 3: supervisor restarts the agent (agent v2) very quickly --- + print("\n[step 3] supervisor restarts the agent (agent v2 binds at the same path)") + _fresh_path() + agent2 = _bind_server() + + # --- Step 4: client reconnects --- + print("[step 4] client closes its old socket and reconnects") + client.close() + client = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + try: + client.connect(SOCK_PATH) + print(" reconnect SUCCEEDED (agent v2 is genuinely up right now)") + except OSError as e: + print(f" reconnect failed: {errname(e)} ({e}) -- unexpected for this narrative") + return + + # --- Step 5: agent v2 crashes immediately (flaky restart) --- + print("\n[step 5] agent v2 CRASHES immediately, before the client's next write") + agent2.close() + + # --- Step 6: the very next write after the successful reconnect fails too --- + print("[step 6] client writes again, right after the successful reconnect ...") + try: + client.send(b"metric.c:1|c") + print(" unexpectedly succeeded") + except OSError as e: + print(f" write FAILED again: {errname(e)} ({e})") + print("\n >>> proven: a successful reconnect does not guarantee the next write survives <<<") + + client.close() + _fresh_path() + + +def through_dogstatsd_client(): + print("\n" + "=" * 70) + print("PART 2: the same sequence, driven through the real DogStatsd client") + print("=" * 70) + print("_get_uds_socket now makes exactly one connect attempt -- no internal") + print("retry loop. Reconnect-and-retry is the background sender's job now, gated") + print("by socket_connect_retry, and it lives in _sender_main_loop: call") + print("_xmit_packet(..., queue_mode=True), and if it reports a retryable failure") + print("(None), back off and call it again. That's what this drives directly, to") + print("keep watching the same connect/send failures _xmit_packet_attempt sees.") + + from datadog.dogstatsd.base import ( + DogStatsd, + UDS_CONNECT_RETRY_INITIAL_BACKOFF, + UDS_CONNECT_RETRY_MAX_BACKOFF, + ) + + _fresh_path() + agent = {"sock": _bind_server(), "generation": 0} + + statsd = DogStatsd(socket_path=SOCK_PATH, socket_connect_timeout=2.0) + + real_get_uds_socket = DogStatsd._get_uds_socket.__func__ + call_log = [] + state = {"restart_soon_scheduled": False, "post_reconnect_crash_done": False} + + def scripted_get_uds_socket(cls, socket_path, timeout): + """ + Wraps the real (now single-attempt) connect, but scripts the agent's + state around it so the narrative is deterministic regardless of + exactly which attempt number the retry loop below is on when each + event happens: + first call ever: agent is already dead (crashed) before this + connect -- this attempt fails; the retry loop below + (standing in for _sender_main_loop) is what waits for + agent v2 and calls again + first call that actually succeeds: connect succeeds against agent + v2, which we then kill immediately afterwards so the + following send fails again + """ + call_log.append(1) + n = len(call_log) + + if not state["restart_soon_scheduled"]: + state["restart_soon_scheduled"] = True + print("\n[client] first connect attempt -- agent is currently down") + agent["sock"].close() + _fresh_path() + + # Simulate the supervisor's restart happening a little while into + # the retry loop below, so its backoff-and-retry is what catches + # the agent coming back -- not necessarily the very next attempt. + def restart_soon(): + time.sleep(0.15) + _fresh_path() + agent["sock"] = _bind_server() + agent["generation"] = 1 + import threading + threading.Thread(target=restart_soon, daemon=True).start() + + try: + sock = real_get_uds_socket(cls, socket_path, timeout) + except Exception as e: + print(f"[client] connect attempt #{n} failed: {errname(e)} (agent still down)") + raise + + print(f"[client] connect attempt #{n} SUCCEEDED (agent v2 is up)") + if not state["post_reconnect_crash_done"]: + state["post_reconnect_crash_done"] = True + print("[client] ...but agent v2 crashes again immediately, before the send:") + agent["sock"].close() + _fresh_path() + + # Agent v3 comes back shortly after -- this is what lets a *later* + # retry attempt (triggered by the send failure right after this + # connect) actually succeed, so we can see the client ride out + # both failures end to end. + def restart_again(): + time.sleep(0.15) + _fresh_path() + agent["sock"] = _bind_server() + agent["generation"] = 2 + import threading + threading.Thread(target=restart_again, daemon=True).start() + + return sock + + DogStatsd._get_uds_socket = classmethod(scripted_get_uds_socket) + try: + t0 = time.time() + backoff = UDS_CONNECT_RETRY_INITIAL_BACKOFF + attempt = 0 + sent = None # None means "retryable failure, try again" -- see _xmit_packet. + while sent is None: + attempt += 1 + sent = statsd._xmit_packet("emulated.metric:1|c", False, queue_mode=True) + if sent is None: + time.sleep(backoff) + backoff = min(backoff * 2, UDS_CONNECT_RETRY_MAX_BACKOFF) + elapsed = time.time() - t0 + finally: + DogStatsd._get_uds_socket = classmethod(real_get_uds_socket) + try: + agent["sock"].close() + except OSError: + pass + _fresh_path() + + print(f"\n[result] _xmit_packet(..., queue_mode=True) -> sent={sent}, elapsed={elapsed:.3f}s, " + f"retry-loop attempts={attempt}, get_uds_socket calls={len(call_log)}") + print(f"[result] packets_dropped_writer={statsd.packets_dropped_writer}") + if sent: + print(" -> the retry loop (standing in for the background sender) absorbed BOTH") + print(" failures (the initial dead-agent connect AND the post-reconnect send") + print(" failure) and eventually delivered the packet once the agent stayed up.") + else: + print(" -> the packet was dropped.") + + +if __name__ == "__main__": + raw_socket_narrative() + through_dogstatsd_client() diff --git a/tests/manual/test_gauge_with_timestamp_aggregation.py b/tests/manual/test_gauge_with_timestamp_aggregation.py new file mode 100644 index 000000000..6ab276a74 --- /dev/null +++ b/tests/manual/test_gauge_with_timestamp_aggregation.py @@ -0,0 +1,22 @@ +import time + +from datadog.dogstatsd.base import DogStatsd + +client = DogStatsd( + socket_path="/tmp/dsd.sock", + disable_aggregation=True, + disable_buffering=False, + flush_interval=1.0, + disable_telemetry=True, +) + +start = time.time() +i = 0 + +while time.time() - start < 10: + i += 1 + client.gauge_with_timestamp("test.aggregation", float(i), tags=["env:test"], timestamp=time.time()) + +client.stop() +print("Done.") + diff --git a/tests/manual/test_sender_queue_manual.py b/tests/manual/test_sender_queue_manual.py new file mode 100644 index 000000000..e90ee06f3 --- /dev/null +++ b/tests/manual/test_sender_queue_manual.py @@ -0,0 +1,564 @@ +""" +Manual, narrated test-drive of `datadog.dogstatsd.sender_queue.SenderQueue`. + +Unlike the unit tests, this script is meant to be *read while it runs*: every +scenario prints what it's about to do, what actually happened, and then +asserts on the outcome. It exercises the queue in isolation first (no +threads, no sockets), then stresses it with real concurrent producers/ +consumers, and finally drives it through the real `DogStatsd` client end to +end (drop-oldest, real-time expiry, and requeue-on-connection-failure). + +Usage: + python3 tests/manual/test_sender_queue_manual.py + +Exits 0 if every check passed, 1 otherwise. +""" +import errno +import os +import socket +import sys +import threading +import time + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + +from datadog.dogstatsd.sender_queue import ( # noqa: E402 + PendingPayload, + SenderQueue, + Stop, + is_replay_safe, + payload_text, +) +from datadog.util.compat import monotonic # noqa: E402 +import datadog.dogstatsd.base as base_module # noqa: E402 +from datadog.dogstatsd.base import DogStatsd # noqa: E402 + + +# -------------------------------------------------------------------------- +# Small helpers: narration + a pass/fail ledger shared across every scenario. +# -------------------------------------------------------------------------- +_RESULTS = {"pass": 0, "fail": 0, "failures": []} + + +def section(title): + print() + print("=" * 78) + print(title) + print("=" * 78) + + +def note(msg): + print(" . {}".format(msg)) + + +def check(condition, description): + if condition: + _RESULTS["pass"] += 1 + print(" [PASS] {}".format(description)) + else: + _RESULTS["fail"] += 1 + _RESULTS["failures"].append(description) + print(" [FAIL] {}".format(description)) + return condition + + +def recorder(label): + """A drop callback that remembers every item it was called with and prints it.""" + events = [] + + def _on_drop(item): + events.append(item) + rs = is_replay_safe(item) + # Replay-safe entries are bare strings with no enqueued_at to age. + age = "n/a" if rs else "{:.3f}s".format(monotonic() - item.enqueued_at) + print( + " -> {label} fired: payload={payload!r} age={age} replay_safe={rs}".format( + label=label, payload=payload_text(item), age=age, rs=rs + ) + ) + + _on_drop.events = events + return _on_drop + + +def new_recorders(): + return recorder("on_drop_queue_full"), recorder("on_drop_expired") + + +def quiet_recorder(label, report_every=100): + """Like recorder(), but only prints a running count every `report_every` + hits instead of one line per drop -- for scenarios with hundreds of them, + where line-per-event narration would just be noise.""" + events = [] + + def _on_drop(item): + events.append(item) + if len(events) % report_every == 0: + print(" -> {label}: {count} drops so far (e.g. {sample!r})".format( + label=label, count=len(events), sample=payload_text(item).strip() + )) + + _on_drop.events = events + return _on_drop + + +def payload(text, replay_safe=False, enqueued_at=None): + """Build a queue entry: a bare string when replay-safe, else a wrapper.""" + if replay_safe: + return text + return PendingPayload(text, enqueued_at if enqueued_at is not None else monotonic()) + + +# -------------------------------------------------------------------------- +# Scenario 1: plain FIFO, no eviction, no expiry. +# -------------------------------------------------------------------------- +def scenario_1_fifo_order(): + section("1. Basic FIFO ordering (unbounded, nothing expires)") + drop_full, drop_expired = new_recorders() + q = SenderQueue(maxsize=0, expiry_seconds=100.0, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) + + for name in ("a", "b", "c"): + q.put(payload(name + "\n")) + note("put({!r}) -> qsize={}".format(name, q.qsize())) + + order = [payload_text(q.get()) for _ in range(3)] + note("got, in order: {}".format(order)) + + check(order == ["a\n", "b\n", "c\n"], "FIFO order preserved") + check(not drop_full.events and not drop_expired.events, "no drops during a plain unbounded run") + check(q.qsize() == 0, "queue fully drained") + + +# -------------------------------------------------------------------------- +# Scenario 2: overflow evicts the oldest entry to make room. +# -------------------------------------------------------------------------- +def scenario_2_overflow_drops_oldest(): + section("2. Overflow: the oldest entry is evicted to make room") + drop_full, drop_expired = new_recorders() + q = SenderQueue(maxsize=2, expiry_seconds=100.0, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) + + q.put(payload("first\n")) + note("put('first') -> qsize={}".format(q.qsize())) + q.put(payload("second\n")) + note("put('second') -> qsize={} (queue is now at maxsize=2)".format(q.qsize())) + note("putting a third payload...") + q.put(payload("third\n")) + note("put('third') -> qsize={}".format(q.qsize())) + + check(q.qsize() == 2, "queue never grew past maxsize") + check([payload_text(e) for e in drop_full.events] == ["first\n"], "the OLDEST entry was evicted via on_drop_queue_full") + check(not drop_expired.events, "nothing was expired -- this was a pure capacity eviction") + + remaining = [payload_text(q.get()) for _ in range(2)] + note("remaining, in order: {}".format(remaining)) + check(remaining == ["second\n", "third\n"], "the two newest survivors come out in FIFO order") + + +# -------------------------------------------------------------------------- +# Scenario 3: an evicted oldest entry that's ALSO stale is attributed to expiry. +# -------------------------------------------------------------------------- +def scenario_3_overflow_prefers_expiry_reason(): + section("3. Overflow where the evicted entry is ALSO already stale") + drop_full, drop_expired = new_recorders() + q = SenderQueue(maxsize=1, expiry_seconds=0.2, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) + + q.put(payload("stale\n")) + note("put('stale'); sleeping 0.25s so it ages past the 0.2s expiry window...") + time.sleep(0.25) + + note("queue is at maxsize=1; putting a fresh payload now forces an eviction") + q.put(payload("fresh\n")) + + check(not drop_full.events, "NOT counted as a plain capacity drop") + check([payload_text(e) for e in drop_expired.events] == ["stale\n"], "counted as an EXPIRY drop instead -- more informative") + check(payload_text(q.get()) == "fresh\n", "the fresh payload survived") + + +# -------------------------------------------------------------------------- +# Scenario 4: get() lazily drains every stale entry at the front. +# -------------------------------------------------------------------------- +def scenario_4_get_drains_stale_entries(): + section("4. get() drains every stale entry at the front before returning") + drop_full, drop_expired = new_recorders() + q = SenderQueue(maxsize=0, expiry_seconds=0.2, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) + + q.put(payload("stale-1\n")) + q.put(payload("stale-2\n")) + note("put two payloads; sleeping 0.25s so both age past the 0.2s expiry window...") + time.sleep(0.25) + q.put(payload("fresh\n")) + note("qsize before get(): {}".format(q.qsize())) + + item = q.get() + note("qsize after get(): {}".format(q.qsize())) + + check(payload_text(item) == "fresh\n", "get() skipped both stale entries and returned the fresh one") + check([payload_text(e) for e in drop_expired.events] == ["stale-1\n", "stale-2\n"], "both stale entries dropped, in order, along the way") + + +# -------------------------------------------------------------------------- +# Scenario 5: replay_safe payloads never expire. +# -------------------------------------------------------------------------- +def scenario_5_replay_safe_never_expires(): + section("5. replay_safe=True payloads are exempt from expiry") + drop_full, drop_expired = new_recorders() + q = SenderQueue(maxsize=0, expiry_seconds=0.1, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) + + q.put(payload("timestamped\n", replay_safe=True)) + note("put a replay_safe payload; sleeping 0.3s, well past the 0.1s expiry window...") + time.sleep(0.3) + + item = q.get() + note("get() returned: {!r}".format(payload_text(item) if item is not None else None)) + check(item is not None and payload_text(item) == "timestamped\n", "still returned by get(), never dropped") + check(not drop_expired.events, "on_drop_expired was never called for it") + + +# -------------------------------------------------------------------------- +# Scenario 6: requeue_front() -- the three outcomes. +# -------------------------------------------------------------------------- +def scenario_6a_requeue_front_with_room(): + section("6a. requeue_front(): succeeds when there's room, rejoins at the FRONT") + drop_full, drop_expired = new_recorders() + q = SenderQueue(maxsize=2, expiry_seconds=100.0, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) + + q.put(payload("in-flight\n")) + in_flight = q.get() + note("sender thread picked up 'in-flight' and (simulated) failed to send it") + q.requeue_front(in_flight) + note("requeued 'in-flight'; qsize={}".format(q.qsize())) + q.put(payload("new\n")) + note("put 'new'; qsize={}".format(q.qsize())) + + order = [payload_text(q.get()) for _ in range(2)] + note("got, in order: {}".format(order)) + check(order == ["in-flight\n", "new\n"], "the requeued item is retried BEFORE the newer one") + check(not drop_full.events and not drop_expired.events, "nothing was dropped") + + +def scenario_6b_requeue_front_drops_when_full(): + section("6b. requeue_front(): dropped via on_drop_queue_full when the queue is already full") + drop_full, drop_expired = new_recorders() + q = SenderQueue(maxsize=1, expiry_seconds=100.0, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) + + q.put(payload("in-flight\n")) + in_flight = q.get() + note("sender thread picked up 'in-flight'; meanwhile a fresh payload fills the now-empty slot") + q.put(payload("new\n")) + note("qsize is already at maxsize=1; the failed send now tries to requeue 'in-flight'...") + q.requeue_front(in_flight) + + check(q.qsize() == 1, "queue never grew past maxsize") + check([payload_text(e) for e in drop_full.events] == ["in-flight\n"], "'in-flight' was dropped via on_drop_queue_full") + check(payload_text(q.get()) == "new\n", "'new' is untouched and still gets sent") + + +def scenario_6c_requeue_front_drops_when_expired(): + section("6c. requeue_front(): dropped via on_drop_expired when it went stale in flight, even with room to spare") + drop_full, drop_expired = new_recorders() + q = SenderQueue(maxsize=0, expiry_seconds=0.15, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) + + q.put(payload("slow-send\n")) + picked_up = q.get() + note("sender thread picked up 'slow-send' and is (simulated) stuck retrying the connection...") + time.sleep(0.2) + note("...0.2s later the send finally fails; the queue is unbounded, so there is plenty of room") + q.requeue_front(picked_up) + + check(q.qsize() == 0, "NOT requeued despite there being room -- expiry wins") + check([payload_text(e) for e in drop_expired.events] == ["slow-send\n"], "dropped via on_drop_expired") + + +# -------------------------------------------------------------------------- +# Scenario 7: _unfinished_tasks / task_done() / join() bookkeeping. +# -------------------------------------------------------------------------- +def scenario_7_task_done_and_join(): + section("7. _unfinished_tasks bookkeeping across put()/task_done()/join()") + drop_full, drop_expired = new_recorders() + q = SenderQueue(maxsize=0, expiry_seconds=100.0, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) + + for name in ("x", "y", "z"): + q.put(payload(name + "\n")) + note("put 3 payloads -> _unfinished_tasks={}".format(q._unfinished_tasks)) + check(q._unfinished_tasks == 3, "one unfinished task recorded per put()") + + joined = {"done": False} + + def joiner(): + q.join() + joined["done"] = True + + t = threading.Thread(target=joiner) + t.start() + time.sleep(0.1) + note("join() called on a background thread; joined={}".format(joined["done"])) + check(not joined["done"], "join() is still blocked -- 3 tasks are still outstanding") + + for _ in range(3): + item = q.get() + note("get() -> {!r}, calling task_done()".format(payload_text(item))) + q.task_done() + + t.join(timeout=2) + note("after 3x task_done(): joined={} _unfinished_tasks={}".format(joined["done"], q._unfinished_tasks)) + check(joined["done"], "join() returned once every task was marked done") + check(q._unfinished_tasks == 0, "_unfinished_tasks back to zero") + + +# -------------------------------------------------------------------------- +# Scenario 8: the Stop sentinel bypasses capacity and expiry entirely. +# -------------------------------------------------------------------------- +def scenario_8_stop_sentinel(): + section("8. The Stop sentinel bypasses capacity limits and expiry entirely") + drop_full, drop_expired = new_recorders() + q = SenderQueue(maxsize=1, expiry_seconds=100.0, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) + + q.put(payload("only-slot\n")) + note("queue is at maxsize=1; putting Stop now...") + q.put(Stop) + note("qsize={} (Stop did not evict 'only-slot', nor was it evicted itself)".format(q.qsize())) + + check(q.qsize() == 2, "Stop was appended past maxsize instead of triggering eviction") + + first = q.get() + second = q.get() + check(payload_text(first) == "only-slot\n", "the real payload still comes out first (FIFO)") + check(second is Stop, "Stop comes out exactly as put in, untouched by drop logic") + q.task_done() + q.task_done() + + +# -------------------------------------------------------------------------- +# Scenario 9: concurrency smoke test -- multiple producers racing a consumer. +# -------------------------------------------------------------------------- +def scenario_9_concurrency_smoke_test(): + section("9. Concurrency smoke test: 4 producer threads racing 1 consumer thread") + note("this scenario can trigger hundreds of evictions -- using quiet_recorder() to summarize instead of narrating every one") + drop_full, drop_expired = quiet_recorder("on_drop_queue_full"), quiet_recorder("on_drop_expired") + q = SenderQueue(maxsize=20, expiry_seconds=5.0, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) + + n_per_producer = 200 + n_producers = 4 + total = n_per_producer * n_producers + received = [] + + def producer(pid): + for i in range(n_per_producer): + q.put(payload("p{}-{}\n".format(pid, i))) + + def consumer(): + while True: + item = q.get() + if item is Stop: + q.task_done() + return + received.append(item) + q.task_done() + # Artificial slowness so the 4 producers reliably outrun the + # consumer and we actually get to see maxsize=20 evictions + # happen, instead of everything just being consumed in time. + time.sleep(0.0005) + + note("maxsize={} expiry_seconds={} total_payloads={}".format(q._maxsize, q._expiry_seconds, total)) + producers = [threading.Thread(target=producer, args=(pid,)) for pid in range(n_producers)] + consumer_thread = threading.Thread(target=consumer) + + t0 = time.time() + consumer_thread.start() + for p in producers: + p.start() + for p in producers: + p.join() + note("all {} producers finished putting {} payloads in {:.3f}s".format(n_producers, total, time.time() - t0)) + + q.put(Stop) + consumer_thread.join(timeout=10) + + accounted = len(received) + len(drop_full.events) + len(drop_expired.events) + note( + "received={} dropped_full={} dropped_expired={} accounted_total={} expected_total={}".format( + len(received), len(drop_full.events), len(drop_expired.events), accounted, total + ) + ) + check(not consumer_thread.is_alive(), "consumer thread exited cleanly after Stop") + check(accounted == total, "every put() payload is accounted for exactly once (sent, capacity-dropped, or expired)") + check(q.qsize() == 0, "queue fully drained") + check(q._unfinished_tasks == 0, "_unfinished_tasks settled back to zero under real concurrency") + check(len(drop_full.events) > 0, "the slow consumer + maxsize=20 did trigger at least one real eviction") + + +# -------------------------------------------------------------------------- +# Scenario 10: end-to-end through the real DogStatsd client. +# -------------------------------------------------------------------------- +class ScriptedSocket(object): + """Just enough of a socket to drive DogStatsd's send path without a real network.""" + + family = socket.AF_UNIX + + def __init__(self): + self.received = [] + + def send(self, data): + self.received.append(data) + return len(data) + + def sendall(self, data): + self.received.append(data) + + def settimeout(self, *_args): + pass + + def getsockopt(self, *_args): + return socket.SOCK_DGRAM + + def close(self): + pass + + +def scenario_10a_client_drop_oldest(): + section("10a. End-to-end through DogStatsd: drop-oldest via the real client") + # No background sender thread on purpose. With one running it races this + # sequence: if it drains 'first' before 'second' is queued, the queue is + # never full, nothing is evicted, and the checks below describe a schedule + # that didn't happen (this scenario failed ~7 runs in 10 that way). Build + # the queue by hand -- the same wiring _start_sender_thread() uses -- then + # drive the real sender loop synchronously at the end. + statsd = DogStatsd(disable_background_sender=True, disable_telemetry=True) + statsd.socket = ScriptedSocket() + statsd._queue = SenderQueue( + 1, + base_module.PENDING_PAYLOAD_EXPIRY_SECONDS, + statsd._account_dropped_queue_full, + statsd._account_dropped_expired, + ) + + statsd._send_to_server("first") + note("_send_to_server('first') -> qsize={}".format(statsd._queue.qsize())) + statsd._send_to_server("second") + note("_send_to_server('second') -> qsize={}".format(statsd._queue.qsize())) + note("bytes_dropped_queue={} packets_dropped_queue={}".format(statsd.bytes_dropped_queue, statsd.packets_dropped_queue)) + + check(statsd.packets_dropped_queue == 1, "one packet ('first') was dropped for capacity") + + # Drain through the real sender loop; Stop makes it return once done. + statsd._queue.put(Stop) + statsd._sender_main_loop(statsd._queue) + note("socket received: {}".format(statsd.socket.received)) + check(statsd.socket.received == [b"second\n"], "only the surviving (newest) payload was actually sent") + + +def scenario_10b_client_real_time_expiry(): + section("10b. End-to-end through DogStatsd: real-time expiry with no reachable agent") + # NOTE: base.py does `from datadog.dogstatsd.sender_queue import ... PENDING_PAYLOAD_EXPIRY_SECONDS`, + # which binds its OWN name in base's namespace at import time. Patching + # sender_queue.PENDING_PAYLOAD_EXPIRY_SECONDS after that has no effect on + # _start_sender_thread(), which reads base's copy of the name -- so that's + # the one that has to be patched here. + original_expiry = base_module.PENDING_PAYLOAD_EXPIRY_SECONDS + base_module.PENDING_PAYLOAD_EXPIRY_SECONDS = 0.3 + note("patched datadog.dogstatsd.base.PENDING_PAYLOAD_EXPIRY_SECONDS: {} -> {}".format(original_expiry, base_module.PENDING_PAYLOAD_EXPIRY_SECONDS)) + + statsd = None + try: + statsd = DogStatsd( + socket_path="/tmp/sender-queue-manual-test-nonexistent-{}.sock".format(os.getpid()), + socket_connect_timeout=0.05, + disable_background_sender=False, + disable_telemetry=True, + ) + statsd.increment("will.expire") + note("queued 'will.expire' against a socket path that doesn't exist; waiting for wait_for_pending()...") + + t0 = time.time() + statsd.wait_for_pending() + elapsed = time.time() - t0 + note("wait_for_pending() returned after {:.3f}s".format(elapsed)) + + check(statsd.packets_dropped_expired == 1, "the packet expired instead of being retried forever") + check(statsd.packets_dropped_writer == 0, "it was NOT mistaken for a hard write failure") + check(elapsed < 5.0, "expiry actually bounded how long wait_for_pending() took") + finally: + base_module.PENDING_PAYLOAD_EXPIRY_SECONDS = original_expiry + note("restored PENDING_PAYLOAD_EXPIRY_SECONDS to {}".format(original_expiry)) + if statsd is not None: + statsd.stop() + + +def scenario_10c_client_requeue_then_succeeds(): + section("10c. End-to-end through DogStatsd: requeue-and-retry survives a flaky reconnect, then succeeds") + working_socket = ScriptedSocket() + attempts = {"count": 0} + fail_until = 4 + + def flaky_get_uds_socket(_cls, _socket_path, _timeout, _connect_timeout): + attempts["count"] += 1 + if attempts["count"] < fail_until: + note("reconnect attempt #{}: still refused".format(attempts["count"])) + raise socket.error(errno.ECONNREFUSED, "still refused") + note("reconnect attempt #{}: agent is back up".format(attempts["count"])) + return working_socket + + real_get_uds_socket = DogStatsd._get_uds_socket + DogStatsd._get_uds_socket = classmethod(flaky_get_uds_socket) + statsd = None + try: + statsd = DogStatsd( + socket_path="/tmp/sender-queue-manual-test-flaky-{}.sock".format(os.getpid()), + socket_connect_timeout=0.05, + disable_background_sender=False, + disable_telemetry=True, + ) + statsd.gauge("eventually.sent", 1) + t0 = time.time() + statsd.wait_for_pending() + elapsed = time.time() - t0 + finally: + DogStatsd._get_uds_socket = real_get_uds_socket + + note("wait_for_pending() returned after {:.3f}s and {} reconnect attempts".format(elapsed, attempts["count"])) + check(attempts["count"] >= fail_until, "it took multiple reconnect attempts, exercising requeue-and-retry") + check(statsd.packets_dropped_writer == 0, "never hard-dropped as a write failure") + check(statsd.packets_dropped_expired == 0, "never expired -- it succeeded well before the TTL") + check( + bool(working_socket.received) and working_socket.received[0].startswith(b"eventually.sent:1|g"), + "the packet was actually delivered once the agent came back", + ) + + statsd.stop() + + +# -------------------------------------------------------------------------- +def main(): + scenarios = [ + scenario_1_fifo_order, + scenario_2_overflow_drops_oldest, + scenario_3_overflow_prefers_expiry_reason, + scenario_4_get_drains_stale_entries, + scenario_5_replay_safe_never_expires, + scenario_6a_requeue_front_with_room, + scenario_6b_requeue_front_drops_when_full, + scenario_6c_requeue_front_drops_when_expired, + scenario_7_task_done_and_join, + scenario_8_stop_sentinel, + scenario_9_concurrency_smoke_test, + scenario_10a_client_drop_oldest, + scenario_10b_client_real_time_expiry, + scenario_10c_client_requeue_then_succeeds, + ] + + start = time.time() + for scenario in scenarios: + scenario() + + section("SUMMARY") + print(" {} passed, {} failed, {:.2f}s total".format(_RESULTS["pass"], _RESULTS["fail"], time.time() - start)) + if _RESULTS["failures"]: + print(" Failed checks:") + for description in _RESULTS["failures"]: + print(" - {}".format(description)) + + sys.exit(1 if _RESULTS["fail"] else 0) + + +if __name__ == "__main__": + main() diff --git a/tests/manual/test_shutdown_bound.py b/tests/manual/test_shutdown_bound.py new file mode 100644 index 000000000..11f90fd52 --- /dev/null +++ b/tests/manual/test_shutdown_bound.py @@ -0,0 +1,70 @@ +"""Manual check that shutdown stays bounded while the sender is retrying. + +Usage: python tests/manual/test_shutdown_bound.py [n_packets] [stop_timeout] + +Fills the background sender queue while the agent is unreachable, with +socket_connect_timeout=2.0 so the sender keeps retrying the head-of-queue +payload indefinitely, backing off up to UDS_CONNECT_RETRY_MAX_BACKOFF (a minute) +between attempts. + +Two things used to make stop() drag or hang here, both fixed: + + * The backoff was a plain time.sleep(), so stop() had to wait out whatever + was left of it -- up to a minute. + * requeue_front() puts the failed payload back at the *front* of the queue, + ahead of the Stop sentinel, so the sender only ever noticed Stop once the + head payload was finally resolved. Ordinary payloads resolve via the + queue's expiry, but replay-safe ones (gauge_with_timestamp, events with + date_happened, service checks with a timestamp) are exempt from expiry -- + so one of those at the head starved Stop forever and stop() never + returned at all. + +Both are now handled by a stopping Event that the sender waits on instead of +sleeping, so expect stop() to return promptly and report True regardless of +what is queued. Pass a replay-safe metric through (see REPLAY_SAFE below) to +exercise the case that used to hang. +""" +import os +import sys +import tempfile +import time + +from datadog.dogstatsd.base import DogStatsd + +n_packets = int(sys.argv[1]) if len(sys.argv) > 1 else 50000 +stop_timeout = float(sys.argv[2]) if len(sys.argv) > 2 else 5.0 +REPLAY_SAFE = os.environ.get("REPLAY_SAFE") == "1" + +socket_path = os.path.join(tempfile.mkdtemp(), "dsd.socket") # never created + +client = DogStatsd( + socket_path="unix://" + socket_path, + socket_connect_timeout=2.0, + disable_background_sender=False, + disable_buffering=True, + disable_aggregation=True, + sender_queue_size=n_packets, +) + +if REPLAY_SAFE: + # Exempt from queue expiry -- this is the shape that used to hang stop(). + for i in range(n_packets): + client.gauge_with_timestamp("metric.{}".format(i), 1, timestamp=int(time.time())) +else: + for i in range(n_packets): + client._send_to_server("metric.{}:1|c".format(i)) + +print("queued={} replay_safe={} socket_connect_timeout=2.0".format(n_packets, REPLAY_SAFE)) +print("sender is retrying the head of the queue; stop() must interrupt that rather than wait it out") + +started = time.time() +result = client.stop() +elapsed = time.time() - started + +print("stop({!r}) returned {!r} after {:.2f}s".format(stop_timeout, result, elapsed)) +print("packets_dropped_writer={} bytes_dropped_writer={}".format( + client.packets_dropped_writer, client.bytes_dropped_writer)) +if result and elapsed < 1.0: + print(" -> shutdown was interrupted promptly, as intended") +else: + print(" -> UNEXPECTED: shutdown was not prompt") diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index 3a8a76f9c..314a52ac9 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -1805,7 +1805,7 @@ def test_mixed_batch_respects_max_payload_size_per_buffer(self): # single line breaches the cap, too large and nothing ever overflows. self.statsd.open_buffer() self.statsd.gauge("plain.filler.0", 0) - line_size = self.statsd._buffer_sizes[False] + line_size = self.statsd._buffer_size self.statsd.close_buffer() del sent[:] From ce68cfb45e1ba0a93ed629003196b760e1beb3d8 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 17 Sep 2026 16:55:42 +0100 Subject: [PATCH 16/30] Remove extraneous comment --- datadog/dogstatsd/base.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index 014962cca..61c7bc6fe 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -1167,15 +1167,6 @@ def _reset_buffer(self): # Buffered lines are kept in two separate batches, one per expiry # policy: replay-safe metrics carry their own timestamp, the rest # are stamped on receipt. - # - # HELD AS FOUR PLAIN ATTRIBUTES, not as {False: ..., True: ...} - # dicts. This is the hottest path in the library -- touched once - # per metric, i.e. ~18k/s per client in the benchmark -- and the - # dict form cost a bool() coercion plus a subscript on every - # access. Measured: _send_to_buffer 0.6388us -> 0.7344us per metric - # (+15.0%) for the dict version, which was ~72% of this branch's - # entire per-metric CPU regression against master (+0.132us, - # +7.2%). Attribute access costs nothing extra and reads no worse. self._buffer = [] # type: List[Text] # non-replay-safe self._buffer_rs = [] # type: List[Text] # replay-safe # Running packet size per buffer, each including the newline that From c7226f57fb8c18acaea9127fcce292b731038f29 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 17 Sep 2026 17:04:09 +0100 Subject: [PATCH 17/30] Drop manual and benchmark scripts from the PR These four are development aids, not part of the change under review: tests/manual/emulate_reconnect_then_write_fails.py tests/manual/test_sender_queue_manual.py tests/manual/test_shutdown_bound.py tests/performance/test_sender_queue_benchmark.py They are ad-hoc drivers and a benchmark harness rather than tests the suite runs, and together they accounted for 1278 of the ~1970 added lines under tests/, which buried the unit coverage that does matter. Retained locally via .git/info/exclude (which is not committed) so they stay available for development without shipping in the review. --- .../emulate_reconnect_then_write_fails.py | 234 -------- tests/manual/test_sender_queue_manual.py | 564 ------------------ tests/manual/test_shutdown_bound.py | 70 --- .../test_sender_queue_benchmark.py | 410 ------------- 4 files changed, 1278 deletions(-) delete mode 100644 tests/manual/emulate_reconnect_then_write_fails.py delete mode 100644 tests/manual/test_sender_queue_manual.py delete mode 100644 tests/manual/test_shutdown_bound.py delete mode 100644 tests/performance/test_sender_queue_benchmark.py diff --git a/tests/manual/emulate_reconnect_then_write_fails.py b/tests/manual/emulate_reconnect_then_write_fails.py deleted file mode 100644 index d66695311..000000000 --- a/tests/manual/emulate_reconnect_then_write_fails.py +++ /dev/null @@ -1,234 +0,0 @@ -""" -Emulates: server crashes -> write fails -> client reconnects -> reconnect -succeeds -> the very next write fails again. - -This is a controlled, deterministic emulation (not a probabilistic race): we -puppet the "agent" side explicitly so every step happens in the exact order -described, every time you run this script. - -Two parts: - - 1. `raw_socket_narrative()` - plain UDS SOCK_DGRAM sockets, no datadogpy - involved, just to show the sequence of syscalls/return values in - isolation. - - 2. `through_dogstatsd_client()` - drives the actual - `datadog.dogstatsd.base.DogStatsd` client through the same induced - sequence (by monkeypatching `_get_uds_socket` to kill the "agent" at the - right moments) so you can see how the real retry/backoff code reacts: - does it recover, retry again, or drop the packet. - -Run: - python3 tests/manual/emulate_reconnect_then_write_fails.py -""" -import errno -import os -import socket -import sys -import time - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) - -SOCK_PATH = "/tmp/emulate_reconnect_then_write_fails.sock" - - -def errname(exc): - return errno.errorcode.get(getattr(exc, "errno", None), str(exc)) - - -def _fresh_path(): - try: - os.unlink(SOCK_PATH) - except OSError: - pass - - -def _bind_server(): - """Start a UDS SOCK_DGRAM 'agent' bound at SOCK_PATH.""" - srv = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - srv.bind(SOCK_PATH) - return srv - - -def raw_socket_narrative(): - print("=" * 70) - print("PART 1: raw UDS sockets, step by step") - print("=" * 70) - - _fresh_path() - - # --- Step 0: agent v1 is up, client is connected and happily sending --- - print("\n[step 0] agent v1 starts, client connects") - agent1 = _bind_server() - client = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - client.connect(SOCK_PATH) - client.send(b"metric.a:1|c") - print(" client wrote successfully:", agent1.recv(1024)) - - # --- Step 1: agent crashes --- - print("\n[step 1] agent v1 CRASHES (socket closed, file left on disk)") - agent1.close() - - # --- Step 2: client's next write fails --- - print("[step 2] client writes on the now-dead connection ...") - try: - client.send(b"metric.b:1|c") - print(" unexpectedly succeeded") - except OSError as e: - print(f" write FAILED as expected: {errname(e)} ({e})") - - # --- Step 3: supervisor restarts the agent (agent v2) very quickly --- - print("\n[step 3] supervisor restarts the agent (agent v2 binds at the same path)") - _fresh_path() - agent2 = _bind_server() - - # --- Step 4: client reconnects --- - print("[step 4] client closes its old socket and reconnects") - client.close() - client = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - try: - client.connect(SOCK_PATH) - print(" reconnect SUCCEEDED (agent v2 is genuinely up right now)") - except OSError as e: - print(f" reconnect failed: {errname(e)} ({e}) -- unexpected for this narrative") - return - - # --- Step 5: agent v2 crashes immediately (flaky restart) --- - print("\n[step 5] agent v2 CRASHES immediately, before the client's next write") - agent2.close() - - # --- Step 6: the very next write after the successful reconnect fails too --- - print("[step 6] client writes again, right after the successful reconnect ...") - try: - client.send(b"metric.c:1|c") - print(" unexpectedly succeeded") - except OSError as e: - print(f" write FAILED again: {errname(e)} ({e})") - print("\n >>> proven: a successful reconnect does not guarantee the next write survives <<<") - - client.close() - _fresh_path() - - -def through_dogstatsd_client(): - print("\n" + "=" * 70) - print("PART 2: the same sequence, driven through the real DogStatsd client") - print("=" * 70) - print("_get_uds_socket now makes exactly one connect attempt -- no internal") - print("retry loop. Reconnect-and-retry is the background sender's job now, gated") - print("by socket_connect_retry, and it lives in _sender_main_loop: call") - print("_xmit_packet(..., queue_mode=True), and if it reports a retryable failure") - print("(None), back off and call it again. That's what this drives directly, to") - print("keep watching the same connect/send failures _xmit_packet_attempt sees.") - - from datadog.dogstatsd.base import ( - DogStatsd, - UDS_CONNECT_RETRY_INITIAL_BACKOFF, - UDS_CONNECT_RETRY_MAX_BACKOFF, - ) - - _fresh_path() - agent = {"sock": _bind_server(), "generation": 0} - - statsd = DogStatsd(socket_path=SOCK_PATH, socket_connect_timeout=2.0) - - real_get_uds_socket = DogStatsd._get_uds_socket.__func__ - call_log = [] - state = {"restart_soon_scheduled": False, "post_reconnect_crash_done": False} - - def scripted_get_uds_socket(cls, socket_path, timeout): - """ - Wraps the real (now single-attempt) connect, but scripts the agent's - state around it so the narrative is deterministic regardless of - exactly which attempt number the retry loop below is on when each - event happens: - first call ever: agent is already dead (crashed) before this - connect -- this attempt fails; the retry loop below - (standing in for _sender_main_loop) is what waits for - agent v2 and calls again - first call that actually succeeds: connect succeeds against agent - v2, which we then kill immediately afterwards so the - following send fails again - """ - call_log.append(1) - n = len(call_log) - - if not state["restart_soon_scheduled"]: - state["restart_soon_scheduled"] = True - print("\n[client] first connect attempt -- agent is currently down") - agent["sock"].close() - _fresh_path() - - # Simulate the supervisor's restart happening a little while into - # the retry loop below, so its backoff-and-retry is what catches - # the agent coming back -- not necessarily the very next attempt. - def restart_soon(): - time.sleep(0.15) - _fresh_path() - agent["sock"] = _bind_server() - agent["generation"] = 1 - import threading - threading.Thread(target=restart_soon, daemon=True).start() - - try: - sock = real_get_uds_socket(cls, socket_path, timeout) - except Exception as e: - print(f"[client] connect attempt #{n} failed: {errname(e)} (agent still down)") - raise - - print(f"[client] connect attempt #{n} SUCCEEDED (agent v2 is up)") - if not state["post_reconnect_crash_done"]: - state["post_reconnect_crash_done"] = True - print("[client] ...but agent v2 crashes again immediately, before the send:") - agent["sock"].close() - _fresh_path() - - # Agent v3 comes back shortly after -- this is what lets a *later* - # retry attempt (triggered by the send failure right after this - # connect) actually succeed, so we can see the client ride out - # both failures end to end. - def restart_again(): - time.sleep(0.15) - _fresh_path() - agent["sock"] = _bind_server() - agent["generation"] = 2 - import threading - threading.Thread(target=restart_again, daemon=True).start() - - return sock - - DogStatsd._get_uds_socket = classmethod(scripted_get_uds_socket) - try: - t0 = time.time() - backoff = UDS_CONNECT_RETRY_INITIAL_BACKOFF - attempt = 0 - sent = None # None means "retryable failure, try again" -- see _xmit_packet. - while sent is None: - attempt += 1 - sent = statsd._xmit_packet("emulated.metric:1|c", False, queue_mode=True) - if sent is None: - time.sleep(backoff) - backoff = min(backoff * 2, UDS_CONNECT_RETRY_MAX_BACKOFF) - elapsed = time.time() - t0 - finally: - DogStatsd._get_uds_socket = classmethod(real_get_uds_socket) - try: - agent["sock"].close() - except OSError: - pass - _fresh_path() - - print(f"\n[result] _xmit_packet(..., queue_mode=True) -> sent={sent}, elapsed={elapsed:.3f}s, " - f"retry-loop attempts={attempt}, get_uds_socket calls={len(call_log)}") - print(f"[result] packets_dropped_writer={statsd.packets_dropped_writer}") - if sent: - print(" -> the retry loop (standing in for the background sender) absorbed BOTH") - print(" failures (the initial dead-agent connect AND the post-reconnect send") - print(" failure) and eventually delivered the packet once the agent stayed up.") - else: - print(" -> the packet was dropped.") - - -if __name__ == "__main__": - raw_socket_narrative() - through_dogstatsd_client() diff --git a/tests/manual/test_sender_queue_manual.py b/tests/manual/test_sender_queue_manual.py deleted file mode 100644 index e90ee06f3..000000000 --- a/tests/manual/test_sender_queue_manual.py +++ /dev/null @@ -1,564 +0,0 @@ -""" -Manual, narrated test-drive of `datadog.dogstatsd.sender_queue.SenderQueue`. - -Unlike the unit tests, this script is meant to be *read while it runs*: every -scenario prints what it's about to do, what actually happened, and then -asserts on the outcome. It exercises the queue in isolation first (no -threads, no sockets), then stresses it with real concurrent producers/ -consumers, and finally drives it through the real `DogStatsd` client end to -end (drop-oldest, real-time expiry, and requeue-on-connection-failure). - -Usage: - python3 tests/manual/test_sender_queue_manual.py - -Exits 0 if every check passed, 1 otherwise. -""" -import errno -import os -import socket -import sys -import threading -import time - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) - -from datadog.dogstatsd.sender_queue import ( # noqa: E402 - PendingPayload, - SenderQueue, - Stop, - is_replay_safe, - payload_text, -) -from datadog.util.compat import monotonic # noqa: E402 -import datadog.dogstatsd.base as base_module # noqa: E402 -from datadog.dogstatsd.base import DogStatsd # noqa: E402 - - -# -------------------------------------------------------------------------- -# Small helpers: narration + a pass/fail ledger shared across every scenario. -# -------------------------------------------------------------------------- -_RESULTS = {"pass": 0, "fail": 0, "failures": []} - - -def section(title): - print() - print("=" * 78) - print(title) - print("=" * 78) - - -def note(msg): - print(" . {}".format(msg)) - - -def check(condition, description): - if condition: - _RESULTS["pass"] += 1 - print(" [PASS] {}".format(description)) - else: - _RESULTS["fail"] += 1 - _RESULTS["failures"].append(description) - print(" [FAIL] {}".format(description)) - return condition - - -def recorder(label): - """A drop callback that remembers every item it was called with and prints it.""" - events = [] - - def _on_drop(item): - events.append(item) - rs = is_replay_safe(item) - # Replay-safe entries are bare strings with no enqueued_at to age. - age = "n/a" if rs else "{:.3f}s".format(monotonic() - item.enqueued_at) - print( - " -> {label} fired: payload={payload!r} age={age} replay_safe={rs}".format( - label=label, payload=payload_text(item), age=age, rs=rs - ) - ) - - _on_drop.events = events - return _on_drop - - -def new_recorders(): - return recorder("on_drop_queue_full"), recorder("on_drop_expired") - - -def quiet_recorder(label, report_every=100): - """Like recorder(), but only prints a running count every `report_every` - hits instead of one line per drop -- for scenarios with hundreds of them, - where line-per-event narration would just be noise.""" - events = [] - - def _on_drop(item): - events.append(item) - if len(events) % report_every == 0: - print(" -> {label}: {count} drops so far (e.g. {sample!r})".format( - label=label, count=len(events), sample=payload_text(item).strip() - )) - - _on_drop.events = events - return _on_drop - - -def payload(text, replay_safe=False, enqueued_at=None): - """Build a queue entry: a bare string when replay-safe, else a wrapper.""" - if replay_safe: - return text - return PendingPayload(text, enqueued_at if enqueued_at is not None else monotonic()) - - -# -------------------------------------------------------------------------- -# Scenario 1: plain FIFO, no eviction, no expiry. -# -------------------------------------------------------------------------- -def scenario_1_fifo_order(): - section("1. Basic FIFO ordering (unbounded, nothing expires)") - drop_full, drop_expired = new_recorders() - q = SenderQueue(maxsize=0, expiry_seconds=100.0, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) - - for name in ("a", "b", "c"): - q.put(payload(name + "\n")) - note("put({!r}) -> qsize={}".format(name, q.qsize())) - - order = [payload_text(q.get()) for _ in range(3)] - note("got, in order: {}".format(order)) - - check(order == ["a\n", "b\n", "c\n"], "FIFO order preserved") - check(not drop_full.events and not drop_expired.events, "no drops during a plain unbounded run") - check(q.qsize() == 0, "queue fully drained") - - -# -------------------------------------------------------------------------- -# Scenario 2: overflow evicts the oldest entry to make room. -# -------------------------------------------------------------------------- -def scenario_2_overflow_drops_oldest(): - section("2. Overflow: the oldest entry is evicted to make room") - drop_full, drop_expired = new_recorders() - q = SenderQueue(maxsize=2, expiry_seconds=100.0, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) - - q.put(payload("first\n")) - note("put('first') -> qsize={}".format(q.qsize())) - q.put(payload("second\n")) - note("put('second') -> qsize={} (queue is now at maxsize=2)".format(q.qsize())) - note("putting a third payload...") - q.put(payload("third\n")) - note("put('third') -> qsize={}".format(q.qsize())) - - check(q.qsize() == 2, "queue never grew past maxsize") - check([payload_text(e) for e in drop_full.events] == ["first\n"], "the OLDEST entry was evicted via on_drop_queue_full") - check(not drop_expired.events, "nothing was expired -- this was a pure capacity eviction") - - remaining = [payload_text(q.get()) for _ in range(2)] - note("remaining, in order: {}".format(remaining)) - check(remaining == ["second\n", "third\n"], "the two newest survivors come out in FIFO order") - - -# -------------------------------------------------------------------------- -# Scenario 3: an evicted oldest entry that's ALSO stale is attributed to expiry. -# -------------------------------------------------------------------------- -def scenario_3_overflow_prefers_expiry_reason(): - section("3. Overflow where the evicted entry is ALSO already stale") - drop_full, drop_expired = new_recorders() - q = SenderQueue(maxsize=1, expiry_seconds=0.2, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) - - q.put(payload("stale\n")) - note("put('stale'); sleeping 0.25s so it ages past the 0.2s expiry window...") - time.sleep(0.25) - - note("queue is at maxsize=1; putting a fresh payload now forces an eviction") - q.put(payload("fresh\n")) - - check(not drop_full.events, "NOT counted as a plain capacity drop") - check([payload_text(e) for e in drop_expired.events] == ["stale\n"], "counted as an EXPIRY drop instead -- more informative") - check(payload_text(q.get()) == "fresh\n", "the fresh payload survived") - - -# -------------------------------------------------------------------------- -# Scenario 4: get() lazily drains every stale entry at the front. -# -------------------------------------------------------------------------- -def scenario_4_get_drains_stale_entries(): - section("4. get() drains every stale entry at the front before returning") - drop_full, drop_expired = new_recorders() - q = SenderQueue(maxsize=0, expiry_seconds=0.2, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) - - q.put(payload("stale-1\n")) - q.put(payload("stale-2\n")) - note("put two payloads; sleeping 0.25s so both age past the 0.2s expiry window...") - time.sleep(0.25) - q.put(payload("fresh\n")) - note("qsize before get(): {}".format(q.qsize())) - - item = q.get() - note("qsize after get(): {}".format(q.qsize())) - - check(payload_text(item) == "fresh\n", "get() skipped both stale entries and returned the fresh one") - check([payload_text(e) for e in drop_expired.events] == ["stale-1\n", "stale-2\n"], "both stale entries dropped, in order, along the way") - - -# -------------------------------------------------------------------------- -# Scenario 5: replay_safe payloads never expire. -# -------------------------------------------------------------------------- -def scenario_5_replay_safe_never_expires(): - section("5. replay_safe=True payloads are exempt from expiry") - drop_full, drop_expired = new_recorders() - q = SenderQueue(maxsize=0, expiry_seconds=0.1, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) - - q.put(payload("timestamped\n", replay_safe=True)) - note("put a replay_safe payload; sleeping 0.3s, well past the 0.1s expiry window...") - time.sleep(0.3) - - item = q.get() - note("get() returned: {!r}".format(payload_text(item) if item is not None else None)) - check(item is not None and payload_text(item) == "timestamped\n", "still returned by get(), never dropped") - check(not drop_expired.events, "on_drop_expired was never called for it") - - -# -------------------------------------------------------------------------- -# Scenario 6: requeue_front() -- the three outcomes. -# -------------------------------------------------------------------------- -def scenario_6a_requeue_front_with_room(): - section("6a. requeue_front(): succeeds when there's room, rejoins at the FRONT") - drop_full, drop_expired = new_recorders() - q = SenderQueue(maxsize=2, expiry_seconds=100.0, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) - - q.put(payload("in-flight\n")) - in_flight = q.get() - note("sender thread picked up 'in-flight' and (simulated) failed to send it") - q.requeue_front(in_flight) - note("requeued 'in-flight'; qsize={}".format(q.qsize())) - q.put(payload("new\n")) - note("put 'new'; qsize={}".format(q.qsize())) - - order = [payload_text(q.get()) for _ in range(2)] - note("got, in order: {}".format(order)) - check(order == ["in-flight\n", "new\n"], "the requeued item is retried BEFORE the newer one") - check(not drop_full.events and not drop_expired.events, "nothing was dropped") - - -def scenario_6b_requeue_front_drops_when_full(): - section("6b. requeue_front(): dropped via on_drop_queue_full when the queue is already full") - drop_full, drop_expired = new_recorders() - q = SenderQueue(maxsize=1, expiry_seconds=100.0, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) - - q.put(payload("in-flight\n")) - in_flight = q.get() - note("sender thread picked up 'in-flight'; meanwhile a fresh payload fills the now-empty slot") - q.put(payload("new\n")) - note("qsize is already at maxsize=1; the failed send now tries to requeue 'in-flight'...") - q.requeue_front(in_flight) - - check(q.qsize() == 1, "queue never grew past maxsize") - check([payload_text(e) for e in drop_full.events] == ["in-flight\n"], "'in-flight' was dropped via on_drop_queue_full") - check(payload_text(q.get()) == "new\n", "'new' is untouched and still gets sent") - - -def scenario_6c_requeue_front_drops_when_expired(): - section("6c. requeue_front(): dropped via on_drop_expired when it went stale in flight, even with room to spare") - drop_full, drop_expired = new_recorders() - q = SenderQueue(maxsize=0, expiry_seconds=0.15, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) - - q.put(payload("slow-send\n")) - picked_up = q.get() - note("sender thread picked up 'slow-send' and is (simulated) stuck retrying the connection...") - time.sleep(0.2) - note("...0.2s later the send finally fails; the queue is unbounded, so there is plenty of room") - q.requeue_front(picked_up) - - check(q.qsize() == 0, "NOT requeued despite there being room -- expiry wins") - check([payload_text(e) for e in drop_expired.events] == ["slow-send\n"], "dropped via on_drop_expired") - - -# -------------------------------------------------------------------------- -# Scenario 7: _unfinished_tasks / task_done() / join() bookkeeping. -# -------------------------------------------------------------------------- -def scenario_7_task_done_and_join(): - section("7. _unfinished_tasks bookkeeping across put()/task_done()/join()") - drop_full, drop_expired = new_recorders() - q = SenderQueue(maxsize=0, expiry_seconds=100.0, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) - - for name in ("x", "y", "z"): - q.put(payload(name + "\n")) - note("put 3 payloads -> _unfinished_tasks={}".format(q._unfinished_tasks)) - check(q._unfinished_tasks == 3, "one unfinished task recorded per put()") - - joined = {"done": False} - - def joiner(): - q.join() - joined["done"] = True - - t = threading.Thread(target=joiner) - t.start() - time.sleep(0.1) - note("join() called on a background thread; joined={}".format(joined["done"])) - check(not joined["done"], "join() is still blocked -- 3 tasks are still outstanding") - - for _ in range(3): - item = q.get() - note("get() -> {!r}, calling task_done()".format(payload_text(item))) - q.task_done() - - t.join(timeout=2) - note("after 3x task_done(): joined={} _unfinished_tasks={}".format(joined["done"], q._unfinished_tasks)) - check(joined["done"], "join() returned once every task was marked done") - check(q._unfinished_tasks == 0, "_unfinished_tasks back to zero") - - -# -------------------------------------------------------------------------- -# Scenario 8: the Stop sentinel bypasses capacity and expiry entirely. -# -------------------------------------------------------------------------- -def scenario_8_stop_sentinel(): - section("8. The Stop sentinel bypasses capacity limits and expiry entirely") - drop_full, drop_expired = new_recorders() - q = SenderQueue(maxsize=1, expiry_seconds=100.0, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) - - q.put(payload("only-slot\n")) - note("queue is at maxsize=1; putting Stop now...") - q.put(Stop) - note("qsize={} (Stop did not evict 'only-slot', nor was it evicted itself)".format(q.qsize())) - - check(q.qsize() == 2, "Stop was appended past maxsize instead of triggering eviction") - - first = q.get() - second = q.get() - check(payload_text(first) == "only-slot\n", "the real payload still comes out first (FIFO)") - check(second is Stop, "Stop comes out exactly as put in, untouched by drop logic") - q.task_done() - q.task_done() - - -# -------------------------------------------------------------------------- -# Scenario 9: concurrency smoke test -- multiple producers racing a consumer. -# -------------------------------------------------------------------------- -def scenario_9_concurrency_smoke_test(): - section("9. Concurrency smoke test: 4 producer threads racing 1 consumer thread") - note("this scenario can trigger hundreds of evictions -- using quiet_recorder() to summarize instead of narrating every one") - drop_full, drop_expired = quiet_recorder("on_drop_queue_full"), quiet_recorder("on_drop_expired") - q = SenderQueue(maxsize=20, expiry_seconds=5.0, on_drop_queue_full=drop_full, on_drop_expired=drop_expired) - - n_per_producer = 200 - n_producers = 4 - total = n_per_producer * n_producers - received = [] - - def producer(pid): - for i in range(n_per_producer): - q.put(payload("p{}-{}\n".format(pid, i))) - - def consumer(): - while True: - item = q.get() - if item is Stop: - q.task_done() - return - received.append(item) - q.task_done() - # Artificial slowness so the 4 producers reliably outrun the - # consumer and we actually get to see maxsize=20 evictions - # happen, instead of everything just being consumed in time. - time.sleep(0.0005) - - note("maxsize={} expiry_seconds={} total_payloads={}".format(q._maxsize, q._expiry_seconds, total)) - producers = [threading.Thread(target=producer, args=(pid,)) for pid in range(n_producers)] - consumer_thread = threading.Thread(target=consumer) - - t0 = time.time() - consumer_thread.start() - for p in producers: - p.start() - for p in producers: - p.join() - note("all {} producers finished putting {} payloads in {:.3f}s".format(n_producers, total, time.time() - t0)) - - q.put(Stop) - consumer_thread.join(timeout=10) - - accounted = len(received) + len(drop_full.events) + len(drop_expired.events) - note( - "received={} dropped_full={} dropped_expired={} accounted_total={} expected_total={}".format( - len(received), len(drop_full.events), len(drop_expired.events), accounted, total - ) - ) - check(not consumer_thread.is_alive(), "consumer thread exited cleanly after Stop") - check(accounted == total, "every put() payload is accounted for exactly once (sent, capacity-dropped, or expired)") - check(q.qsize() == 0, "queue fully drained") - check(q._unfinished_tasks == 0, "_unfinished_tasks settled back to zero under real concurrency") - check(len(drop_full.events) > 0, "the slow consumer + maxsize=20 did trigger at least one real eviction") - - -# -------------------------------------------------------------------------- -# Scenario 10: end-to-end through the real DogStatsd client. -# -------------------------------------------------------------------------- -class ScriptedSocket(object): - """Just enough of a socket to drive DogStatsd's send path without a real network.""" - - family = socket.AF_UNIX - - def __init__(self): - self.received = [] - - def send(self, data): - self.received.append(data) - return len(data) - - def sendall(self, data): - self.received.append(data) - - def settimeout(self, *_args): - pass - - def getsockopt(self, *_args): - return socket.SOCK_DGRAM - - def close(self): - pass - - -def scenario_10a_client_drop_oldest(): - section("10a. End-to-end through DogStatsd: drop-oldest via the real client") - # No background sender thread on purpose. With one running it races this - # sequence: if it drains 'first' before 'second' is queued, the queue is - # never full, nothing is evicted, and the checks below describe a schedule - # that didn't happen (this scenario failed ~7 runs in 10 that way). Build - # the queue by hand -- the same wiring _start_sender_thread() uses -- then - # drive the real sender loop synchronously at the end. - statsd = DogStatsd(disable_background_sender=True, disable_telemetry=True) - statsd.socket = ScriptedSocket() - statsd._queue = SenderQueue( - 1, - base_module.PENDING_PAYLOAD_EXPIRY_SECONDS, - statsd._account_dropped_queue_full, - statsd._account_dropped_expired, - ) - - statsd._send_to_server("first") - note("_send_to_server('first') -> qsize={}".format(statsd._queue.qsize())) - statsd._send_to_server("second") - note("_send_to_server('second') -> qsize={}".format(statsd._queue.qsize())) - note("bytes_dropped_queue={} packets_dropped_queue={}".format(statsd.bytes_dropped_queue, statsd.packets_dropped_queue)) - - check(statsd.packets_dropped_queue == 1, "one packet ('first') was dropped for capacity") - - # Drain through the real sender loop; Stop makes it return once done. - statsd._queue.put(Stop) - statsd._sender_main_loop(statsd._queue) - note("socket received: {}".format(statsd.socket.received)) - check(statsd.socket.received == [b"second\n"], "only the surviving (newest) payload was actually sent") - - -def scenario_10b_client_real_time_expiry(): - section("10b. End-to-end through DogStatsd: real-time expiry with no reachable agent") - # NOTE: base.py does `from datadog.dogstatsd.sender_queue import ... PENDING_PAYLOAD_EXPIRY_SECONDS`, - # which binds its OWN name in base's namespace at import time. Patching - # sender_queue.PENDING_PAYLOAD_EXPIRY_SECONDS after that has no effect on - # _start_sender_thread(), which reads base's copy of the name -- so that's - # the one that has to be patched here. - original_expiry = base_module.PENDING_PAYLOAD_EXPIRY_SECONDS - base_module.PENDING_PAYLOAD_EXPIRY_SECONDS = 0.3 - note("patched datadog.dogstatsd.base.PENDING_PAYLOAD_EXPIRY_SECONDS: {} -> {}".format(original_expiry, base_module.PENDING_PAYLOAD_EXPIRY_SECONDS)) - - statsd = None - try: - statsd = DogStatsd( - socket_path="/tmp/sender-queue-manual-test-nonexistent-{}.sock".format(os.getpid()), - socket_connect_timeout=0.05, - disable_background_sender=False, - disable_telemetry=True, - ) - statsd.increment("will.expire") - note("queued 'will.expire' against a socket path that doesn't exist; waiting for wait_for_pending()...") - - t0 = time.time() - statsd.wait_for_pending() - elapsed = time.time() - t0 - note("wait_for_pending() returned after {:.3f}s".format(elapsed)) - - check(statsd.packets_dropped_expired == 1, "the packet expired instead of being retried forever") - check(statsd.packets_dropped_writer == 0, "it was NOT mistaken for a hard write failure") - check(elapsed < 5.0, "expiry actually bounded how long wait_for_pending() took") - finally: - base_module.PENDING_PAYLOAD_EXPIRY_SECONDS = original_expiry - note("restored PENDING_PAYLOAD_EXPIRY_SECONDS to {}".format(original_expiry)) - if statsd is not None: - statsd.stop() - - -def scenario_10c_client_requeue_then_succeeds(): - section("10c. End-to-end through DogStatsd: requeue-and-retry survives a flaky reconnect, then succeeds") - working_socket = ScriptedSocket() - attempts = {"count": 0} - fail_until = 4 - - def flaky_get_uds_socket(_cls, _socket_path, _timeout, _connect_timeout): - attempts["count"] += 1 - if attempts["count"] < fail_until: - note("reconnect attempt #{}: still refused".format(attempts["count"])) - raise socket.error(errno.ECONNREFUSED, "still refused") - note("reconnect attempt #{}: agent is back up".format(attempts["count"])) - return working_socket - - real_get_uds_socket = DogStatsd._get_uds_socket - DogStatsd._get_uds_socket = classmethod(flaky_get_uds_socket) - statsd = None - try: - statsd = DogStatsd( - socket_path="/tmp/sender-queue-manual-test-flaky-{}.sock".format(os.getpid()), - socket_connect_timeout=0.05, - disable_background_sender=False, - disable_telemetry=True, - ) - statsd.gauge("eventually.sent", 1) - t0 = time.time() - statsd.wait_for_pending() - elapsed = time.time() - t0 - finally: - DogStatsd._get_uds_socket = real_get_uds_socket - - note("wait_for_pending() returned after {:.3f}s and {} reconnect attempts".format(elapsed, attempts["count"])) - check(attempts["count"] >= fail_until, "it took multiple reconnect attempts, exercising requeue-and-retry") - check(statsd.packets_dropped_writer == 0, "never hard-dropped as a write failure") - check(statsd.packets_dropped_expired == 0, "never expired -- it succeeded well before the TTL") - check( - bool(working_socket.received) and working_socket.received[0].startswith(b"eventually.sent:1|g"), - "the packet was actually delivered once the agent came back", - ) - - statsd.stop() - - -# -------------------------------------------------------------------------- -def main(): - scenarios = [ - scenario_1_fifo_order, - scenario_2_overflow_drops_oldest, - scenario_3_overflow_prefers_expiry_reason, - scenario_4_get_drains_stale_entries, - scenario_5_replay_safe_never_expires, - scenario_6a_requeue_front_with_room, - scenario_6b_requeue_front_drops_when_full, - scenario_6c_requeue_front_drops_when_expired, - scenario_7_task_done_and_join, - scenario_8_stop_sentinel, - scenario_9_concurrency_smoke_test, - scenario_10a_client_drop_oldest, - scenario_10b_client_real_time_expiry, - scenario_10c_client_requeue_then_succeeds, - ] - - start = time.time() - for scenario in scenarios: - scenario() - - section("SUMMARY") - print(" {} passed, {} failed, {:.2f}s total".format(_RESULTS["pass"], _RESULTS["fail"], time.time() - start)) - if _RESULTS["failures"]: - print(" Failed checks:") - for description in _RESULTS["failures"]: - print(" - {}".format(description)) - - sys.exit(1 if _RESULTS["fail"] else 0) - - -if __name__ == "__main__": - main() diff --git a/tests/manual/test_shutdown_bound.py b/tests/manual/test_shutdown_bound.py deleted file mode 100644 index 11f90fd52..000000000 --- a/tests/manual/test_shutdown_bound.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Manual check that shutdown stays bounded while the sender is retrying. - -Usage: python tests/manual/test_shutdown_bound.py [n_packets] [stop_timeout] - -Fills the background sender queue while the agent is unreachable, with -socket_connect_timeout=2.0 so the sender keeps retrying the head-of-queue -payload indefinitely, backing off up to UDS_CONNECT_RETRY_MAX_BACKOFF (a minute) -between attempts. - -Two things used to make stop() drag or hang here, both fixed: - - * The backoff was a plain time.sleep(), so stop() had to wait out whatever - was left of it -- up to a minute. - * requeue_front() puts the failed payload back at the *front* of the queue, - ahead of the Stop sentinel, so the sender only ever noticed Stop once the - head payload was finally resolved. Ordinary payloads resolve via the - queue's expiry, but replay-safe ones (gauge_with_timestamp, events with - date_happened, service checks with a timestamp) are exempt from expiry -- - so one of those at the head starved Stop forever and stop() never - returned at all. - -Both are now handled by a stopping Event that the sender waits on instead of -sleeping, so expect stop() to return promptly and report True regardless of -what is queued. Pass a replay-safe metric through (see REPLAY_SAFE below) to -exercise the case that used to hang. -""" -import os -import sys -import tempfile -import time - -from datadog.dogstatsd.base import DogStatsd - -n_packets = int(sys.argv[1]) if len(sys.argv) > 1 else 50000 -stop_timeout = float(sys.argv[2]) if len(sys.argv) > 2 else 5.0 -REPLAY_SAFE = os.environ.get("REPLAY_SAFE") == "1" - -socket_path = os.path.join(tempfile.mkdtemp(), "dsd.socket") # never created - -client = DogStatsd( - socket_path="unix://" + socket_path, - socket_connect_timeout=2.0, - disable_background_sender=False, - disable_buffering=True, - disable_aggregation=True, - sender_queue_size=n_packets, -) - -if REPLAY_SAFE: - # Exempt from queue expiry -- this is the shape that used to hang stop(). - for i in range(n_packets): - client.gauge_with_timestamp("metric.{}".format(i), 1, timestamp=int(time.time())) -else: - for i in range(n_packets): - client._send_to_server("metric.{}:1|c".format(i)) - -print("queued={} replay_safe={} socket_connect_timeout=2.0".format(n_packets, REPLAY_SAFE)) -print("sender is retrying the head of the queue; stop() must interrupt that rather than wait it out") - -started = time.time() -result = client.stop() -elapsed = time.time() - started - -print("stop({!r}) returned {!r} after {:.2f}s".format(stop_timeout, result, elapsed)) -print("packets_dropped_writer={} bytes_dropped_writer={}".format( - client.packets_dropped_writer, client.bytes_dropped_writer)) -if result and elapsed < 1.0: - print(" -> shutdown was interrupted promptly, as intended") -else: - print(" -> UNEXPECTED: shutdown was not prompt") diff --git a/tests/performance/test_sender_queue_benchmark.py b/tests/performance/test_sender_queue_benchmark.py deleted file mode 100644 index b24561781..000000000 --- a/tests/performance/test_sender_queue_benchmark.py +++ /dev/null @@ -1,410 +0,0 @@ -""" -Microbenchmark: SenderQueue vs. stdlib queue.Queue. - -This isolates just the hand-off queue's own overhead -- no sockets, no -network variance -- because that's the piece that changed when the -background sender moved off queue.Queue. put() runs synchronously on every -metric emission's calling thread (the application's hot path), so its -*latency* matters at least as much as raw throughput; get() runs on the -background sender thread. - -queue.Queue is used as the baseline throughout via a thin adapter -(_OldStyleQueueAdapter) that reproduces the OLD behavior being replaced: -put_nowait() and drop-with-a-counter on queue.Full, get()+task_done() to -drain. That's the fairest apples-to-apples comparison, since it's literally -what SenderQueue's put()/get() replaced. - -Scenarios: - 1. Unbounded put() then get(), single-threaded (best case for both -- - no eviction, no contention). - 2. Bounded queue, kept permanently full: every put() forces an eviction - for SenderQueue, vs an immediate reject-with-exception for - queue.Queue. This is the main new cost the redesign introduces. - 3. A single put() that has to walk past a large backlog of already-EXPIRED - entries at the front (SenderQueue's opportunistic-cleanup loop has no - upper bound tied to "just free one slot" -- it clears every consecutive - stale entry it finds). Reports cost as a function of backlog size, to - surface whether this can spike a calling thread's latency. - 4. Producer/consumer concurrency: N producer threads hammering put() while - 1 consumer thread drains, measuring achieved producer throughput and - put() latency percentiles under real lock contention. - 5. Per-item memory footprint: PendingPayload wrapper vs a bare str. - -Usage: - python3 tests/performance/test_sender_queue_benchmark.py [--quick] - - --quick shrinks every N so it finishes in a few seconds (CI-friendly); - default sizes are big enough to get low-noise numbers on a quiet - machine. - -This prints numbers and interpretation guidance; it does not hard-fail on -absolute thresholds (those are too hardware/noise dependent to gate CI on -reliably). The one thing it does assert on is the *shape* of the eviction -cost in scenario 3 -- that it's linear in backlog size, not something worse. -Read the printed numbers yourself before/after a change and compare. -""" -import os -import sys -import threading -import time - -try: - import queue as stdlib_queue -except ImportError: - import Queue as stdlib_queue # type: ignore[no-redef] - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) - -from datadog.dogstatsd.sender_queue import ( # noqa: E402 - PendingPayload, - SenderQueue, -) -from datadog.util.compat import monotonic # noqa: E402 - -QUICK = "--quick" in sys.argv - - -def section(title): - print() - print("=" * 78) - print(title) - print("=" * 78) - - -def note(msg): - print(" . {}".format(msg)) - - -PACKET = "some.metric.name:1|c|#tag1:val1,tag2:val2\n" - - -# -------------------------------------------------------------------------- -# Baseline adapter: reproduces the OLD (pre-SenderQueue) put/get contract on -# top of stdlib queue.Queue, so scenario code can treat both implementations -# uniformly. -# -------------------------------------------------------------------------- -class _OldStyleQueueAdapter(object): - def __init__(self, maxsize): - self._q = stdlib_queue.Queue(maxsize) - self.dropped = 0 - - def put(self, item): - try: - self._q.put_nowait(item) - except stdlib_queue.Full: - self.dropped += 1 - - def get(self): - return self._q.get() - - def task_done(self): - self._q.task_done() - - def qsize(self): - return self._q.qsize() - - -def make_sender_queue(maxsize, expiry_seconds=3600.0): - drops = {"full": 0, "expired": 0} - - def on_full(_item): - drops["full"] += 1 - - def on_expired(_item): - drops["expired"] += 1 - - q = SenderQueue(maxsize, expiry_seconds, on_full, on_expired) - q.drops = drops - return q - - -def percentiles(samples_us): - samples_us = sorted(samples_us) - n = len(samples_us) - - def pct(p): - idx = min(n - 1, int(n * p)) - return samples_us[idx] - - return { - "p50": pct(0.50), - "p90": pct(0.90), - "p99": pct(0.99), - "max": samples_us[-1], - } - - -def time_puts(put_fn, n): - """Time n individual put() calls, returning (total_seconds, [latency_us, ...]).""" - samples = [0.0] * n - t_start = time.perf_counter() - for i in range(n): - t0 = time.perf_counter() - put_fn() - samples[i] = (time.perf_counter() - t0) * 1e6 - total = time.perf_counter() - t_start - return total, samples - - -def report(label, n, total_seconds, samples_us): - p = percentiles(samples_us) - print( - " {:<28s} ops/sec={:>10,.0f} p50={:>7.3f}us p90={:>7.3f}us p99={:>7.3f}us max={:>9.3f}us".format( - label, n / total_seconds, p["p50"], p["p90"], p["p99"], p["max"] - ) - ) - return p - - -# -------------------------------------------------------------------------- -# Scenario 1: unbounded, no contention, no eviction. -# -------------------------------------------------------------------------- -def scenario_1_unbounded_single_threaded(): - section("1. Unbounded put()/get(), single-threaded (best case, no eviction)") - n = 20000 if not QUICK else 2000 - - old = _OldStyleQueueAdapter(maxsize=0) - total, samples = time_puts(lambda: old.put(PACKET), n) - report("queue.Queue (baseline)", n, total, samples) - for _ in range(n): - old.get() - old.task_done() - - new = make_sender_queue(maxsize=0) - total, samples = time_puts(lambda: new.put(PendingPayload(PACKET, monotonic())), n) - new_p = report("SenderQueue", n, total, samples) - for _ in range(n): - new.get() - new.task_done() - - note("SenderQueue p99 put() latency: {:.3f}us for {:,} plain puts with headroom to spare".format(new_p["p99"], n)) - - -# -------------------------------------------------------------------------- -# Scenario 2: bounded queue, kept permanently full -- every put() evicts. -# -------------------------------------------------------------------------- -def scenario_2_sustained_overflow(): - section("2. Bounded queue kept permanently full: every put() forces eviction (new cost)") - n = 20000 if not QUICK else 2000 - maxsize = 8 - - old = _OldStyleQueueAdapter(maxsize=maxsize) - for _ in range(maxsize): - old.put(PACKET) - total, samples = time_puts(lambda: old.put(PACKET), n) - old_p = report("queue.Queue (baseline)", n, total, samples) - note("queue.Queue just rejects with an exception when full -- O(1), no eviction work at all") - - new = make_sender_queue(maxsize=maxsize) - for _ in range(maxsize): - new.put(PendingPayload(PACKET, monotonic())) - total, samples = time_puts(lambda: new.put(PendingPayload(PACKET, monotonic())), n) - new_p = report("SenderQueue", n, total, samples) - - ratio = new_p["p99"] / old_p["p99"] if old_p["p99"] else float("inf") - note("SenderQueue's drop-oldest-and-evict costs {:.1f}x queue.Queue's reject-with-exception at p99".format(ratio)) - note("(each put() here evicts exactly one item -- the mandatory oldest -- since nothing is expired)") - - -# -------------------------------------------------------------------------- -# Scenario 3: one put() that has to walk past a large expired backlog. -# -------------------------------------------------------------------------- -def scenario_3_large_expired_backlog(): - section("3. Cost of ONE put() as a function of an already-expired backlog size") - note("SenderQueue's opportunistic cleanup has no cap tied to 'free just one slot': it clears") - note("every consecutive stale entry at the front. This measures whether that can spike latency.") - - backlog_sizes = [1, 10, 100, 1000, 5000] if not QUICK else [1, 10, 100] - results = [] - for backlog in backlog_sizes: - # expiry_seconds=0 with a backdated enqueued_at makes every backlog - # entry expired the instant it's queued. maxsize=backlog (exactly - # full) so the next put() below is what actually triggers eviction. - q = make_sender_queue(maxsize=backlog, expiry_seconds=0.0) - stale_at = monotonic() - 1000.0 - for _ in range(backlog): - q.put(PendingPayload(PACKET, stale_at)) - - t0 = time.perf_counter() - q.put(PendingPayload(PACKET, monotonic())) - elapsed_us = (time.perf_counter() - t0) * 1e6 - - note("backlog={:>5d} stale entries -> single put() took {:>9.3f}us, evicted {:d}".format( - backlog, elapsed_us, q.drops["expired"] + q.drops["full"] - )) - results.append((backlog, elapsed_us)) - - # Sanity check on the *shape*: cost should scale roughly linearly with - # backlog size, not blow up super-linearly. Compare the per-entry cost - # at the smallest and largest backlog sizes; allow a generous margin for - # fixed overhead and noise, but a large deviation would indicate a real - # algorithmic problem worth investigating. - (small_n, small_us), (large_n, large_us) = results[1], results[-1] - small_per_entry = small_us / small_n - large_per_entry = large_us / large_n - ratio = large_per_entry / small_per_entry if small_per_entry else float("inf") - note( - "per-entry eviction cost: {:.3f}us/entry at backlog={} vs {:.3f}us/entry at backlog={} (ratio={:.2f}x)".format( - small_per_entry, small_n, large_per_entry, large_n, ratio - ) - ) - if ratio > 5.0: - print(" [WARN] per-entry eviction cost grew by {:.1f}x from a small to a large backlog".format(ratio)) - print(" -- that's worse than linear; investigate before shipping.") - else: - print(" [OK] per-entry eviction cost stayed roughly flat as backlog size grew (linear, as expected)") - note("takeaway: a single put() CAN take noticeably longer if a huge stale backlog piles up (e.g. a") - note("long outage with a very large sender_queue_size). Keep sender_queue_size sized to what you're") - note("actually willing to let one put() walk through in the worst case.") - - -# -------------------------------------------------------------------------- -# Scenario 4: producer/consumer concurrency. -# -------------------------------------------------------------------------- -def _run_concurrent(put_fn, get_and_ack_fn, n_producers, n_per_producer, duration_cap=15.0): - latencies = [] - latencies_lock = threading.Lock() - stop = threading.Event() - - def producer(): - local_latencies = [] - for _ in range(n_per_producer): - t0 = time.perf_counter() - put_fn() - local_latencies.append((time.perf_counter() - t0) * 1e6) - with latencies_lock: - latencies.extend(local_latencies) - - def consumer(): - while not stop.is_set(): - get_and_ack_fn() - - consumer_thread = threading.Thread(target=consumer) - consumer_thread.daemon = True - consumer_thread.start() - - producers = [threading.Thread(target=producer) for _ in range(n_producers)] - t0 = time.perf_counter() - for p in producers: - p.start() - for p in producers: - p.join(timeout=duration_cap) - elapsed = time.perf_counter() - t0 - stop.set() - - total_ops = n_producers * n_per_producer - return elapsed, total_ops, latencies - - -def scenario_4_concurrency(): - section("4. Producer/consumer concurrency: N producers hammering put(), 1 consumer draining") - n_producers = 4 - n_per_producer = 5000 if not QUICK else 500 - - old = _OldStyleQueueAdapter(maxsize=1000) - elapsed, total_ops, samples = _run_concurrent( - lambda: old.put(PACKET), - lambda: (old.get(), old.task_done()), - n_producers, - n_per_producer, - ) - old_p = report("queue.Queue (baseline)", total_ops, elapsed, samples) - note("queue.Queue: {} producers x {} puts in {:.3f}s, {} dropped-on-full".format( - n_producers, n_per_producer, elapsed, old.dropped - )) - - new = make_sender_queue(maxsize=1000) - elapsed, total_ops, samples = _run_concurrent( - lambda: new.put(PendingPayload(PACKET, monotonic())), - lambda: (new.get(), new.task_done()), - n_producers, - n_per_producer, - ) - new_p = report("SenderQueue", total_ops, elapsed, samples) - note("SenderQueue: {} producers x {} puts in {:.3f}s, {} dropped-full, {} dropped-expired".format( - n_producers, n_per_producer, elapsed, new.drops["full"], new.drops["expired"] - )) - - ratio_p99 = new_p["p99"] / old_p["p99"] if old_p["p99"] else float("inf") - note("under real thread contention, SenderQueue's p99 put() latency is {:.2f}x queue.Queue's".format(ratio_p99)) - - -# -------------------------------------------------------------------------- -# Scenario 5: per-item memory footprint. -# -------------------------------------------------------------------------- -def scenario_5_memory_footprint(): - section("5. Per-item memory footprint: bare str (replay-safe) vs PendingPayload (expiring)") - note("sys.getsizeof() is shallow: PendingPayload holds a *reference* to the payload string,") - note("not a copy, so its own size doesn't include the string's bytes. The string is shared") - note("either way, so the real per-item cost is the wrapper object plus its enqueued_at float.") - - payload_str = PACKET - str_size = sys.getsizeof(payload_str) - wrapper_size = sys.getsizeof(PendingPayload(payload_str, monotonic())) - float_size = sys.getsizeof(monotonic()) - - note("payload str (shared either way): {} bytes".format(str_size)) - note("PendingPayload wrapper itself (__slots__, no __dict__): {} bytes".format(wrapper_size)) - print() - - note("Replay-safe payloads (gauge_with_timestamp, event(date_happened=...),") - note("service_check(timestamp=...)) never expire, so they carry no enqueued_at and are queued") - note("as the BARE STRING -- SenderQueue infers replay-safety from the entry's type. That means") - note("zero wrapper overhead for them, not merely a skipped float:") - note(" replay-safe entry: {} bytes (just the shared str) -> +0 bytes overhead".format(str_size)) - note(" expiring entry: {} + {} + {} = {} bytes -> +{} bytes overhead".format( - str_size, wrapper_size, float_size, - str_size + wrapper_size + float_size, wrapper_size + float_size, - )) - print() - - saved_per_entry = wrapper_size - note("Saving vs wrapping replay-safe payloads too (the previous design): {} bytes/entry,".format( - saved_per_entry - )) - note("which is {:.0f}% of what such an entry used to occupy.".format( - 100.0 * saved_per_entry / (str_size + saved_per_entry) - )) - for n in (100, 10000, 100000): - note(" at sender_queue_size={:<7d} that's ~{:.1f}KB less resident memory".format( - n, saved_per_entry * n / 1024.0 - )) - print() - - note("The second effect is GC pressure, and it is the bigger one in practice: PendingPayload") - note("holds references, so every instance is tracked by the cyclic collector and traversed on") - note("each gen-2 pass. str is atomic and never traversed. A queue full of replay-safe payloads") - note("therefore contributes nothing to GC pause time now -- see scenario 1's max latency, which") - note("was dominated by exactly this traversal when every entry was wrapped.") - note("(An earlier version coalesced timestamps into a shared per-100ms-bucket float to cut the") - note("expiring-entry cost too. Reverted: global mutable state and cross-instance coupling for") - note("~234KB at sender_queue_size=10,000, i.e. ~0.09% of a 256MB container's RSS.)") - - -# -------------------------------------------------------------------------- -def main(): - print("SenderQueue performance microbenchmark") - print("Python {}.{}.{} {}".format(sys.version_info[0], sys.version_info[1], sys.version_info[2], sys.platform)) - if QUICK: - print("(--quick mode: reduced iteration counts)") - - scenario_1_unbounded_single_threaded() - scenario_2_sustained_overflow() - scenario_3_large_expired_backlog() - scenario_4_concurrency() - scenario_5_memory_footprint() - - section("DONE") - print(" This script can't literally run against the pre-SenderQueue commit (SenderQueue") - print(" didn't exist), so the queue.Queue lines above ARE the 'before' baseline: they") - print(" faithfully reproduce the old put_nowait()/get()/task_done() contract SenderQueue") - print(" replaced. Compare SenderQueue's numbers against the queue.Queue numbers *in the") - print(" same run* (same machine, same moment, same load) rather than against an absolute") - print(" number, and re-run a few times to see how much that ratio itself varies with noise.") - print(" For an end-to-end (with real socket I/O) before/after comparison instead, run") - print(" tests/performance/test_statsd_throughput.py with disable_background_sender=False") - print(" against both the current commit and the one before this queue was introduced.") - - -if __name__ == "__main__": - main() From 4e89ffb4db1bb3aeae705115600c2bcf34b14c36 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Thu, 17 Sep 2026 17:13:15 +0100 Subject: [PATCH 18/30] Drop the last tests/manual file from the PR tests/manual/ holds ad-hoc development drivers, not tests the suite runs, so none of it belongs in the review. This removes the one remaining tracked file, test_gauge_with_timestamp_aggregation.py, leaving tests/manual/ entirely untracked. Retained on disk via .git/info/exclude, which now excludes the whole directory so future scratch scripts there cannot be added by accident. --- .../test_gauge_with_timestamp_aggregation.py | 22 ------------------- 1 file changed, 22 deletions(-) delete mode 100644 tests/manual/test_gauge_with_timestamp_aggregation.py diff --git a/tests/manual/test_gauge_with_timestamp_aggregation.py b/tests/manual/test_gauge_with_timestamp_aggregation.py deleted file mode 100644 index 6ab276a74..000000000 --- a/tests/manual/test_gauge_with_timestamp_aggregation.py +++ /dev/null @@ -1,22 +0,0 @@ -import time - -from datadog.dogstatsd.base import DogStatsd - -client = DogStatsd( - socket_path="/tmp/dsd.sock", - disable_aggregation=True, - disable_buffering=False, - flush_interval=1.0, - disable_telemetry=True, -) - -start = time.time() -i = 0 - -while time.time() - start < 10: - i += 1 - client.gauge_with_timestamp("test.aggregation", float(i), tags=["env:test"], timestamp=time.time()) - -client.stop() -print("Done.") - From aa0f067413ebe555eb27592585df2ae83414d5cc Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Fri, 18 Sep 2026 09:48:10 +0100 Subject: [PATCH 19/30] Changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 994b55951..8455b91a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +* [Changed] DogStatsD background sender queue now evicts the oldest queued payloads when full, and drops payloads without an explicit timestamp after they have been queued for more than 10 seconds. See [#986](https://github.com/DataDog/datadogpy/pull/986). + ## v0.53.0 / 2026-07-24 * [Fixed] Add DD_DOGSTATSD_URL support for Unix and UDP URLs. See [#968](https://github.com/DataDog/datadogpy/pull/968). From 18d3018ca563ad8b857c1909314af1505e9ee556 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Fri, 18 Sep 2026 16:46:51 +0100 Subject: [PATCH 20/30] Track items requeued to ensure double requeues dont corrupt counter --- datadog/dogstatsd/base.py | 4 +- datadog/dogstatsd/sender_queue.py | 97 +++++++++++++- tests/unit/dogstatsd/test_statsd.py | 189 +++++++++++++++++++++++++++- 3 files changed, 279 insertions(+), 11 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index 61c7bc6fe..a1204719f 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -2209,7 +2209,7 @@ def _sender_main_loop(self, pending_queue): while True: item = pending_queue.get() if item is Stop: - pending_queue.task_done() + pending_queue.task_done(item) return # payload_text() also narrows the type: 'if item is Stop' above is @@ -2232,7 +2232,7 @@ def _sender_main_loop(self, pending_queue): # Sent, or a definitive failure that _xmit_packet already # accounted for as a dropped packet -- either way, this # payload's story is over. - pending_queue.task_done() + pending_queue.task_done(item) # type: ignore[arg-type] backoff = UDS_CONNECT_RETRY_INITIAL_BACKOFF def wait_for_pending(self): diff --git a/datadog/dogstatsd/sender_queue.py b/datadog/dogstatsd/sender_queue.py index 95098e389..4e84d0168 100644 --- a/datadog/dogstatsd/sender_queue.py +++ b/datadog/dogstatsd/sender_queue.py @@ -1,11 +1,14 @@ import collections +import logging import sys import threading from datadog.util.compat import monotonic +log = logging.getLogger("datadog.dogstatsd") + if sys.version_info[:2] >= (3, 5): - from typing import Callable, Optional, Union # noqa: F401 + from typing import Callable, Dict, Optional, Union # noqa: F401 # Sentinel telling the background sender thread to shut down. @@ -119,6 +122,19 @@ def __init__(self, maxsize, expiry_seconds, on_drop_queue_full, on_drop_expired, # all tasks have been dropped or sent. self._unfinished_tasks = 0 + # The items currently handed out by get() and not yet finished via + # requeue_front() or task_done(), keyed by id(item). SenderQueue is + # single-consumer by design (one background sender thread), so this + # normally holds at most one entry at a time. Tracking it lets + # requeue_front()/task_done() verify their precondition -- that the + # item they're handed really is the one get() currently has out of + # the queue -- so the misuse that would otherwise silently corrupt + # _unfinished_tasks (a double finish, a double requeue, or a requeue + # of an already-finished item) fails loudly instead of drifting the + # counter. The item itself is held in the value to keep it alive + # (and its id stable) for as long as the entry exists. + self._in_flight = {} # type: Dict[int, QueuedItemOrStop] + def _expired(self, item, now): # type: (QueuedItem, float) -> bool if not isinstance(item, PendingPayload): @@ -209,6 +225,19 @@ def requeue_front(self, item): Either way, a drop here finishes the task that put() started. """ with self._not_empty: + # The item handed back must be exactly the one get() currently has + # out of the queue. SenderQueue is single-consumer; a double + # requeue or a requeue of an already-finished item would otherwise + # corrupt _unfinished_tasks. If the item isn't in flight, log and + # bail out without touching the deque or the counter -- it's + # already been accounted for elsewhere, so this is a no-op rather + # than a crash. (Releasing it from in flight here is correct in + # every branch below: it's either requeued back onto the deque -- + # where a future get() will pick it up again -- or dropped for + # good.) + if not self._release_in_flight_locked(item, "requeue_front"): + return + if self._expired(item, monotonic()): self._on_drop_expired(item) self._finish_task_locked() @@ -233,6 +262,10 @@ def get(self): # A slot just opened up: wake one thread blocked in put()'s # wait-for-room loop, if any (harmless no-op otherwise). self._not_full.notify() + # Record this item as in flight, owned by the current thread, + # until requeue_front() or task_done() releases it (see + # _in_flight in __init__). + self._take_in_flight_locked(item) if item is Stop: return item @@ -244,24 +277,78 @@ def get(self): # expire, so the value was computed and immediately discarded. if isinstance(item, PendingPayload) and self._expired(item, monotonic()): self._on_drop_expired(item) - self.task_done() + self.task_done(item) continue return item + def _take_in_flight_locked(self, item): + # type: (QueuedItemOrStop) -> None + # Caller already holds self._lock (shared by _not_empty / _all_tasks_done). + # Records `item` as the one currently handed out by get(). A duplicate + # here means a previous get() was never finished (or the same object + # was queued twice); we log it and overwrite so the new handout is the + # one tracked, rather than crashing the sender thread. + key = id(item) + if key in self._in_flight: + log.error( + "dogstatsd sender queue: get() handed out an item already tracked as " + "in flight; a previous get() was never finished with task_done() / " + "requeue_front(), or the same object was queued more than once. " + "Counter bookkeeping may drift." + ) + self._in_flight[key] = item + + def _release_in_flight_locked(self, item, action): + # type: (QueuedItem, str) -> bool + # Caller already holds self._lock (shared by _not_empty / _all_tasks_done). + # Verifies `item` is currently in flight, then drops it from the + # in-flight map. `action` names the caller ("requeue_front"/ + # "task_done") for the log message. Returns False (after logging) when + # the item is not in flight, so the caller can skip the counter/deque + # mutation that would otherwise drift _unfinished_tasks -- without + # crashing the sender thread. + key = id(item) + if key not in self._in_flight: + log.error( + "dogstatsd sender queue: %s() was called on an item that is not " + "currently in flight (it was never returned by get(), or was " + "already finished). Ignoring it to keep the task counter consistent.", + action, + ) + return False + del self._in_flight[key] + return True + def _finish_task_locked(self): # type: () -> None # Caller already holds self._lock (shared by _not_empty / _all_tasks_done). unfinished = self._unfinished_tasks - 1 if unfinished < 0: - raise ValueError("task_done() called too many times") + # More finishes than puts: a real bookkeeping bug. Log it and + # clamp at zero rather than raising, so the sender thread stays + # alive. Notify in case a join() is waiting, so it doesn't hang. + log.error( + "dogstatsd sender queue: task accounting went negative " + "(_unfinished_tasks below zero); clamping. This indicates a " + "double finish or a finish without a matching put()." + ) + unfinished = 0 self._unfinished_tasks = unfinished if unfinished == 0: self._all_tasks_done.notify_all() - def task_done(self): - # type: () -> None + def task_done(self, item): + # type: (QueuedItemOrStop) -> None with self._all_tasks_done: + # The item being finished must be the one get() currently has out + # of the queue. This is the counterpart to get()'s + # _take_in_flight_locked(); a second task_done() (double finish) + # would otherwise let _unfinished_tasks drift. If it's not in + # flight, log and bail out without decrementing -- the task was + # already finished elsewhere -- rather than crashing the sender. + if not self._release_in_flight_locked(item, "task_done"): + return self._finish_task_locked() def join(self): diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index 314a52ac9..8ae3d48d1 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -10,6 +10,7 @@ # Standard libraries from collections import deque from contextlib import closing +import logging import struct from threading import Thread import errno @@ -2896,8 +2897,9 @@ def blocked_put(): # Now free up room: the blocked put() should wake up and # succeed without ever having dropped anything. - self.assertEqual(pending_queue.get().payload, "first\n") - pending_queue.task_done() + first = pending_queue.get() + self.assertEqual(first.payload, "first\n") + pending_queue.task_done(first) finally: t.join(timeout=5.0) @@ -2932,8 +2934,9 @@ def blocked_put(): self.assertTrue(t.is_alive(), "put() should still be waiting for room") # Drain the one slot: the blocked put() should wake up promptly. - self.assertEqual(pending_queue.get().payload, "first\n") - pending_queue.task_done() + first = pending_queue.get() + self.assertEqual(first.payload, "first\n") + pending_queue.task_done(first) t.join(timeout=5.0) self.assertFalse(t.is_alive()) @@ -3216,6 +3219,184 @@ def test_sender_queue_requeue_front_drops_when_expired(self): self.assertEqual([p.payload for p in dropped_expired], ["stale\n"]) self.assertEqual(pending_queue.qsize(), 0) + def test_sender_queue_requeue_front_ignores_item_not_in_flight(self): + # requeue_front() must be called on the exact item get() returned, and + # only while it's still in flight. Requeuing something that was never + # get() (or was already finished) would otherwise corrupt + # _unfinished_tasks. The guard logs the misuse and ignores the call + # (no deque change, no counter change) so the sender thread stays + # alive rather than crashing on a bookkeeping bug. + pending_queue = SenderQueue( + maxsize=0, + expiry_seconds=20.0, + on_drop_queue_full=lambda item: self.fail("unexpected queue-full drop"), + on_drop_expired=lambda item: self.fail("unexpected expiry drop"), + ) + + never_got = PendingPayload("never-get\n", sender_queue_clock()) + with self.assertLogs("datadog.dogstatsd", level="ERROR"): + pending_queue.requeue_front(never_got) + self.assertEqual(pending_queue.qsize(), 0, "never-got item was not added to the deque") + self.assertEqual(pending_queue._unfinished_tasks, 0, "counter untouched") + + # And requeuing an item that was already finished (get then task_done) + # is equally ignored: it's no longer in flight, the counter stays put. + finished = PendingPayload("finished\n", sender_queue_clock()) + pending_queue.put(finished) + got = pending_queue.get() + self.assertIs(got, finished) + pending_queue.task_done(got) + self.assertEqual(pending_queue._unfinished_tasks, 0, "finished -> counter back to zero") + with self.assertLogs("datadog.dogstatsd", level="ERROR"): + pending_queue.requeue_front(got) + self.assertEqual(pending_queue._unfinished_tasks, 0, "counter did not drift on the ignored requeue") + self.assertEqual(pending_queue.qsize(), 0, "already-finished item was not re-added") + + def test_sender_queue_requeue_front_ignores_double_requeue(self): + # A second requeue_front() of the same item (e.g. two threads both + # handed the same in-flight reference) would put it in the deque twice + # while only one task was ever counted. The in-flight guard logs the + # second call and ignores it, so the deque and counter stay consistent. + pending_queue = SenderQueue( + maxsize=0, + expiry_seconds=20.0, + on_drop_queue_full=lambda item: self.fail("unexpected queue-full drop"), + on_drop_expired=lambda item: self.fail("unexpected expiry drop"), + ) + + item = PendingPayload("once\n", sender_queue_clock()) + pending_queue.put(item) + got = pending_queue.get() + self.assertIs(got, item) + pending_queue.requeue_front(got) # first requeue: fine, back in the deque + self.assertEqual(pending_queue.qsize(), 1) + with self.assertLogs("datadog.dogstatsd", level="ERROR"): + pending_queue.requeue_front(got) # second: no longer in flight + self.assertEqual(pending_queue.qsize(), 1, "item was NOT added to the deque a second time") + self.assertEqual(pending_queue._unfinished_tasks, 1, "counter unchanged") + + def test_sender_queue_task_done_ignores_double_finish(self): + # A second task_done() on the same item (double finish) would drive + # _unfinished_tasks negative. The in-flight guard logs the second call + # and skips the decrement, so the counter stays at zero instead of + # going to -1 -- and the sender thread stays alive. + pending_queue = SenderQueue( + maxsize=0, + expiry_seconds=20.0, + on_drop_queue_full=lambda item: self.fail("unexpected queue-full drop"), + on_drop_expired=lambda item: self.fail("unexpected expiry drop"), + ) + + item = PendingPayload("once\n", sender_queue_clock()) + pending_queue.put(item) + got = pending_queue.get() + pending_queue.task_done(got) + self.assertEqual(pending_queue._unfinished_tasks, 0, "first finish -> counter at zero") + with self.assertLogs("datadog.dogstatsd", level="ERROR"): + pending_queue.task_done(got) + self.assertEqual(pending_queue._unfinished_tasks, 0, "second finish did NOT drive the counter negative") + + def test_sender_queue_many_threads_get_and_requeue_never_logs_error(self): + # 100 threads, split into getters and requeuers. Each getter runs + # get() and hands the item to a requeuer via a thread-safe handoff; + # each requeuer takes that item and calls requeue_front() on it. So + # the thread that get() the item is NOT the thread that requeues it -- + # the item crosses thread boundaries. The identity guard is + # thread-agnostic by design, so this must stay clean: no ERROR log, + # no counter drift, no crash. This is the future-proofing proof: a + # legitimate cross-thread get/requeue workload stays clean. + try: + import queue as queue_mod + except ImportError: # Python 2 + import Queue as queue_mod + + pending_queue = SenderQueue( + maxsize=0, # unbounded: requeue never drops for capacity + expiry_seconds=100.0, + on_drop_queue_full=lambda item: self.fail("unexpected queue-full drop"), + on_drop_expired=lambda item: self.fail("unexpected expiry drop"), + ) + + n_items = 200 + n_getters = 50 + n_requeuers = 50 + iterations = 100 + + for i in range(n_items): + pending_queue.put(PendingPayload("item-{}\n".format(i), sender_queue_clock())) + self.assertEqual(pending_queue._unfinished_tasks, n_items) + + handoff = queue_mod.Queue() + SENTINEL = object() + + # Capture any ERROR logged to the dogstatsd logger from any thread. + captured = [] + + class _CaptureHandler(logging.Handler): + def emit(self, record): + captured.append(record) + + dogstatsd_logger = logging.getLogger("datadog.dogstatsd") + handler = _CaptureHandler(level=logging.ERROR) + dogstatsd_logger.addHandler(handler) + prev_level = dogstatsd_logger.level + dogstatsd_logger.setLevel(min(prev_level, logging.ERROR)) + try: + def getter(): + for _ in range(iterations): + item = pending_queue.get() + handoff.put(item) + + def requeuer(): + while True: + item = handoff.get() + if item is SENTINEL: + return + pending_queue.requeue_front(item) + + getters = [threading.Thread(target=getter) for _ in range(n_getters)] + requeuers = [threading.Thread(target=requeuer) for _ in range(n_requeuers)] + + # Start requeuers first so they're draining the handoff before + # getters begin filling it; otherwise the handoff could grow + # unbounded and the SenderQueue could drain to empty (getters + # would then block in get() until requeuers put items back). + for t in requeuers: + t.start() + for t in getters: + t.start() + + for t in getters: + t.join() + # All getters done: every real item is either in the handoff or + # already requeued. Sentinels go behind them, so no real item is + # stranded. + for _ in range(n_requeuers): + handoff.put(SENTINEL) + for t in requeuers: + t.join() + finally: + dogstatsd_logger.removeHandler(handler) + dogstatsd_logger.setLevel(prev_level) + + # No ERROR log ever fired: the in-flight guard never tripped even + # though every item crossed from the getter thread to a different + # requeuer thread. + self.assertEqual( + captured, [], + "no error should be logged when get() and requeue_front() run on " + "different threads; got: {!r}".format([r.getMessage() for r in captured]), + ) + + # Every get() was matched by a requeue_front() (no task_done, no + # drops on the unbounded queue), so the queue is fully populated and + # the task counter is unchanged -- no drift. + self.assertEqual(pending_queue.qsize(), n_items, "all items back in the queue") + self.assertEqual( + pending_queue._unfinished_tasks, n_items, + "_unfinished_tasks never drifted under cross-thread get/requeue contention", + ) + def test_replay_safety_is_carried_by_the_queued_entry_type(self): # Replay-safety is not a stored flag: an entry subject to expiry is a # PendingPayload (carrying the enqueued_at it will be judged against), From 39f453b9fa5536b767cdb58a1cf4d6295aebb764 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Fri, 18 Sep 2026 16:52:19 +0100 Subject: [PATCH 21/30] Fix CI error --- datadog/dogstatsd/sender_queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datadog/dogstatsd/sender_queue.py b/datadog/dogstatsd/sender_queue.py index 4e84d0168..f5f9f997c 100644 --- a/datadog/dogstatsd/sender_queue.py +++ b/datadog/dogstatsd/sender_queue.py @@ -300,7 +300,7 @@ def _take_in_flight_locked(self, item): self._in_flight[key] = item def _release_in_flight_locked(self, item, action): - # type: (QueuedItem, str) -> bool + # type: (QueuedItemOrStop, str) -> bool # Caller already holds self._lock (shared by _not_empty / _all_tasks_done). # Verifies `item` is currently in flight, then drops it from the # in-flight map. `action` names the caller ("requeue_front"/ From a25955a807e8ac57a59eade8f33f0da1db9744d4 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Fri, 18 Sep 2026 16:58:03 +0100 Subject: [PATCH 22/30] Fix python 2 --- tests/unit/dogstatsd/test_statsd.py | 53 +++++++++++++++++------------ 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index 8ae3d48d1..2f3915951 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -9,7 +9,7 @@ """ # Standard libraries from collections import deque -from contextlib import closing +from contextlib import closing, contextmanager import logging import struct from threading import Thread @@ -176,6 +176,28 @@ def tearDown(self): """ self._procfs_mock.stop() + @contextmanager + def _capture_error_logs(self): + # assertLogs() is Python 3.4+ only, but this suite still runs on + # Python 2.7 / pypy2.7. Capture ERROR records on the dogstatsd logger + # with a plain handler instead, so the guard tests work everywhere. + captured = [] + + class _CaptureHandler(logging.Handler): + def emit(self, record): + captured.append(record) + + dogstatsd_logger = logging.getLogger("datadog.dogstatsd") + handler = _CaptureHandler(level=logging.ERROR) + dogstatsd_logger.addHandler(handler) + prev_level = dogstatsd_logger.level + dogstatsd_logger.setLevel(min(prev_level, logging.ERROR)) + try: + yield captured + finally: + dogstatsd_logger.removeHandler(handler) + dogstatsd_logger.setLevel(prev_level) + def assert_equal_telemetry(self, expected_payload, actual_payload, telemetry=None, **kwargs): if telemetry is None: telemetry = telemetry_metrics(bytes_sent=len(expected_payload), **kwargs) @@ -3234,8 +3256,9 @@ def test_sender_queue_requeue_front_ignores_item_not_in_flight(self): ) never_got = PendingPayload("never-get\n", sender_queue_clock()) - with self.assertLogs("datadog.dogstatsd", level="ERROR"): + with self._capture_error_logs() as captured: pending_queue.requeue_front(never_got) + self.assertTrue(captured, "the not-in-flight requeue should have logged an error") self.assertEqual(pending_queue.qsize(), 0, "never-got item was not added to the deque") self.assertEqual(pending_queue._unfinished_tasks, 0, "counter untouched") @@ -3247,8 +3270,9 @@ def test_sender_queue_requeue_front_ignores_item_not_in_flight(self): self.assertIs(got, finished) pending_queue.task_done(got) self.assertEqual(pending_queue._unfinished_tasks, 0, "finished -> counter back to zero") - with self.assertLogs("datadog.dogstatsd", level="ERROR"): + with self._capture_error_logs() as captured: pending_queue.requeue_front(got) + self.assertTrue(captured, "the already-finished requeue should have logged an error") self.assertEqual(pending_queue._unfinished_tasks, 0, "counter did not drift on the ignored requeue") self.assertEqual(pending_queue.qsize(), 0, "already-finished item was not re-added") @@ -3270,8 +3294,9 @@ def test_sender_queue_requeue_front_ignores_double_requeue(self): self.assertIs(got, item) pending_queue.requeue_front(got) # first requeue: fine, back in the deque self.assertEqual(pending_queue.qsize(), 1) - with self.assertLogs("datadog.dogstatsd", level="ERROR"): + with self._capture_error_logs() as captured: pending_queue.requeue_front(got) # second: no longer in flight + self.assertTrue(captured, "the double requeue should have logged an error") self.assertEqual(pending_queue.qsize(), 1, "item was NOT added to the deque a second time") self.assertEqual(pending_queue._unfinished_tasks, 1, "counter unchanged") @@ -3292,8 +3317,9 @@ def test_sender_queue_task_done_ignores_double_finish(self): got = pending_queue.get() pending_queue.task_done(got) self.assertEqual(pending_queue._unfinished_tasks, 0, "first finish -> counter at zero") - with self.assertLogs("datadog.dogstatsd", level="ERROR"): + with self._capture_error_logs() as captured: pending_queue.task_done(got) + self.assertTrue(captured, "the double finish should have logged an error") self.assertEqual(pending_queue._unfinished_tasks, 0, "second finish did NOT drive the counter negative") def test_sender_queue_many_threads_get_and_requeue_never_logs_error(self): @@ -3329,19 +3355,7 @@ def test_sender_queue_many_threads_get_and_requeue_never_logs_error(self): handoff = queue_mod.Queue() SENTINEL = object() - # Capture any ERROR logged to the dogstatsd logger from any thread. - captured = [] - - class _CaptureHandler(logging.Handler): - def emit(self, record): - captured.append(record) - - dogstatsd_logger = logging.getLogger("datadog.dogstatsd") - handler = _CaptureHandler(level=logging.ERROR) - dogstatsd_logger.addHandler(handler) - prev_level = dogstatsd_logger.level - dogstatsd_logger.setLevel(min(prev_level, logging.ERROR)) - try: + with self._capture_error_logs() as captured: def getter(): for _ in range(iterations): item = pending_queue.get() @@ -3375,9 +3389,6 @@ def requeuer(): handoff.put(SENTINEL) for t in requeuers: t.join() - finally: - dogstatsd_logger.removeHandler(handler) - dogstatsd_logger.setLevel(prev_level) # No ERROR log ever fired: the in-flight guard never tripped even # though every item crossed from the getter thread to a different From 354428bd969b90142a5a46cba8639f156f4be825 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Mon, 21 Sep 2026 11:47:13 +0100 Subject: [PATCH 23/30] Make sender_queue_expiry_seconds configurable. --- datadog/dogstatsd/base.py | 40 ++++++++++++++++++++++++----- datadog/dogstatsd/sender_queue.py | 7 ----- tests/unit/dogstatsd/test_statsd.py | 22 ++++++++++++++++ 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index a1204719f..e622c9a1e 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -49,7 +49,6 @@ SenderQueue, PendingPayload, Stop, - PENDING_PAYLOAD_EXPIRY_SECONDS, payload_text, ) @@ -189,6 +188,14 @@ def reverse(self): UDS_CONNECT_RETRY_INITIAL_BACKOFF = 0.025 UDS_CONNECT_RETRY_MAX_BACKOFF = 1.0 UDS_TRANSIENT_CONNECT_ERRORS = set([errno.ENOENT, errno.ECONNREFUSED]) + +# How long (in seconds) a non-replay-safe payload may sit in the background +# sender queue before it's considered stale and dropped instead of sent. +# Payloads that carry their own explicit timestamp (replay-safe) are exempt: +# delivering those late doesn't change what they mean, so they're kept +# around until they can actually be sent. This is the default for +# sender_queue_expiry_seconds; it can be overridden per client. +PENDING_PAYLOAD_EXPIRY_SECONDS = 10.0 # Errors seen while sending on an already-connected socket that indicate the # peer went away (e.g. the agent crashed/restarted). These are worth a single # reconnect-and-resend attempt instead of dropping the packet outright. @@ -310,6 +317,7 @@ def __init__( disable_background_sender=True, # type: bool sender_queue_size=0, # type: int sender_queue_timeout=0, # type: Optional[float] + sender_queue_expiry_seconds=PENDING_PAYLOAD_EXPIRY_SECONDS, # type: float track_instance=True, # type: bool socket_connect_timeout=DEFAULT_SOCKET_CONNECT_TIMEOUT, # type: Optional[float] ): # type: (...) -> None @@ -506,6 +514,13 @@ def __init__( Default: 0 (no wait) :type sender_queue_timeout: float + :param sender_queue_expiry_seconds: How long, in seconds, a non-replay-safe payload + may sit in the sender queue before it's considered stale and dropped instead of sent. + Payloads that carry their own explicit timestamp (replay-safe) are exempt: delivering + those late doesn't change what they mean, so they're kept until they can be sent. + Default: PENDING_PAYLOAD_EXPIRY_SECONDS (10.0). + :type sender_queue_expiry_seconds: float + :param track_instance: Keep track of this instance and automatically handle cleanup when os.fork() is called, if supported. Default: True. @@ -641,7 +656,9 @@ def __init__( self._sender_enabled = False if not disable_background_sender: - self.enable_background_sender(sender_queue_size, sender_queue_timeout) + self.enable_background_sender( + sender_queue_size, sender_queue_timeout, sender_queue_expiry_seconds + ) if TRACK_INSTANCES and track_instance: _instances.add(self) @@ -705,8 +722,13 @@ def telemetry_socket(self, t_socket): log.info("Unexpected telemetry socket provided with no support for getsockopt") self._telemetry_socket_kind = None - def enable_background_sender(self, sender_queue_size=0, sender_queue_timeout=0): - # type: (int, Optional[float]) -> None + def enable_background_sender( + self, + sender_queue_size=0, + sender_queue_timeout=0, + sender_queue_expiry_seconds=PENDING_PAYLOAD_EXPIRY_SECONDS, + ): + # type: (int, Optional[float], float) -> None """ Use a background thread to communicate with the dogstatsd server. When enabled, a background thread will be used to send metric payloads to the Agent. @@ -726,12 +748,18 @@ def enable_background_sender(self, sender_queue_size=0, sender_queue_timeout=0): If set to None, wait forever. If set to zero drop the packet immediately if the queue is full. Default: 0 (no wait). :type sender_queue_timeout: float, optional + :param sender_queue_expiry_seconds: How long, in seconds, a non-replay-safe payload may sit in the + sender queue before it's considered stale and dropped instead of sent. Replay-safe payloads + (those carrying their own explicit timestamp) are exempt and are kept until they can be sent. + Default: PENDING_PAYLOAD_EXPIRY_SECONDS (10.0). + :type sender_queue_expiry_seconds: float, optional """ with self._config_lock: self._sender_enabled = True self._sender_queue_size = sender_queue_size self._sender_queue_timeout = sender_queue_timeout + self._sender_queue_expiry_seconds = sender_queue_expiry_seconds self._start_sender_thread() @@ -1645,7 +1673,7 @@ def _account_dropped_queue_full(self, item): def _account_dropped_expired(self, item): # type: (QueuedItem) -> None - """A payload sat in the sender queue longer than PENDING_PAYLOAD_EXPIRY_SECONDS.""" + """A payload sat in the sender queue longer than the configured expiry (sender_queue_expiry_seconds).""" self.packets_dropped_expired += 1 self.bytes_dropped_expired += len(payload_text(item).encode(self.encoding)) @@ -2175,7 +2203,7 @@ def _start_sender_thread(self): self._queue = SenderQueue( self._sender_queue_size, - PENDING_PAYLOAD_EXPIRY_SECONDS, + self._sender_queue_expiry_seconds, self._account_dropped_queue_full, self._account_dropped_expired, put_timeout=self._sender_queue_timeout, diff --git a/datadog/dogstatsd/sender_queue.py b/datadog/dogstatsd/sender_queue.py index f5f9f997c..f6f699e76 100644 --- a/datadog/dogstatsd/sender_queue.py +++ b/datadog/dogstatsd/sender_queue.py @@ -21,13 +21,6 @@ QueuedItem = Union[str, "PendingPayload"] # noqa: F401 QueuedItemOrStop = Union[str, "PendingPayload", object] # noqa: F401 -# How long (in seconds) a non-replay-safe payload may sit in the background -# sender queue before it's considered stale and dropped instead of sent. -# Payloads that carry their own explicit timestamp (replay-safe) are exempt: -# delivering those late doesn't change what they mean, so they're kept -# around until they can actually be sent. -PENDING_PAYLOAD_EXPIRY_SECONDS = 10.0 - class PendingPayload(object): """A packet queued for the background sender that can go stale. diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index 2f3915951..d6179841e 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -2772,6 +2772,28 @@ def test_sender_mode(self): statsd = DogStatsd(disable_background_sender=False) self.assertIsNotNone(statsd._queue) + def test_sender_queue_expiry_seconds_defaults_to_constant(self): + statsd = DogStatsd(disable_background_sender=False) + self.assertEqual(statsd._sender_queue_expiry_seconds, PENDING_PAYLOAD_EXPIRY_SECONDS) + self.assertEqual(statsd._queue._expiry_seconds, PENDING_PAYLOAD_EXPIRY_SECONDS) + statsd.stop() + + def test_sender_queue_expiry_seconds_is_configurable(self): + statsd = DogStatsd( + disable_background_sender=False, + sender_queue_expiry_seconds=0.5, + ) + self.assertEqual(statsd._sender_queue_expiry_seconds, 0.5) + self.assertEqual(statsd._queue._expiry_seconds, 0.5) + statsd.stop() + + def test_enable_background_sender_accepts_expiry_seconds(self): + statsd = DogStatsd(disable_background_sender=True) + statsd.enable_background_sender(sender_queue_expiry_seconds=1.5) + self.assertEqual(statsd._sender_queue_expiry_seconds, 1.5) + self.assertEqual(statsd._queue._expiry_seconds, 1.5) + statsd.stop() + def test_sender_calls_task_done(self): statsd = DogStatsd(disable_background_sender=False) statsd.socket = OverflownSocket() From 8225d26b1078d535080261efbea0fae19ca8e733 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 22 Sep 2026 11:01:41 +0100 Subject: [PATCH 24/30] Add changelog entry for #987 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8455b91a1..15d86428a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased * [Changed] DogStatsD background sender queue now evicts the oldest queued payloads when full, and drops payloads without an explicit timestamp after they have been queued for more than 10 seconds. See [#986](https://github.com/DataDog/datadogpy/pull/986). +* [Changed] Replace DogStatsD's `socket_connect_timeout` option with a boolean `socket_connect_retry`: when enabled, a UDS connection failure from the background sender is retried indefinitely with backoff instead of dropping the payload immediately. Also add an optional `timeout` parameter to `stop()` and `wait_for_pending()` to bound how long they wait for the queue to drain. See [#987](https://github.com/DataDog/datadogpy/pull/987). ## v0.53.0 / 2026-07-24 From 1e8e1a466783429f97401c1aa3c84edf8a9603d4 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 22 Sep 2026 11:44:28 +0100 Subject: [PATCH 25/30] Honor the shutdown deadline before abandoning a payload stuck retrying --- datadog/dogstatsd/base.py | 207 +++++++++++++++++++++------- tests/unit/dogstatsd/test_statsd.py | 75 ++++++++-- 2 files changed, 222 insertions(+), 60 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index ead159b37..b61520b30 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -191,6 +191,21 @@ def reverse(self): # regardless of socket_connect_retry. SENDER_RETRY_INITIAL_BACKOFF = 0.025 SENDER_RETRY_MAX_BACKOFF = 60.0 +# How long an *unbounded* shutdown (pre_fork(), or stop()/wait_for_pending() +# called with timeout=None) still waits for a payload stuck in the retry +# loop above to resolve, before giving up on it. Without some bound here, a +# payload that can never succeed (the Agent permanently unreachable, plus a +# replay-safe payload, which never expires) would starve the Stop sentinel +# and hang the shutdown forever. Reuses SENDER_RETRY_MAX_BACKOFF's magnitude: +# that's already the longest gap between two retry attempts in steady state, +# so an unbounded shutdown should be at least that patient before giving up. +SENDER_UNBOUNDED_STOP_GRACE_SECONDS = SENDER_RETRY_MAX_BACKOFF +# How often the sender retries a connection once a shutdown has been +# requested but its deadline (the caller's own timeout, or the grace period +# above) hasn't passed yet. Deliberately much shorter than the normal +# backoff cap, so a still-recovering Agent gets drained before the shutdown +# gives up, without hammering a connection that keeps failing. +SENDER_STOP_RETRY_INTERVAL = 0.5 # How long (in seconds) a non-replay-safe payload may sit in the background # sender queue before it's considered stale and dropped instead of sent. @@ -492,9 +507,13 @@ def __init__( If True, a connection failure while sending a queued payload to a UNIX socket is retried indefinitely, backing off up to once a minute between attempts, instead of dropping the payload immediately. Ordinary payloads stuck retrying are eventually dropped as stale by the sender - queue's own expiry stop() and pre_fork() interrupt the retrying rather than waiting for it. - Direct/synchronous sends (the default mode) always fail fast on a connection error and are - unaffected by this setting. + queue's own expiry; replay-safe payloads never expire, so only a shutdown (stop(), + disable_background_sender(), pre_fork()) can end their retrying -- and it waits up to its own + timeout (or a bounded grace period, SENDER_UNBOUNDED_STOP_GRACE_SECONDS, for an unbounded call + such as pre_fork()) before giving up on one, at which point it is counted as a writer drop and + the shutdown call reports failure rather than success. wait_for_pending() never forces this: it + only waits for the queue's own retries and expiry to run their course. Direct/synchronous sends + (the default mode) always fail fast on a connection error and are unaffected by this setting. Default: False (fail fast, matching the previous socket_connect_timeout=0 default). :type socket_connect_retry: bool @@ -664,6 +683,16 @@ def __init__( # Set to ask a running sender thread to stop. Also what makes its # retry backoff interruptible -- see _sender_main_loop. self._sender_stopping = threading.Event() + # The monotonic deadline by which a requested shutdown must give up + # on a payload stuck in the retry loop, set by _stop_sender_thread() + # and read by _sender_main_loop. None means no shutdown has been + # requested (or a fresh sender hasn't seen one yet). + self._sender_stop_deadline = None # type: Optional[float] + # Set by _sender_main_loop when it gives up on a payload because the + # deadline above passed while still retrying a connection failure -- + # i.e. the queue did NOT fully drain even though the thread exited. + # _stop_sender_thread() reports this as failure rather than success. + self._sender_abandoned_payload = False self._sender_enabled = False if not disable_background_sender: @@ -781,10 +810,20 @@ def disable_background_sender(self, timeout=None): This call will block until all previously queued payloads are sent. :param timeout: Maximum number of seconds to wait for the sender thread - to drain the queue and exit. None (the default) waits indefinitely. + to drain the queue and exit. None (the default) waits indefinitely + for a sender that is genuinely busy (e.g. blocked inside a slow + send()), but not for a payload stuck retrying a connection + failure with socket_connect_retry enabled -- that case is bounded + by SENDER_UNBOUNDED_STOP_GRACE_SECONDS regardless of this + parameter, so an Agent that never comes back cannot hang this + call forever. :type timeout: float, optional - :return: True if the sender thread finished, False if timeout elapsed - while it was still running. + :return: True if the sender thread finished AND the queue actually + drained. False if timeout (or the grace period above) elapsed + first -- either because the sender thread is still running, or + because it gave up on a payload still retrying a connection + failure and dropped it (counted in packets_dropped_writer) + instead of delivering it. """ with self._config_lock: self._sender_enabled = False @@ -1668,6 +1707,12 @@ def _account_dropped_expired(self, item): self.packets_dropped_expired += 1 self.bytes_dropped_expired += len(payload_text(item).encode(self.encoding)) + def _account_dropped_writer(self, item): + # type: (QueuedItem) -> None + """A payload could not be written and is not being retried further.""" + self.packets_dropped_writer += 1 + self.bytes_dropped_writer += len(payload_text(item).encode(self.encoding)) + def _flush_telemetry(self): # type: () -> str tags = self._client_tags[:] @@ -2126,6 +2171,8 @@ def _start_sender_thread(self): # A previous _stop_sender_thread() leaves this set; clear it before the # new sender starts so it doesn't immediately think it's shutting down. self._sender_stopping.clear() + self._sender_stop_deadline = None + self._sender_abandoned_payload = False self._queue = SenderQueue( self._sender_queue_size, @@ -2146,11 +2193,21 @@ def _start_sender_thread(self): def _stop_sender_thread(self, timeout=None): # type: (Optional[float]) -> bool - # Ask the sender to stop before anything else: this is what breaks it - # out of a retry backoff (which can be as long as - # SENDER_RETRY_MAX_BACKOFF) instead of having to wait that out, and - # what lets it give up on a payload that would otherwise starve the - # Stop sentinel forever (see _sender_main_loop). + # Ask the sender to stop, with a deadline it must honor BEFORE + # abandoning a payload stuck in its retry-by-requeuing loop (see + # _sender_main_loop): a bounded caller's own timeout IS that + # deadline, so the sender keeps retrying for close to as long as the + # caller asked instead of giving up the instant a shutdown is + # requested. An unbounded caller (timeout=None, e.g. pre_fork()) gets + # a bounded grace period instead, so it can't hang forever on a + # payload that can genuinely never succeed. + grace = SENDER_UNBOUNDED_STOP_GRACE_SECONDS if timeout is None else timeout + self._sender_stop_deadline = monotonic() + grace + # Setting this second (after the deadline above is already visible to + # any thread this wakes) is what breaks the sender out of a retry + # backoff (which can be as long as SENDER_RETRY_MAX_BACKOFF) instead + # of having to wait that out -- see _sender_main_loop, which + # re-checks the deadline as soon as this wakes it. self._sender_stopping.set() # Lock ensures that nothing gets added to the queue after we disable it. @@ -2177,13 +2234,19 @@ def _stop_sender_thread(self, timeout=None): # exits (see _sender_main_loop). return False + # The thread exited, but that alone doesn't mean it drained: it may + # have hit the deadline above with a payload still stuck retrying and + # given up on it instead (see _sender_main_loop). That payload was + # never delivered, so report failure rather than claiming success. + abandoned = self._sender_abandoned_payload + # _sender_main_loop clears this state on its way out when the thread # has drained the queue, so this may already be a no-op; it also covers # a thread that exited without draining (e.g. never actually started). with self._buffer_lock: self._queue = None self._sender_thread = None - return True + return not abandoned def _release_sender_state(self, pending_queue): # type: (SenderQueue) -> None @@ -2219,22 +2282,47 @@ def _sender_main_loop(self, pending_queue): ) if sent is None: - # Connection trouble: keep the payload for the next attempt + # Connection trouble. A shutdown may already have been + # requested (see _stop_sender_thread) with a deadline this + # payload must be given a real chance against before it's + # given up on -- check that now, before requeuing, while the + # queue still considers this item in flight and can finish it + # outright instead. + deadline = self._sender_stop_deadline + if deadline is not None and monotonic() >= deadline: + # The deadline has passed: give up on this payload for + # good rather than requeuing it for a retry that will + # never be awaited. Account for it as a writer drop -- + # it was never delivered -- instead of letting it vanish + # with no telemetry at all. + self._account_dropped_writer(item) # type: ignore[arg-type] + pending_queue.task_done(item) # type: ignore[arg-type] + self._sender_abandoned_payload = True + self._release_sender_state(pending_queue) + return + + # Still worth retrying: keep the payload for the next attempt # instead of losing it. The queue's own expiry check (on a # future get()) is what eventually gives up on a payload # that's been stuck for too long, unless it's replay-safe. pending_queue.requeue_front(item) # type: ignore[arg-type] - # Interruptible backoff. - self._sender_stopping.wait(backoff) - if self._sender_stopping.is_set(): - # Checked rather than using wait()'s return value, which - # is only meaningful on Python 2.7+ -- and this module - # still supports 2.7, where several other wait() APIs - # return None. - self._release_sender_state(pending_queue) - return - backoff = min(backoff * 2, SENDER_RETRY_MAX_BACKOFF) + if deadline is None: + # No shutdown requested (yet): normal interruptible + # backoff. wait() returns early -- before backoff fully + # elapses -- the instant a shutdown IS requested, so the + # very next iteration's deadline check above fires + # promptly instead of waiting out a long backoff. + self._sender_stopping.wait(backoff) + backoff = min(backoff * 2, SENDER_RETRY_MAX_BACKOFF) + else: + # A shutdown was requested and its deadline hasn't + # passed yet: keep retrying -- a still-recovering Agent + # should still get drained -- but pace attempts instead + # of hammering a connection that keeps failing, and + # never sleep past the deadline. + remaining = deadline - monotonic() + time.sleep(min(SENDER_STOP_RETRY_INTERVAL, max(remaining, 0))) continue # Sent, or a definitive failure that _xmit_packet already @@ -2273,6 +2361,11 @@ def pre_fork(self): Flush any pending payloads and stop all background threads. + A payload stuck retrying a connection failure (socket_connect_retry) + is given up on -- and counted as a writer drop -- after a bounded + grace period (SENDER_UNBOUNDED_STOP_GRACE_SECONDS) rather than + blocking the fork indefinitely; that payload is not delivered. + The client should not be used from this point until state is restored by calling post_fork_parent() or post_fork_child(). @@ -2328,17 +2421,25 @@ def stop(self, timeout=None): :param timeout: Maximum number of seconds to wait for the background sender to drain its queue and exit. None (the default) waits - indefinitely, however long that takes. + indefinitely for a sender that is genuinely busy (e.g. blocked + inside a slow send()). It does NOT wait indefinitely for a + payload stuck retrying a connection failure with + socket_connect_retry enabled: that case is bounded by + SENDER_UNBOUNDED_STOP_GRACE_SECONDS regardless of this parameter, + so an Agent that never comes back cannot hang stop() forever. :type timeout: float, optional :return: True if the background sender drained and stopped, and the - final flush and socket close ran. False if timeout elapsed first, - in which case the sender thread is still running and neither the - final flush nor the socket close ran (see below). Do not call - stop() again while that sender is still running: it queues a - second internal shutdown signal that is never drained, which can - make wait_for_pending() block forever on the abandoned queue. Use - wait_for_pending() to wait for the sender instead, then call - stop() again once it has actually stopped. + final flush and socket close ran. False if timeout (or the grace + period above) elapsed first, in which case neither the final + flush nor the socket close ran (see below) -- either because the + sender thread is still running, or because it gave up on a + payload still retrying a connection failure and dropped it + (counted in packets_dropped_writer) instead of delivering it. Do + not call stop() again while that sender is still running: it + queues a second internal shutdown signal that is never drained, + which can make wait_for_pending() block forever on the abandoned + queue. Use wait_for_pending() to wait for the sender instead, + then call stop() again once it has actually stopped. """ stopped = self.disable_background_sender(timeout) @@ -2346,21 +2447,33 @@ def stop(self, timeout=None): self._disable_aggregation = True if not stopped: - # We gave up waiting, so the sender thread is still running and - # still owns the socket -- it can be parked inside a send() with - # _socket_lock held. Flushing or closing here would block on that - # same lock for as long as the sender stays wedged, which would - # make timeout meaningless: the caller asked for a bounded stop(). - # Pushing more data through that socket could not succeed anyway, - # and closing it from under a thread mid-write is not safe. Leave - # it open; the OS reclaims the fd when the process exits, and the - # sender thread is a daemon so it never holds up interpreter - # shutdown. - log.warning( - "stop() timed out after %ss with the background sender still running; " - "skipping the final flush and socket close", - timeout, - ) + if self._sender_abandoned_payload: + # The sender thread did exit, but only by giving up on a + # payload still retrying a connection failure once its + # deadline passed (see _sender_main_loop) -- that payload was + # dropped, not delivered, so this is not a clean stop either. + log.warning( + "stop() gave up on a payload stuck retrying a connection failure after " + "%ss; it was dropped instead of delivered (see packets_dropped_writer). " + "Skipping the final flush and socket close", + timeout, + ) + else: + # We gave up waiting, so the sender thread is still running and + # still owns the socket -- it can be parked inside a send() with + # _socket_lock held. Flushing or closing here would block on that + # same lock for as long as the sender stays wedged, which would + # make timeout meaningless: the caller asked for a bounded stop(). + # Pushing more data through that socket could not succeed anyway, + # and closing it from under a thread mid-write is not safe. Leave + # it open; the OS reclaims the fd when the process exits, and the + # sender thread is a daemon so it never holds up interpreter + # shutdown. + log.warning( + "stop() timed out after %ss with the background sender still running; " + "skipping the final flush and socket close", + timeout, + ) return False self.flush_aggregated_metrics() diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index a5b75f572..94e4eee7f 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -3800,6 +3800,44 @@ def flaky_get_uds_socket(cls, socket_path, timeout): statsd.stop() + def test_stop_timeout_retries_until_the_deadline_before_abandoning_a_payload(self): + # The actual regression this guards against: stop(timeout) must keep + # retrying a connection failure for close to the requested timeout -- + # not abandon on the very first backoff check -- so a payload that + # would have succeeded on a later attempt is not silently lost. + working_socket = FakeSocket() + attempts = {"count": 0} + + def flaky_get_uds_socket(cls, socket_path, timeout): + attempts["count"] += 1 + if attempts["count"] < 4: + raise socket.error(errno.ECONNREFUSED, "still refused") + return working_socket + + with mock.patch.object(DogStatsd, "_get_uds_socket", classmethod(flaky_get_uds_socket)), \ + patch("datadog.dogstatsd.base.SENDER_RETRY_INITIAL_BACKOFF", 0.05): + statsd = DogStatsd( + socket_path="/tmp/dogstatsd-test-stop-retries.sock", + disable_telemetry=True, + disable_background_sender=False, + socket_connect_retry=True, + ) + + statsd.gauge("eventually.sent", 1) + time.sleep(0.05) # let the first attempt fail and start retrying + + # A generous bounded timeout: comfortably enough for a few fast + # (patched-down) backoff doublings to reach the 4th, working + # attempt, but nowhere near SENDER_UNBOUNDED_STOP_GRACE_SECONDS. + # Before the fix, stop() abandoned on the very first backoff + # check regardless of this value, so this would have failed: + # returning near-instantly with the payload never sent. + self.assertIs(self._call_bounded(statsd.stop, (3.0,), limit=5.0), True) + + self.assertGreaterEqual(attempts["count"], 4) + self.assertEqual(statsd.packets_dropped_writer, 0, "the payload must not have been abandoned") + self.assertTrue(working_socket.payloads[0].decode("utf-8").startswith("eventually.sent:1|g")) + def test_queue_mode_drops_immediately_by_default(self): # socket_connect_retry defaults to False for the background sender # too: without it, a connection failure on a queued payload is @@ -3851,16 +3889,19 @@ def test_stop_is_bounded_with_a_stuck_replay_safe_payload(self): # requeue_front() puts a failed payload back ahead of the Stop # sentinel, and replay-safe payloads are exempt from the queue's # expiry, so such a payload never resolves while the Agent is down. - # Without an interruptible shutdown signal the sender never reaches - # Stop at all and stop() hangs forever. + # Without a bounded grace period the sender never reaches Stop at all + # and stop() hangs forever. A small patched grace makes the test fast + # and deterministic instead of depending on the real 60s default. statsd = self._unreachable_retrying_client() statsd.gauge_with_timestamp("replay.safe", 1, timestamp=int(time.time())) time.sleep(0.1) # let the sender pick it up and start retrying - t0 = time.time() - self.assertIs(self._call_bounded(statsd.stop, ()), True) - self.assertLess(time.time() - t0, 5.0, "stop() did not interrupt the retry loop") + with patch("datadog.dogstatsd.base.SENDER_UNBOUNDED_STOP_GRACE_SECONDS", 0.3): + t0 = time.time() + self.assertIs(self._call_bounded(statsd.stop, ()), False, "the payload was never delivered") + self.assertLess(time.time() - t0, 5.0, "stop() did not bound the retry loop") self.assertIsNone(statsd._queue) + self.assertEqual(statsd.packets_dropped_writer, 1, "the abandoned payload must be accounted for") def test_pre_fork_is_bounded_with_a_stuck_replay_safe_payload(self): # Same starvation, reached through pre_fork() -- which matters more: @@ -3877,22 +3918,25 @@ def test_pre_fork_is_bounded_with_a_stuck_replay_safe_payload(self): # instance is deliberately abandoned rather than restored. It is # constructed with track_instance=False precisely so nothing else can # ever try to take that lock again. - t0 = time.time() - self._call_bounded(statsd.pre_fork, ()) - self.assertLess(time.time() - t0, 5.0, "pre_fork() would have blocked os.fork()") + with patch("datadog.dogstatsd.base.SENDER_UNBOUNDED_STOP_GRACE_SECONDS", 0.3): + t0 = time.time() + self._call_bounded(statsd.pre_fork, ()) + self.assertLess(time.time() - t0, 5.0, "pre_fork() would have blocked os.fork()") self.assertIsNone(statsd._sender_thread, "pre_fork() should have stopped the sender") + self.assertEqual(statsd.packets_dropped_writer, 1, "the abandoned payload must be accounted for") def test_stop_interrupts_a_long_retry_backoff_instead_of_waiting_it_out(self): # The backoff cap is a full minute. A plain time.sleep() would make # stop() wait out however much of it is left; the shutdown signal must - # cut it short. + # cut it short well before its own (also patched down) grace period. statsd = self._unreachable_retrying_client() - with patch("datadog.dogstatsd.base.SENDER_RETRY_INITIAL_BACKOFF", 30.0): + with patch("datadog.dogstatsd.base.SENDER_RETRY_INITIAL_BACKOFF", 30.0), \ + patch("datadog.dogstatsd.base.SENDER_UNBOUNDED_STOP_GRACE_SECONDS", 0.3): statsd.gauge("ordinary", 1) time.sleep(0.3) # fail once, then settle into the 30s backoff t0 = time.time() - self.assertIs(self._call_bounded(statsd.stop, ()), True) + self.assertIs(self._call_bounded(statsd.stop, ()), False, "the payload was never delivered") self.assertLess(time.time() - t0, 5.0, "stop() waited out the backoff sleep") def test_sender_can_restart_after_a_stop_cleared_the_stopping_signal(self): @@ -3901,7 +3945,8 @@ def test_sender_can_restart_after_a_stop_cleared_the_stopping_signal(self): statsd = self._unreachable_retrying_client() statsd.gauge("ordinary", 1) time.sleep(0.1) - self.assertIs(self._call_bounded(statsd.stop, ()), True) + with patch("datadog.dogstatsd.base.SENDER_UNBOUNDED_STOP_GRACE_SECONDS", 0.3): + self.assertIs(self._call_bounded(statsd.stop, ()), False, "the payload was never delivered") statsd.enable_background_sender() try: @@ -3919,7 +3964,11 @@ def test_sender_can_restart_after_a_stop_cleared_the_stopping_signal(self): self.assertTrue(fresh.is_alive(), "fresh sender exited on a stale stopping signal") self.assertIs(statsd._sender_thread, fresh) finally: - statsd.stop(5.0) + # Bounded cleanup: the fresh sender is genuinely retrying against + # a socket path that will never exist, so a bounded stop() now + # legitimately waits close to its own timeout before giving up + # (see the fix above) -- keep it short so this cleanup stays fast. + statsd.stop(1.0) def test_set_socket_timeout(self): statsd = DogStatsd(disable_background_sender=False) From f2a72ed04420d2767067d0c2ab44471956babbe2 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 22 Sep 2026 12:11:15 +0100 Subject: [PATCH 26/30] Close the queue to producers atomically with appending Stop --- datadog/dogstatsd/base.py | 81 ++++++++++++++++++++------- tests/unit/dogstatsd/test_statsd.py | 86 ++++++++++++++++++++++++++++- 2 files changed, 146 insertions(+), 21 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index b61520b30..4de06cfd7 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -679,6 +679,15 @@ def __init__( log.debug("Statsd buffering and aggregation is disabled") self._queue = None # type: Optional[SenderQueue] + # The queue object owned by the current sender thread, for as long as + # that thread might still be draining it -- kept separate from + # self._queue, which is what _send_to_server() checks and which is + # closed to new producers the instant a shutdown is requested (see + # _stop_sender_thread). This lets wait_for_pending() keep reaching + # the real, still-draining queue during that window, and stops + # _start_sender_thread() from minting a second sender while the first + # one is still finishing up. + self._active_queue = None # type: Optional[SenderQueue] self._sender_thread = None # type: Optional[threading.Thread] # Set to ask a running sender thread to stop. Also what makes its # retry backoff interruptible -- see _sender_main_loop. @@ -2165,7 +2174,11 @@ def _start_sender_thread(self): if not self._sender_enabled or self._forking: return - if self._queue is not None: + # _active_queue (not self._queue) is the source of truth for whether + # a sender is already running: self._queue can already be None while + # a previous sender is still draining (see _stop_sender_thread), and + # starting a second one in that window would leak the first thread. + if self._active_queue is not None: return # A previous _stop_sender_thread() leaves this set; clear it before the @@ -2181,6 +2194,7 @@ def _start_sender_thread(self): self._account_dropped_expired, put_timeout=self._sender_queue_timeout, ) + self._active_queue = self._queue log.debug("Starting background sender thread") self._sender_thread = threading.Thread( @@ -2210,28 +2224,41 @@ def _stop_sender_thread(self, timeout=None): # re-checks the deadline as soon as this wakes it. self._sender_stopping.set() - # Lock ensures that nothing gets added to the queue after we disable it. + # Close the queue to new producers and enqueue Stop as ONE atomic + # step: _send_to_server() takes this same lock and re-checks + # self._queue before it puts, so a producer racing this either lands + # its payload on the still-open queue strictly before Stop is + # appended (delivered normally), or sees self._queue already None + # and falls back to a direct send -- never behind Stop, where it + # would be silently lost once the sender reaches Stop and exits. + # self._active_queue deliberately keeps pointing at the real object + # (see its declaration in __init__): wait_for_pending() and + # _start_sender_thread() still need to reach/recognise it while it + # drains. with self._buffer_lock: - if self._queue is not None: + queue = self._queue + self._queue = None + if queue is not None: # put() lets the Stop sentinel past the size limit, so this # never blocks even when the queue is full. - self._queue.put(Stop) + queue.put(Stop) thread = self._sender_thread if thread is None: # Nothing left to stop: no thread ever runs, so clear any residual - # queue. + # state. with self._buffer_lock: - self._queue = None + self._active_queue = None return True thread.join(timeout) if thread.is_alive(): - # Timed out. Leave _queue in place: it is what stops - # _start_sender_thread() from minting a second sender, and it keeps - # wait_for_pending()/_send_to_server() targeting the real queue. - # The sender thread clears this state itself when it eventually - # exits (see _sender_main_loop). + # Timed out. self._queue is already closed to producers (above). + # Leave _active_queue in place: it is what stops + # _start_sender_thread() from minting a second sender, and it + # keeps wait_for_pending() targeting the real, still-draining + # queue. The sender thread clears this state itself when it + # eventually exits (see _release_sender_state). return False # The thread exited, but that alone doesn't mean it drained: it may @@ -2240,11 +2267,12 @@ def _stop_sender_thread(self, timeout=None): # never delivered, so report failure rather than claiming success. abandoned = self._sender_abandoned_payload - # _sender_main_loop clears this state on its way out when the thread - # has drained the queue, so this may already be a no-op; it also covers - # a thread that exited without draining (e.g. never actually started). + # _release_sender_state clears this on the sender's way out when it + # has finished with the queue, so this may already be a no-op; it + # also covers a thread that exited without draining (e.g. never + # actually started). with self._buffer_lock: - self._queue = None + self._active_queue = None self._sender_thread = None return not abandoned @@ -2260,7 +2288,13 @@ def _release_sender_state(self, pending_queue): """ with self._buffer_lock: if self._queue is pending_queue: + # Already None in the common case: _stop_sender_thread() + # closes it to producers up front. Still guarded the same + # way in case this thread is exiting on its own (queue + # emptied normally, no shutdown ever requested). self._queue = None + if self._active_queue is pending_queue: + self._active_queue = None if self._sender_thread is threading.current_thread(): self._sender_thread = None @@ -2345,10 +2379,19 @@ def wait_for_pending(self, timeout=None): self.flush_buffered_metrics() - # Avoid race with disable_background_sender. We don't need a - # lock, just copy the value so it doesn't change between the - # check and join later. - queue = self._queue + # Prefer _active_queue over self._queue: the latter is closed to + # producers the instant a shutdown is requested (see + # _stop_sender_thread), but the sender may still be actively draining + # real, already-queued payloads at that point -- _active_queue keeps + # pointing at that real queue until the sender thread has actually + # finished with it. Falling back to self._queue covers a queue + # assigned directly rather than through _start_sender_thread() (e.g. + # in tests), where _active_queue was never set at all; reading it + # here for join() only, never to enqueue, doesn't reopen the + # producer race the split exists to close. We don't need a lock, + # just copy the value so it doesn't change between the check and + # join later. + queue = self._active_queue or self._queue if queue is None: return True diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index 94e4eee7f..4d3a07486 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -2825,6 +2825,80 @@ def blocking_xmit(packet, queue_mode=False): release.set() wedged.join(timeout=5.0) + def test_metric_call_racing_stop_does_not_land_behind_the_stop_sentinel(self): + # The actual regression this guards against: a metric call racing + # disable_background_sender() must never land in the queue BEHIND + # the Stop sentinel -- where it would be silently lost once the + # sender reaches Stop and exits without ever seeing it -- because + # self._queue is closed to new producers atomically with appending + # Stop (see _stop_sender_thread), not sometime after the drain. + statsd = DogStatsd(disable_background_sender=False, disable_telemetry=True) + release = threading.Event() + entered_first_send = threading.Event() + sent = [] + + def blocking_xmit(packet, queue_mode=False): + sent.append(packet) + entered_first_send.set() + release.wait(10.0) + return True + + statsd._xmit_packet_with_telemetry = blocking_xmit + statsd._send_to_server("first:1|c") + self.assertTrue(entered_first_send.wait(5.0), "sender never picked up the first payload") + + stop_result = {} + second_result = {} + + def call_stop(): + stop_result["value"] = statsd.stop(5.0) + + def call_second(): + # Deliberately started only once the queue is confirmed already + # closed below -- this IS the race under test. + statsd._send_to_server("second:1|c") + second_result["done"] = True + + stopper = threading.Thread(target=call_stop) + stopper.start() + second_thread = None + try: + # _stop_sender_thread's atomic close-and-append-Stop only needs + # _buffer_lock -- which the sender isn't holding while blocked in + # the wedged send above -- so it doesn't wait for the drain. + # Poll briefly rather than a fixed sleep, to stay fast without + # being flaky on a slower machine. + closed = False + deadline = time.time() + 2.0 + while time.time() < deadline: + if statsd._queue is None: + closed = True + break + time.sleep(0.01) + self.assertTrue(closed, "self._queue must close to producers promptly, without waiting for the drain") + self.assertIsNotNone(statsd._active_queue, "the real, still-draining queue must remain reachable") + self.assertTrue(stopper.is_alive(), "stop() must still be waiting on the wedged sender") + + second_thread = threading.Thread(target=call_second) + second_thread.start() + # Still wedged behind the first send (same mocked entry point, + # same release Event) -- confirms this really took the direct- + # send fallback rather than silently returning after a queue put. + second_thread.join(0.3) + self.assertTrue(second_thread.is_alive(), "expected the direct send to be wedged behind the first one too") + finally: + release.set() + stopper.join(timeout=10.0) + if second_thread is not None: + second_thread.join(timeout=10.0) + + self.assertIs(stop_result.get("value"), True, "stop() should have completed cleanly") + self.assertTrue(second_result.get("done")) + self.assertEqual( + sent, ["first:1|c\n", "second:1|c\n"], + "the racing payload must have been delivered (via a direct send), not lost behind Stop", + ) + def test_stop_timeout_is_bounded_while_the_sender_holds_the_socket_lock(self): # The wedge that matters in practice: the sender is parked inside a # blocking send() and therefore owns _socket_lock. stop()'s own @@ -2912,7 +2986,11 @@ def blocking_send(self, data): try: self.assertIs(self._call_bounded(statsd.stop, (0.2,)), False) self.assertTrue(wedged.is_alive()) - self.assertIsNotNone(statsd._queue, "queue must be retained so state stays coherent") + # self._queue itself is already closed to producers at this point + # (see _stop_sender_thread) -- self._active_queue is what must be + # retained so _start_sender_thread() knows a sender is still + # running and state stays coherent. + self.assertIsNotNone(statsd._active_queue, "active queue must be retained so state stays coherent") statsd.enable_background_sender() # This identity check IS the proof there's no orphan: if a second @@ -3007,7 +3085,10 @@ def blocking_send(self, data): try: self.assertIs(self._call_bounded(statsd.stop, (0.2,)), False) - self.assertIsNotNone(statsd._queue) + # self._queue is already closed to producers at this point (see + # _stop_sender_thread); self._active_queue is what still tracks + # the sender as active while it finishes up. + self.assertIsNotNone(statsd._active_queue) self.assertIs(statsd._sender_thread, wedged) # Caller moved on and never called stop() again; the wedged send @@ -3018,6 +3099,7 @@ def blocking_send(self, data): # The exit must have healed the client state so a re-enable works. self.assertIsNone(statsd._queue) + self.assertIsNone(statsd._active_queue) self.assertIsNone(statsd._sender_thread) statsd.enable_background_sender() self.assertIsNot( From f50ce6af629c298a6ec6fe59127528ef0f0048ab Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 22 Sep 2026 12:28:42 +0100 Subject: [PATCH 27/30] Simplify: drop _active_queue, gate producers on _sender_stopping instead --- datadog/dogstatsd/base.py | 112 +++++++++++----------------- tests/unit/dogstatsd/test_statsd.py | 41 +++++----- 2 files changed, 60 insertions(+), 93 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index 4de06cfd7..11fad3978 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -679,15 +679,6 @@ def __init__( log.debug("Statsd buffering and aggregation is disabled") self._queue = None # type: Optional[SenderQueue] - # The queue object owned by the current sender thread, for as long as - # that thread might still be draining it -- kept separate from - # self._queue, which is what _send_to_server() checks and which is - # closed to new producers the instant a shutdown is requested (see - # _stop_sender_thread). This lets wait_for_pending() keep reaching - # the real, still-draining queue during that window, and stops - # _start_sender_thread() from minting a second sender while the first - # one is still finishing up. - self._active_queue = None # type: Optional[SenderQueue] self._sender_thread = None # type: Optional[threading.Thread] # Set to ask a running sender thread to stop. Also what makes its # retry backoff interruptible -- see _sender_main_loop. @@ -1764,12 +1755,21 @@ def _is_telemetry_flush_time(self): def _send_to_server(self, packet, replay_safe=False): # type: (str, bool) -> None - # Skip the lock if the queue is None. There is no race with enable_background_sender. - if self._queue is not None: - # Prevent a race with disable_background_sender. + # Skip the lock if the queue is None or a shutdown has already been + # requested -- the lock-protected recheck below is what actually has + # to be correct; this is purely an optimization to avoid the lock + # once there is clearly nothing to enqueue onto. + if self._queue is not None and not self._sender_stopping.is_set(): + # Prevent a race with disable_background_sender: _sender_stopping + # is set BEFORE _stop_sender_thread() appends Stop, under this + # same lock, so rechecking it here (not just self._queue) means a + # racing put() either lands strictly before Stop (still open) or + # is rejected outright and falls through to a direct send below + # -- never behind Stop, where it would be silently lost once the + # sender reaches Stop and exits. with self._buffer_lock: packet_with_newline = packet + '\n' - if self._queue is not None: + if self._queue is not None and not self._sender_stopping.is_set(): if replay_safe: # Never expires, so it needs no enqueued_at and no # wrapper at all: queue the bare string and let the @@ -2174,11 +2174,7 @@ def _start_sender_thread(self): if not self._sender_enabled or self._forking: return - # _active_queue (not self._queue) is the source of truth for whether - # a sender is already running: self._queue can already be None while - # a previous sender is still draining (see _stop_sender_thread), and - # starting a second one in that window would leak the first thread. - if self._active_queue is not None: + if self._queue is not None: return # A previous _stop_sender_thread() leaves this set; clear it before the @@ -2194,7 +2190,6 @@ def _start_sender_thread(self): self._account_dropped_expired, put_timeout=self._sender_queue_timeout, ) - self._active_queue = self._queue log.debug("Starting background sender thread") self._sender_thread = threading.Thread( @@ -2217,48 +2212,41 @@ def _stop_sender_thread(self, timeout=None): # payload that can genuinely never succeed. grace = SENDER_UNBOUNDED_STOP_GRACE_SECONDS if timeout is None else timeout self._sender_stop_deadline = monotonic() + grace - # Setting this second (after the deadline above is already visible to - # any thread this wakes) is what breaks the sender out of a retry - # backoff (which can be as long as SENDER_RETRY_MAX_BACKOFF) instead - # of having to wait that out -- see _sender_main_loop, which + # Setting this is also what makes _send_to_server() reject any + # FURTHER producer call outright (see there) instead of letting it + # land behind the Stop sentinel appended below and be lost once the + # sender reaches Stop and exits, and what breaks the sender out of a + # retry backoff (which can be as long as SENDER_RETRY_MAX_BACKOFF) + # instead of having to wait that out -- see _sender_main_loop, which # re-checks the deadline as soon as this wakes it. self._sender_stopping.set() - # Close the queue to new producers and enqueue Stop as ONE atomic - # step: _send_to_server() takes this same lock and re-checks - # self._queue before it puts, so a producer racing this either lands - # its payload on the still-open queue strictly before Stop is - # appended (delivered normally), or sees self._queue already None - # and falls back to a direct send -- never behind Stop, where it - # would be silently lost once the sender reaches Stop and exits. - # self._active_queue deliberately keeps pointing at the real object - # (see its declaration in __init__): wait_for_pending() and - # _start_sender_thread() still need to reach/recognise it while it - # drains. + # Lock ensures that nothing gets added to the queue after the check + # above -- see _send_to_server(), which takes this same lock and + # re-checks _sender_stopping before it puts. with self._buffer_lock: - queue = self._queue - self._queue = None - if queue is not None: + if self._queue is not None: # put() lets the Stop sentinel past the size limit, so this # never blocks even when the queue is full. - queue.put(Stop) + self._queue.put(Stop) thread = self._sender_thread if thread is None: # Nothing left to stop: no thread ever runs, so clear any residual - # state. + # queue. with self._buffer_lock: - self._active_queue = None + self._queue = None return True thread.join(timeout) if thread.is_alive(): - # Timed out. self._queue is already closed to producers (above). - # Leave _active_queue in place: it is what stops - # _start_sender_thread() from minting a second sender, and it - # keeps wait_for_pending() targeting the real, still-draining - # queue. The sender thread clears this state itself when it - # eventually exits (see _release_sender_state). + # Timed out. Leave _queue in place: it is what stops + # _start_sender_thread() from minting a second sender, and it keeps + # wait_for_pending() targeting the real queue -- _send_to_server() + # itself already rejects new puts via the _sender_stopping check + # above, so nothing new lands on it in the meantime. The sender + # thread clears this state itself when it eventually exits (see + # _sender_main_loop). return False # The thread exited, but that alone doesn't mean it drained: it may @@ -2267,12 +2255,11 @@ def _stop_sender_thread(self, timeout=None): # never delivered, so report failure rather than claiming success. abandoned = self._sender_abandoned_payload - # _release_sender_state clears this on the sender's way out when it - # has finished with the queue, so this may already be a no-op; it - # also covers a thread that exited without draining (e.g. never - # actually started). + # _sender_main_loop clears this state on its way out when the thread + # has drained the queue, so this may already be a no-op; it also covers + # a thread that exited without draining (e.g. never actually started). with self._buffer_lock: - self._active_queue = None + self._queue = None self._sender_thread = None return not abandoned @@ -2288,13 +2275,7 @@ def _release_sender_state(self, pending_queue): """ with self._buffer_lock: if self._queue is pending_queue: - # Already None in the common case: _stop_sender_thread() - # closes it to producers up front. Still guarded the same - # way in case this thread is exiting on its own (queue - # emptied normally, no shutdown ever requested). self._queue = None - if self._active_queue is pending_queue: - self._active_queue = None if self._sender_thread is threading.current_thread(): self._sender_thread = None @@ -2379,19 +2360,10 @@ def wait_for_pending(self, timeout=None): self.flush_buffered_metrics() - # Prefer _active_queue over self._queue: the latter is closed to - # producers the instant a shutdown is requested (see - # _stop_sender_thread), but the sender may still be actively draining - # real, already-queued payloads at that point -- _active_queue keeps - # pointing at that real queue until the sender thread has actually - # finished with it. Falling back to self._queue covers a queue - # assigned directly rather than through _start_sender_thread() (e.g. - # in tests), where _active_queue was never set at all; reading it - # here for join() only, never to enqueue, doesn't reopen the - # producer race the split exists to close. We don't need a lock, - # just copy the value so it doesn't change between the check and - # join later. - queue = self._active_queue or self._queue + # Avoid race with disable_background_sender. We don't need a + # lock, just copy the value so it doesn't change between the + # check and join later. + queue = self._queue if queue is None: return True diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index 4d3a07486..fb3f63c36 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -2829,9 +2829,11 @@ def test_metric_call_racing_stop_does_not_land_behind_the_stop_sentinel(self): # The actual regression this guards against: a metric call racing # disable_background_sender() must never land in the queue BEHIND # the Stop sentinel -- where it would be silently lost once the - # sender reaches Stop and exits without ever seeing it -- because - # self._queue is closed to new producers atomically with appending - # Stop (see _stop_sender_thread), not sometime after the drain. + # sender reaches Stop and exits without ever seeing it. Rejecting + # new puts is gated on _sender_stopping (set before Stop is + # appended, both under the same lock _send_to_server() re-checks -- + # see _stop_sender_thread), not on self._queue itself, which stays + # non-None until the sender actually finishes with it. statsd = DogStatsd(disable_background_sender=False, disable_telemetry=True) release = threading.Event() entered_first_send = threading.Event() @@ -2863,20 +2865,21 @@ def call_second(): stopper.start() second_thread = None try: - # _stop_sender_thread's atomic close-and-append-Stop only needs - # _buffer_lock -- which the sender isn't holding while blocked in - # the wedged send above -- so it doesn't wait for the drain. - # Poll briefly rather than a fixed sleep, to stay fast without - # being flaky on a slower machine. - closed = False + # _stop_sender_thread sets _sender_stopping (which is what + # _send_to_server() rejects new puts on) well before the drain + # can possibly finish -- it only needs to set a flag and append + # Stop, not wait for the wedged send. Poll briefly rather than a + # fixed sleep, to stay fast without being flaky on a slower + # machine. + signalled = False deadline = time.time() + 2.0 while time.time() < deadline: - if statsd._queue is None: - closed = True + if statsd._sender_stopping.is_set(): + signalled = True break time.sleep(0.01) - self.assertTrue(closed, "self._queue must close to producers promptly, without waiting for the drain") - self.assertIsNotNone(statsd._active_queue, "the real, still-draining queue must remain reachable") + self.assertTrue(signalled, "the shutdown signal must be set promptly, without waiting for the drain") + self.assertIsNotNone(statsd._queue, "the real, still-draining queue must remain reachable") self.assertTrue(stopper.is_alive(), "stop() must still be waiting on the wedged sender") second_thread = threading.Thread(target=call_second) @@ -2986,11 +2989,7 @@ def blocking_send(self, data): try: self.assertIs(self._call_bounded(statsd.stop, (0.2,)), False) self.assertTrue(wedged.is_alive()) - # self._queue itself is already closed to producers at this point - # (see _stop_sender_thread) -- self._active_queue is what must be - # retained so _start_sender_thread() knows a sender is still - # running and state stays coherent. - self.assertIsNotNone(statsd._active_queue, "active queue must be retained so state stays coherent") + self.assertIsNotNone(statsd._queue, "queue must be retained so state stays coherent") statsd.enable_background_sender() # This identity check IS the proof there's no orphan: if a second @@ -3085,10 +3084,7 @@ def blocking_send(self, data): try: self.assertIs(self._call_bounded(statsd.stop, (0.2,)), False) - # self._queue is already closed to producers at this point (see - # _stop_sender_thread); self._active_queue is what still tracks - # the sender as active while it finishes up. - self.assertIsNotNone(statsd._active_queue) + self.assertIsNotNone(statsd._queue) self.assertIs(statsd._sender_thread, wedged) # Caller moved on and never called stop() again; the wedged send @@ -3099,7 +3095,6 @@ def blocking_send(self, data): # The exit must have healed the client state so a re-enable works. self.assertIsNone(statsd._queue) - self.assertIsNone(statsd._active_queue) self.assertIsNone(statsd._sender_thread) statsd.enable_background_sender() self.assertIsNot( From a60989fa7869cce6b59c89115fedc3e4d85179c1 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 22 Sep 2026 12:59:47 +0100 Subject: [PATCH 28/30] Give SenderQueue a closing signal so a wedged put() can't block stop()'s timeout --- datadog/dogstatsd/base.py | 12 +++++++ datadog/dogstatsd/sender_queue.py | 51 +++++++++++++++++++++++--- tests/unit/dogstatsd/test_statsd.py | 55 +++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 5 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index 11fad3978..6b84e3d03 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -2221,6 +2221,18 @@ def _stop_sender_thread(self, timeout=None): # re-checks the deadline as soon as this wakes it. self._sender_stopping.set() + # A producer already inside put(), waiting for room in a full queue, + # would otherwise hold _buffer_lock for as long as that wait lasts -- + # up to sender_queue_timeout, or forever if it's None -- and block + # the lock acquisition just below, making the timeout parameter to + # this very method meaningless. SenderQueue.close() needs only the + # queue's own internal lock (never _buffer_lock) to wake any such + # wait immediately, so it always runs promptly here regardless of + # what a stuck producer is doing. + queue_to_close = self._queue + if queue_to_close is not None: + queue_to_close.close() + # Lock ensures that nothing gets added to the queue after the check # above -- see _send_to_server(), which takes this same lock and # re-checks _sender_stopping before it puts. diff --git a/datadog/dogstatsd/sender_queue.py b/datadog/dogstatsd/sender_queue.py index 048512a20..23c2873d6 100644 --- a/datadog/dogstatsd/sender_queue.py +++ b/datadog/dogstatsd/sender_queue.py @@ -115,6 +115,12 @@ def __init__(self, maxsize, expiry_seconds, on_drop_queue_full, on_drop_expired, # all tasks have been dropped or sent. self._unfinished_tasks = 0 + # Set by close(): tells a put() that is (or will be) waiting for room + # to stop waiting immediately instead of riding out put_timeout, or + # forever if put_timeout is None. See close()'s docstring for why + # this exists. + self._closing = False + # The items currently handed out by get() and not yet finished via # requeue_front() or task_done(), keyed by id(item). SenderQueue is # single-consumer by design (one background sender thread), so this @@ -178,18 +184,23 @@ def put(self, item): number, or not at all if it's 0 (the default) -- then falls back to evicting the oldest entry (see _make_room_locked()) if the queue is still full once the wait is over. Either way, put() never rejects - the payload outright. + the payload outright. A close() call (from any thread) cuts any of + that waiting short immediately, regardless of put_timeout. """ with self._not_empty: if item is not Stop and self._maxsize > 0 and len(self._deque) >= self._maxsize: - if self._put_timeout is None: + if self._closing: + pass # Already closing: don't wait at all, straight to eviction below. + elif self._put_timeout is None: # Wait forever: an explicit opt-in to unbounded - # backpressure on the calling thread. - while len(self._deque) >= self._maxsize: + # backpressure on the calling thread. close() is what + # keeps this from actually being forever once a shutdown + # is underway. + while len(self._deque) >= self._maxsize and not self._closing: self._not_full.wait() elif self._put_timeout > 0: deadline = monotonic() + self._put_timeout - while len(self._deque) >= self._maxsize: + while len(self._deque) >= self._maxsize and not self._closing: remaining = deadline - monotonic() if remaining <= 0: break @@ -204,6 +215,36 @@ def put(self, item): self._unfinished_tasks += 1 self._not_empty.notify() + def close(self): + # type: () -> None + """Wake any put() currently waiting for room, immediately. + + A put() blocked waiting for space in a full queue holds no lock this + method needs -- Condition.wait() releases the underlying lock while + waiting -- so this always runs promptly, even while some other + thread is stuck inside that wait (that stuck thread is exactly what + this is for). Without it, a put() with put_timeout=None waits + forever for room that will never open up once nothing is draining + the queue, and even a bounded put_timeout can outlast whatever + timeout a caller trying to shut things down asked for -- see + DogStatsd._stop_sender_thread(), which calls this before it needs + the *caller's* lock (_buffer_lock) that a stuck put() would + otherwise be holding for the entire wait. + + Sticky: once closed, no future put() on this queue ever waits for + room again, regardless of put_timeout -- it goes straight to + eviction, like put_timeout=0. There is no matching "reopen": a fresh + shutdown starts with a fresh SenderQueue instead. + + This does not stop put()/get() from working, and does not reject or + drop anything by itself -- it only ends a wait early. Refusing new + payloads outright is the caller's job (see DogStatsd._send_to_server(), + which checks _sender_stopping before ever calling put()). + """ + with self._not_full: + self._closing = True + self._not_full.notify_all() + def requeue_front(self, item): # type: (QueuedItem) -> None """Put an in-flight payload back at the front after a failed send attempt. diff --git a/tests/unit/dogstatsd/test_statsd.py b/tests/unit/dogstatsd/test_statsd.py index fb3f63c36..73f877455 100644 --- a/tests/unit/dogstatsd/test_statsd.py +++ b/tests/unit/dogstatsd/test_statsd.py @@ -2902,6 +2902,61 @@ def call_second(): "the racing payload must have been delivered (via a direct send), not lost behind Stop", ) + def test_stop_timeout_is_bounded_when_a_producer_is_wedged_waiting_for_room(self): + # The actual regression this guards against: a producer thread + # blocked inside SenderQueue.put(), waiting for room in a full + # queue, holds _buffer_lock for as long as that wait lasts -- up to + # sender_queue_timeout, or forever if it's None. Without + # SenderQueue.close() waking it immediately, _stop_sender_thread() + # can't even acquire that lock to append Stop, so stop(timeout) + # ignores its own timeout entirely (bounded only by whatever else + # eventually frees the stuck producer -- nothing, in the worst case). + statsd = DogStatsd( + disable_background_sender=False, + disable_telemetry=True, + sender_queue_size=1, + sender_queue_timeout=None, # wait forever for room if not interrupted + ) + release = threading.Event() + entered_send = threading.Event() + + def blocking_xmit(packet, queue_mode=False): + entered_send.set() + release.wait(30.0) + return True + + statsd._xmit_packet_with_telemetry = blocking_xmit + statsd._send_to_server("first:1|c") + self.assertTrue(entered_send.wait(5.0), "sender never picked up first") + # get() already popped "first" off the queue while the sender is + # wedged inside the mocked send -- refill it to maxsize=1 so the + # NEXT put() has nowhere to go. + statsd._send_to_server("second:1|c") + self.assertEqual(statsd._queue.qsize(), 1) + + producer_started = threading.Event() + + def producer(): + producer_started.set() + statsd._send_to_server("third:1|c") + + producer_thread = threading.Thread(target=producer) + producer_thread.start() + self.assertTrue(producer_started.wait(5.0)) + # Give the producer a moment to actually reach put()'s internal wait + # (queue full, sender_queue_timeout=None) before racing stop(). + time.sleep(0.2) + + try: + t0 = time.time() + self.assertIs(self._call_bounded(statsd.stop, (1.0,), limit=3.0), False) + elapsed = time.time() - t0 + self.assertLess(elapsed, 3.0, "stop(1) must not hang behind a producer wedged in put()") + self.assertGreaterEqual(elapsed, 1.0, "stop(1) should still take roughly its own timeout, not return early") + finally: + release.set() + producer_thread.join(timeout=5.0) + def test_stop_timeout_is_bounded_while_the_sender_holds_the_socket_lock(self): # The wedge that matters in practice: the sender is parked inside a # blocking send() and therefore owns _socket_lock. stop()'s own From 2e97faca6bb0b4a2da250136113bad3a45957921 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 22 Sep 2026 13:10:28 +0100 Subject: [PATCH 29/30] Remove extraneous comments --- datadog/dogstatsd/base.py | 44 ++++++--------------------------------- 1 file changed, 6 insertions(+), 38 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index 6b84e3d03..a59f1db7a 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -193,26 +193,13 @@ def reverse(self): SENDER_RETRY_MAX_BACKOFF = 60.0 # How long an *unbounded* shutdown (pre_fork(), or stop()/wait_for_pending() # called with timeout=None) still waits for a payload stuck in the retry -# loop above to resolve, before giving up on it. Without some bound here, a -# payload that can never succeed (the Agent permanently unreachable, plus a -# replay-safe payload, which never expires) would starve the Stop sentinel -# and hang the shutdown forever. Reuses SENDER_RETRY_MAX_BACKOFF's magnitude: -# that's already the longest gap between two retry attempts in steady state, -# so an unbounded shutdown should be at least that patient before giving up. +# loop above to resolve, before giving up on it. SENDER_UNBOUNDED_STOP_GRACE_SECONDS = SENDER_RETRY_MAX_BACKOFF # How often the sender retries a connection once a shutdown has been # requested but its deadline (the caller's own timeout, or the grace period -# above) hasn't passed yet. Deliberately much shorter than the normal -# backoff cap, so a still-recovering Agent gets drained before the shutdown -# gives up, without hammering a connection that keeps failing. +# above) hasn't passed yet. SENDER_STOP_RETRY_INTERVAL = 0.5 - -# How long (in seconds) a non-replay-safe payload may sit in the background -# sender queue before it's considered stale and dropped instead of sent. -# Payloads that carry their own explicit timestamp (replay-safe) are exempt: -# delivering those late doesn't change what they mean, so they're kept -# around until they can actually be sent. This is the default for -# sender_queue_expiry_seconds; it can be overridden per client. +# Default for sender_queue_expiry_seconds; it can be overridden per client. PENDING_PAYLOAD_EXPIRY_SECONDS = 10.0 # Errors seen while sending on an already-connected socket that indicate the # peer went away (e.g. the agent crashed/restarted). These are worth a single @@ -2212,23 +2199,10 @@ def _stop_sender_thread(self, timeout=None): # payload that can genuinely never succeed. grace = SENDER_UNBOUNDED_STOP_GRACE_SECONDS if timeout is None else timeout self._sender_stop_deadline = monotonic() + grace - # Setting this is also what makes _send_to_server() reject any - # FURTHER producer call outright (see there) instead of letting it - # land behind the Stop sentinel appended below and be lost once the - # sender reaches Stop and exits, and what breaks the sender out of a - # retry backoff (which can be as long as SENDER_RETRY_MAX_BACKOFF) - # instead of having to wait that out -- see _sender_main_loop, which - # re-checks the deadline as soon as this wakes it. + # Setting makes _send_to_server() reject any FURTHER producer call outright. self._sender_stopping.set() - # A producer already inside put(), waiting for room in a full queue, - # would otherwise hold _buffer_lock for as long as that wait lasts -- - # up to sender_queue_timeout, or forever if it's None -- and block - # the lock acquisition just below, making the timeout parameter to - # this very method meaningless. SenderQueue.close() needs only the - # queue's own internal lock (never _buffer_lock) to wake any such - # wait immediately, so it always runs promptly here regardless of - # what a stuck producer is doing. + # Set temporary var to protect from concurrent access to self._queue. queue_to_close = self._queue if queue_to_close is not None: queue_to_close.close() @@ -2252,13 +2226,7 @@ def _stop_sender_thread(self, timeout=None): thread.join(timeout) if thread.is_alive(): - # Timed out. Leave _queue in place: it is what stops - # _start_sender_thread() from minting a second sender, and it keeps - # wait_for_pending() targeting the real queue -- _send_to_server() - # itself already rejects new puts via the _sender_stopping check - # above, so nothing new lands on it in the meantime. The sender - # thread clears this state itself when it eventually exits (see - # _sender_main_loop). + # Timed out. Leave _queue in place. return False # The thread exited, but that alone doesn't mean it drained: it may From f87f98b8f826a7e9c3edf3b4facc23e790eaa918 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 22 Sep 2026 13:34:14 +0100 Subject: [PATCH 30/30] Fix trailing whitespace (flake8 W291) --- datadog/dogstatsd/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datadog/dogstatsd/base.py b/datadog/dogstatsd/base.py index a59f1db7a..41827c8e9 100644 --- a/datadog/dogstatsd/base.py +++ b/datadog/dogstatsd/base.py @@ -197,7 +197,7 @@ def reverse(self): SENDER_UNBOUNDED_STOP_GRACE_SECONDS = SENDER_RETRY_MAX_BACKOFF # How often the sender retries a connection once a shutdown has been # requested but its deadline (the caller's own timeout, or the grace period -# above) hasn't passed yet. +# above) hasn't passed yet. SENDER_STOP_RETRY_INTERVAL = 0.5 # Default for sender_queue_expiry_seconds; it can be overridden per client. PENDING_PAYLOAD_EXPIRY_SECONDS = 10.0 @@ -2199,7 +2199,7 @@ def _stop_sender_thread(self, timeout=None): # payload that can genuinely never succeed. grace = SENDER_UNBOUNDED_STOP_GRACE_SECONDS if timeout is None else timeout self._sender_stop_deadline = monotonic() + grace - # Setting makes _send_to_server() reject any FURTHER producer call outright. + # Setting makes _send_to_server() reject any FURTHER producer call outright. self._sender_stopping.set() # Set temporary var to protect from concurrent access to self._queue.