Skip to content

Stream Queues

Consuming streams (offsets, offset stores, delivery semantics): see the Streams guide.

StreamConsumerConfig dataclass

How a route consumes a stream queue (subscriber(stream=...)).

Attributes:

Name Type Description
offset StreamOffset

where to start when offset_store holds nothing for this consumer (StreamOffset, or "first" / "last" / "next" / an int / a datetime). Default: "next".

consumer_name str | None

the consumer's identity: the key its offset is stored under (default: the queue name) and the x-stream-consumer-name consume argument. Set it when two routes or services consume the same stream through one store.

offset_store OffsetStore | None

where the processed offset is committed. None: an :class:InMemoryOffsetStore (reconnects resume, restarts start at offset again). Pass your own :class:OffsetStore, backed by your service's storage, to resume across restarts.

commit_every int

commit after this many newly processed messages.

commit_interval float | None

also commit when this many seconds passed since the last commit (checked when a message is settled; None: count only). stop() always commits. A crash replays at most the messages processed since the last commit.

Source code in src/rabbitkit/core/streams.py
@dataclass(frozen=True, slots=True)
class StreamConsumerConfig:
    """How a route consumes a stream queue (``subscriber(stream=...)``).

    Attributes:
        offset: where to start when *offset_store* holds nothing for this
            consumer (``StreamOffset``, or ``"first"`` / ``"last"`` /
            ``"next"`` / an int / a datetime). Default: ``"next"``.
        consumer_name: the consumer's identity: the key its offset is stored
            under (default: the queue name) and the ``x-stream-consumer-name``
            consume argument. Set it when two routes or services consume the
            same stream through one store.
        offset_store: where the processed offset is committed. ``None``: an
            :class:`InMemoryOffsetStore` (reconnects resume, restarts start at
            *offset* again). Pass your own :class:`OffsetStore`, backed by
            your service's storage, to resume across restarts.
        commit_every: commit after this many newly processed messages.
        commit_interval: also commit when this many seconds passed since the
            last commit (checked when a message is settled; ``None``: count
            only). ``stop()`` always commits. A crash replays at most the
            messages processed since the last commit.
    """

    offset: StreamOffset = field(default_factory=StreamOffset.next)
    consumer_name: str | None = None
    offset_store: OffsetStore | None = None
    commit_every: int = 100
    commit_interval: float | None = 5.0

    def __post_init__(self) -> None:
        if not isinstance(self.offset, StreamOffset):
            object.__setattr__(self, "offset", StreamOffset.parse(self.offset))
        if isinstance(self.commit_every, bool) or not isinstance(self.commit_every, int) or self.commit_every < 1:
            raise ValueError(f"StreamConsumerConfig.commit_every must be an int >= 1, got {self.commit_every!r}")
        if self.commit_interval is not None and not self.commit_interval > 0:
            raise ValueError(
                f"StreamConsumerConfig.commit_interval must be None or > 0 seconds, got {self.commit_interval!r}"
            )
        if self.consumer_name is not None:
            if not self.consumer_name:
                raise ValueError("StreamConsumerConfig.consumer_name must not be empty")
            validate_amqp_shortstr("StreamConsumerConfig.consumer_name", self.consumer_name)
        if self.offset_store is not None and not isinstance(self.offset_store, OffsetStore):
            raise TypeError(
                f"StreamConsumerConfig.offset_store must implement load(key) and save(key, offset), "
                f"got {type(self.offset_store).__name__}"
            )

    def to_consume_arguments(self) -> dict[str, Any]:
        """The ``basic.consume`` arguments for a first subscription."""
        args = self.offset.to_consume_arguments()
        if self.consumer_name is not None:
            args[STREAM_CONSUMER_NAME_ARG] = self.consumer_name
        return args

    def offset_key(self, queue: str) -> str:
        """The key this consumer's offset is stored under."""
        return self.consumer_name or queue

Methods:

to_consume_arguments() -> dict[str, Any]

The basic.consume arguments for a first subscription.

Source code in src/rabbitkit/core/streams.py
def to_consume_arguments(self) -> dict[str, Any]:
    """The ``basic.consume`` arguments for a first subscription."""
    args = self.offset.to_consume_arguments()
    if self.consumer_name is not None:
        args[STREAM_CONSUMER_NAME_ARG] = self.consumer_name
    return args

offset_key(queue: str) -> str

The key this consumer's offset is stored under.

Source code in src/rabbitkit/core/streams.py
def offset_key(self, queue: str) -> str:
    """The key this consumer's offset is stored under."""
    return self.consumer_name or queue

StreamOffset dataclass

Where a stream consumer starts when no stored offset exists.

Use the class methods, or :meth:parse::

StreamOffset.first()
StreamOffset.last()
StreamOffset.next()
StreamOffset.offset(42)
StreamOffset.timestamp(datetime(2026, 1, 1, tzinfo=UTC))
StreamOffset.parse("first")  # also an int or a datetime

A naive datetime is taken as UTC.

Source code in src/rabbitkit/core/streams.py
@dataclass(frozen=True, slots=True)
class StreamOffset:
    """Where a stream consumer starts when no stored offset exists.

    Use the class methods, or :meth:`parse`::

        StreamOffset.first()
        StreamOffset.last()
        StreamOffset.next()
        StreamOffset.offset(42)
        StreamOffset.timestamp(datetime(2026, 1, 1, tzinfo=UTC))
        StreamOffset.parse("first")  # also an int or a datetime

    A naive ``datetime`` is taken as UTC.
    """

    type: StreamOffsetType = StreamOffsetType.NEXT
    value: int | datetime | None = None

    def __post_init__(self) -> None:
        if not isinstance(self.type, StreamOffsetType):
            try:
                object.__setattr__(self, "type", StreamOffsetType(self.type))
            except ValueError:
                raise ValueError(
                    f"Unknown stream offset type {self.type!r}; expected one of {[t.value for t in StreamOffsetType]}"
                ) from None
        if self.type is StreamOffsetType.OFFSET:
            if isinstance(self.value, bool) or not isinstance(self.value, int):
                raise TypeError(f"An OFFSET stream offset needs an int value, got {self.value!r}")
            if self.value < 0:
                raise ValueError("Stream offset must be non-negative")
        elif self.type is StreamOffsetType.TIMESTAMP:
            if not isinstance(self.value, datetime):
                raise TypeError(f"A TIMESTAMP stream offset needs a datetime value, got {self.value!r}")
            if self.value.tzinfo is None:
                object.__setattr__(self, "value", self.value.replace(tzinfo=UTC))
        elif self.value is not None:
            raise ValueError(f"A {self.type.value.upper()} stream offset takes no value, got {self.value!r}")

    @classmethod
    def first(cls) -> StreamOffset:
        """Start from the oldest message the stream still retains."""
        return cls(type=StreamOffsetType.FIRST)

    @classmethod
    def last(cls) -> StreamOffset:
        """Start from the last chunk written (the most recent messages)."""
        return cls(type=StreamOffsetType.LAST)

    @classmethod
    def next(cls) -> StreamOffset:
        """Only messages published after subscribing (the broker default)."""
        return cls(type=StreamOffsetType.NEXT)

    @classmethod
    def offset(cls, value: int) -> StreamOffset:
        """Start from a numeric offset."""
        return cls(type=StreamOffsetType.OFFSET, value=value)

    @classmethod
    def timestamp(cls, value: datetime) -> StreamOffset:
        """Start from the first chunk published at or after *value* (naive = UTC).

        Chunk granularity: a few messages published just before *value* may
        be delivered too.
        """
        return cls(type=StreamOffsetType.TIMESTAMP, value=value)

    @classmethod
    def parse(cls, value: StreamOffset | str | int | datetime) -> StreamOffset:
        """``"first"`` / ``"last"`` / ``"next"``, an ``int`` offset, a ``datetime``,
        or a :class:`StreamOffset` (returned as is)."""
        if isinstance(value, StreamOffset):
            return value
        if isinstance(value, bool):
            raise TypeError(f"Not a stream offset: {value!r}")
        if isinstance(value, int):
            return cls.offset(value)
        if isinstance(value, datetime):
            return cls.timestamp(value)
        if isinstance(value, str):
            relative = {"first": cls.first, "last": cls.last, "next": cls.next}
            factory = relative.get(str(value.value if isinstance(value, StreamOffsetType) else value))
            if factory is None:
                raise ValueError(
                    f"Unknown stream offset {value!r}; expected 'first', 'last', 'next', an int offset or a datetime"
                )
            return factory()
        raise TypeError(f"Not a stream offset: {value!r}")

    def to_consume_arguments(self) -> dict[str, Any]:
        """The ``basic.consume`` arguments for this start position.

        A timestamp is passed as a ``datetime``: both pika and aio-pika encode
        it as an AMQP timestamp, which RabbitMQ reads as a time. (An integer
        would be read as a numeric offset.)
        """
        if self.type is StreamOffsetType.OFFSET or self.type is StreamOffsetType.TIMESTAMP:
            return {STREAM_OFFSET_ARG: self.value}
        return {STREAM_OFFSET_ARG: self.type.value}

    def __repr__(self) -> str:
        if self.value is not None:
            return f"StreamOffset({self.type.value}={self.value})"
        return f"StreamOffset({self.type.value})"

Methods:

first() -> StreamOffset classmethod

Start from the oldest message the stream still retains.

Source code in src/rabbitkit/core/streams.py
@classmethod
def first(cls) -> StreamOffset:
    """Start from the oldest message the stream still retains."""
    return cls(type=StreamOffsetType.FIRST)

last() -> StreamOffset classmethod

Start from the last chunk written (the most recent messages).

Source code in src/rabbitkit/core/streams.py
@classmethod
def last(cls) -> StreamOffset:
    """Start from the last chunk written (the most recent messages)."""
    return cls(type=StreamOffsetType.LAST)

next() -> StreamOffset classmethod

Only messages published after subscribing (the broker default).

Source code in src/rabbitkit/core/streams.py
@classmethod
def next(cls) -> StreamOffset:
    """Only messages published after subscribing (the broker default)."""
    return cls(type=StreamOffsetType.NEXT)

offset(value: int) -> StreamOffset classmethod

Start from a numeric offset.

Source code in src/rabbitkit/core/streams.py
@classmethod
def offset(cls, value: int) -> StreamOffset:
    """Start from a numeric offset."""
    return cls(type=StreamOffsetType.OFFSET, value=value)

timestamp(value: datetime) -> StreamOffset classmethod

Start from the first chunk published at or after value (naive = UTC).

Chunk granularity: a few messages published just before value may be delivered too.

Source code in src/rabbitkit/core/streams.py
@classmethod
def timestamp(cls, value: datetime) -> StreamOffset:
    """Start from the first chunk published at or after *value* (naive = UTC).

    Chunk granularity: a few messages published just before *value* may
    be delivered too.
    """
    return cls(type=StreamOffsetType.TIMESTAMP, value=value)

parse(value: StreamOffset | str | int | datetime) -> StreamOffset classmethod

"first" / "last" / "next", an int offset, a datetime, or a :class:StreamOffset (returned as is).

Source code in src/rabbitkit/core/streams.py
@classmethod
def parse(cls, value: StreamOffset | str | int | datetime) -> StreamOffset:
    """``"first"`` / ``"last"`` / ``"next"``, an ``int`` offset, a ``datetime``,
    or a :class:`StreamOffset` (returned as is)."""
    if isinstance(value, StreamOffset):
        return value
    if isinstance(value, bool):
        raise TypeError(f"Not a stream offset: {value!r}")
    if isinstance(value, int):
        return cls.offset(value)
    if isinstance(value, datetime):
        return cls.timestamp(value)
    if isinstance(value, str):
        relative = {"first": cls.first, "last": cls.last, "next": cls.next}
        factory = relative.get(str(value.value if isinstance(value, StreamOffsetType) else value))
        if factory is None:
            raise ValueError(
                f"Unknown stream offset {value!r}; expected 'first', 'last', 'next', an int offset or a datetime"
            )
        return factory()
    raise TypeError(f"Not a stream offset: {value!r}")

to_consume_arguments() -> dict[str, Any]

The basic.consume arguments for this start position.

A timestamp is passed as a datetime: both pika and aio-pika encode it as an AMQP timestamp, which RabbitMQ reads as a time. (An integer would be read as a numeric offset.)

Source code in src/rabbitkit/core/streams.py
def to_consume_arguments(self) -> dict[str, Any]:
    """The ``basic.consume`` arguments for this start position.

    A timestamp is passed as a ``datetime``: both pika and aio-pika encode
    it as an AMQP timestamp, which RabbitMQ reads as a time. (An integer
    would be read as a numeric offset.)
    """
    if self.type is StreamOffsetType.OFFSET or self.type is StreamOffsetType.TIMESTAMP:
        return {STREAM_OFFSET_ARG: self.value}
    return {STREAM_OFFSET_ARG: self.type.value}

StreamOffsetType

Bases: str, Enum

Where a stream consumer starts reading (RabbitMQ x-stream-offset).

  • FIRST: the oldest message still retained by the stream.
  • LAST: the last chunk written (the most recent messages).
  • NEXT: only messages published after the consumer subscribes (the broker's default).
  • OFFSET: a numeric offset.
  • TIMESTAMP: the first chunk published at or after a point in time.
Source code in src/rabbitkit/core/types.py
class StreamOffsetType(str, Enum):
    """Where a stream consumer starts reading (RabbitMQ ``x-stream-offset``).

    - FIRST: the oldest message still retained by the stream.
    - LAST: the last chunk written (the most recent messages).
    - NEXT: only messages published after the consumer subscribes (the
      broker's default).
    - OFFSET: a numeric offset.
    - TIMESTAMP: the first chunk published at or after a point in time.
    """

    FIRST = "first"
    LAST = "last"
    NEXT = "next"
    OFFSET = "offset"
    TIMESTAMP = "timestamp"

OffsetStore

Bases: Protocol

Where a stream consumer's processed offset is committed.

key identifies the consumer (StreamConsumerConfig.consumer_name, else the queue name). save stores the last processed offset; the consumer resumes at load(key) + 1.

rabbitkit ships only :class:InMemoryOffsetStore (offsets survive reconnects, not restarts). For restarts, implement these two methods over the storage your service already uses (its database, a key-value store, a file), ideally in the same transaction as the handler's own writes.

Called from whichever thread settles messages (SyncBroker: the I/O or a worker thread) or from a worker thread (AsyncBroker offloads every store except :class:InMemoryOffsetStore), so implementations must be thread-safe. save is only called with increasing offsets per key within one process.

Source code in src/rabbitkit/core/streams.py
@runtime_checkable
class OffsetStore(Protocol):
    """Where a stream consumer's processed offset is committed.

    ``key`` identifies the consumer (``StreamConsumerConfig.consumer_name``,
    else the queue name). ``save`` stores the last *processed* offset; the
    consumer resumes at ``load(key) + 1``.

    rabbitkit ships only :class:`InMemoryOffsetStore` (offsets survive
    reconnects, not restarts). For restarts, implement these two methods over
    the storage your service already uses (its database, a key-value store, a
    file), ideally in the same transaction as the handler's own writes.

    Called from whichever thread settles messages (SyncBroker: the I/O or a
    worker thread) or from a worker thread (AsyncBroker offloads every
    store except :class:`InMemoryOffsetStore`), so implementations must be
    thread-safe. ``save`` is only called with increasing offsets per key
    within one process.
    """

    def load(self, key: str) -> int | None:
        """The committed offset for *key*, or None when nothing is stored."""
        ...

    def save(self, key: str, offset: int) -> None:
        """Commit *offset* as the last processed offset for *key*."""
        ...

Methods:

load(key: str) -> int | None

The committed offset for key, or None when nothing is stored.

Source code in src/rabbitkit/core/streams.py
def load(self, key: str) -> int | None:
    """The committed offset for *key*, or None when nothing is stored."""
    ...

save(key: str, offset: int) -> None

Commit offset as the last processed offset for key.

Source code in src/rabbitkit/core/streams.py
def save(self, key: str, offset: int) -> None:
    """Commit *offset* as the last processed offset for *key*."""
    ...

InMemoryOffsetStore

Offsets kept in process memory: survive reconnects, not restarts.

The default when a stream route sets no offset_store.

Source code in src/rabbitkit/core/streams.py
class InMemoryOffsetStore:
    """Offsets kept in process memory: survive reconnects, not restarts.

    The default when a stream route sets no ``offset_store``.
    """

    def __init__(self) -> None:
        self._offsets: dict[str, int] = {}
        self._lock = threading.Lock()

    def load(self, key: str) -> int | None:
        with self._lock:
            return self._offsets.get(key)

    def save(self, key: str, offset: int) -> None:
        with self._lock:
            self._offsets[key] = offset

    def reset(self, key: str) -> None:
        """Forget *key*: its consumer starts at its configured offset again."""
        with self._lock:
            self._offsets.pop(key, None)

Methods:

reset(key: str) -> None

Forget key: its consumer starts at its configured offset again.

Source code in src/rabbitkit/core/streams.py
def reset(self, key: str) -> None:
    """Forget *key*: its consumer starts at its configured offset again."""
    with self._lock:
        self._offsets.pop(key, None)