Skip to main content

Metrics

tokio_prompt_orchestrator::metrics

Struct Metrics

pub struct Metrics {Show 26 fields
    pub registry: Registry,
    pub requests_total: CounterVec,
    pub requests_shed: CounterVec,
    pub requests_dropped: CounterVec,
    pub errors_total: CounterVec,
    pub stage_duration: HistogramVec,
    pub queue_depth: IntGaugeVec,
    pub inference_cost_usd: Counter,
    pub ttft_seconds: HistogramVec,
    pub dedup_hits_total: Counter,
    pub dedup_waiters_unblocked_total: Counter,
    pub inference_timeouts_total: Counter,
    pub requests_expired_total: Counter,
    pub dlq_lock_poisoned_total: Counter,
    pub rag_requests_expired_total: Counter,
    pub circuit_breaker_state_transitions_total: CounterVec,
    pub circuit_breaker_requests_rejected_total: Counter,
    pub config_reload_errors_total: Counter,
    pub worker_errors_total: CounterVec,
    pub cache_hits_total: Counter,
    pub cache_misses_total: Counter,
    pub rate_limiter_tokens_remaining: GaugeVec,
    pub config_reload_duration_seconds: HistogramVec,
    pub session_affinity_hits_total: Counter,
    pub session_affinity_misses_total: Counter,
    pub dedup_hash_collisions_total: Counter,
}

Expand description

All Prometheus metrics for the orchestrator, bundled together so they can be stored in a single OnceLock and initialised atomically.

Fields§

§registry: Registry

Prometheus registry that owns all metric descriptors.

§requests_total: CounterVec

Total requests processed per stage.

§requests_shed: CounterVec

Requests shed (queue full) per stage.

§requests_dropped: CounterVec

Requests dropped due to full queue, labelled by stage.

§errors_total: CounterVec

Errors by stage and error type.

§stage_duration: HistogramVec

Stage processing latency histogram.

§queue_depth: IntGaugeVec

Current queue depth per stage.

§inference_cost_usd: Counter

Cumulative USD cost of all inference calls.

§ttft_seconds: HistogramVec

Time from request start to first streaming token, labelled by worker and model.

§dedup_hits_total: Counter

Total dedup hits (in-progress or cached responses returned without new inference).

§dedup_waiters_unblocked_total: Counter

Total dedup waiters unblocked (requests that waited and received a broadcast result).

§inference_timeouts_total: Counter

Total inference timeouts.

§requests_expired_total: Counter

Requests dropped because their deadline had already passed at dequeue time.

§dlq_lock_poisoned_total: Counter

Incremented when a DeadLetterQueue mutex is recovered from a poisoned state.

§rag_requests_expired_total: Counter

Incremented when a RAG-stage request is dropped because its deadline expired.

§circuit_breaker_state_transitions_total: CounterVec

Circuit breaker state transitions, labelled by target state.

§circuit_breaker_requests_rejected_total: Counter

Requests rejected by the circuit breaker (open state).

§config_reload_errors_total: Counter

Config hot-reload attempts that failed validation or parse.

§worker_errors_total: CounterVec

Errors returned by a specific named worker, labelled by worker name.

§cache_hits_total: Counter

Cache hits on the response cache (in-memory or Redis).

§cache_misses_total: Counter

Cache misses on the response cache (in-memory or Redis).

§rate_limiter_tokens_remaining: GaugeVec

Current number of tokens remaining in the rate-limiter bucket per session.

§config_reload_duration_seconds: HistogramVec

Duration of config hot-reload operations (parse + validate), in seconds.

§session_affinity_hits_total: Counter

Session affinity routing hits (request routed to its preferred shard).

§session_affinity_misses_total: Counter

Session affinity routing misses (preferred shard unavailable, rerouted).

§dedup_hash_collisions_total: Counter

Dedup hash collisions: two different prompts produced the same hash key.

Auto Trait Implementations§

impl !RefUnwindSafe for Metrics

impl !UnwindSafe for Metrics

impl Freeze for Metrics

impl Send for Metrics

impl Sync for Metrics

impl Unpin for Metrics

impl UnsafeUnpin for Metrics

Blanket Implementations§

impl<T> Any for T
where T: 'static + ?Sized,

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more

impl<T> Borrow<T> for T
where T: ?Sized,

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more

impl<T> BorrowMut<T> for T
where T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more

impl<ST, DT> CastableFrom<ST, Initialized, Initialized> for DT
where ST: ?Sized, DT: ?Sized,

impl<ST, DT> CastableFrom<ST, Uninit, Uninit> for DT
where ST: ?Sized, DT: ?Sized,

impl<T> From<T> for T

fn from(t: T) -> T

Returns the argument unchanged.

impl<T> FutureExt for T

fn with_context(self, otel_cx: Context) -> WithContext<Self>

Attaches the provided Context to this type, returning a WithContext wrapper. Read more

fn with_current_context(self) -> WithContext<Self>

Attaches the current Context to this type, returning a WithContext wrapper. Read more

impl<T> Instrument for T

fn instrument(self, span: Span) -> Instrumented<Self>

Instruments this type with the provided [Span], returning an Instrumented wrapper. Read more

fn in_current_span(self) -> Instrumented<Self>

Instruments this type with the current Span, returning an Instrumented wrapper. Read more

impl<T, U> Into<U> for T
where U: From<T>,

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

impl<T> IntoRequest<T> for T

fn into_request(self) -> Request<T>

Wrap the input message T in a tonic::Request

impl<T> PolicyExt for T
where T: ?Sized,

fn and<P, B, E>(self, other: P) -> And<T, P>
where T: Sized + Policy<B, E>, P: Policy<B, E>,

Create a new Policy that returns [Action::Follow] only if self and other return Action::Follow. Read more

fn or<P, B, E>(self, other: P) -> Or<T, P>
where T: Sized + Policy<B, E>, P: Policy<B, E>,

Create a new Policy that returns [Action::Follow] if either self or other returns Action::Follow. Read more

impl<T> Read<Exclusive, BecauseExclusive> for T
where T: ?Sized,

impl<T> Same for T

type Output = T

Should always be Self

impl<T, U> TryFrom<U> for T
where U: Into<T>,

type Error = Infallible

The type returned in the event of a conversion error.

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

Performs the conversion.

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

fn vzip(self) -> V

impl<T> WithSubscriber for T

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>
where S: Into<Dispatch>,

Attaches the provided Subscriber to this type, returning a [WithDispatch] wrapper. Read more

fn with_current_subscriber(self) -> WithDispatch<Self>

Attaches the current default Subscriber to this type, returning a [WithDispatch] wrapper. Read more