tokio_prompt_orchestrator/enhanced.rs
1//! Enhanced resilience and performance features for the pipeline.
2//!
3//! This module groups production-grade primitives that operate at the
4//! nanosecond-to-microsecond range and are safe to use concurrently across
5//! multiple pipeline stages:
6//!
7//! | Sub-module | Purpose |
8//! |------------|---------|
9//! | [`cache`] | In-process LRU cache with TTL for inference results |
10//! | [`circuit_breaker`] | Prevents cascading failures; opens on threshold, probes recovery with exponential backoff |
11//! | [`dedup`] | Request coalescing — collapses byte-identical in-flight prompts |
12//! | [`semantic_dedup`] | Near-duplicate detection using SimHash LSH — catches paraphrases with configurable Hamming-distance threshold; when a near-duplicate is found, logs the similarity score and optionally returns the cached result |
13//! | [`priority`] | Four-level priority queue for request scheduling |
14//! | [`rate_limit`] | Token-bucket rate limiter with configurable burst |
15//! | [`retry`] | Exponential back-off with jitter and configurable attempt cap |
16//! | [`smart_batch`] | Adaptive micro-batching with prefix-based similarity grouping |
17//! | [`tournament`] | Multi-provider tournament: fan-out and return the best response |
18//!
19//! All types are `Clone` and `Send + Sync`, suitable for sharing across Tokio tasks.
20
21pub mod adaptive_timeout;
22pub mod bulkhead;
23pub mod cache;
24pub mod circuit_breaker;
25pub mod dedup;
26pub mod dlq_replay;
27pub mod priority;
28pub mod rate_limit;
29pub mod retry;
30pub mod semantic_dedup;
31pub mod smart_batch;
32pub mod tournament;
33
34// Re-exports
35pub use adaptive_timeout::AdaptiveTimeout;
36pub use bulkhead::{Bulkhead, BulkheadPermit};
37pub use cache::{cache_key, CacheLayer};
38pub use circuit_breaker::{CircuitBreaker, CircuitStatus};
39pub use dedup::{dedup_key, DeduplicationResult, DeduplicationToken, Deduplicator, DEDUP_CANCELLED_SENTINEL};
40pub use dlq_replay::{DlqReplayScheduler, ReplayEntry};
41pub use priority::{Priority, PriorityQueue, QueueStats};
42pub use rate_limit::RateLimiter;
43pub use retry::{retry_if, RetryPolicy};
44pub use semantic_dedup::{SemanticDeduplicator, SimHashFingerprint};
45pub use smart_batch::{BatchConfig, BatcherStats, SmartBatcher};
46pub use tournament::{
47 FastestResponseScorer, KeywordDensityScorer, LongestResponseScorer, ResponseScorer,
48 TournamentConfig, TournamentResult, TournamentRunner, TournamentStats,
49};