Coverage for integrations / remote_desktop / frame_capture.py: 53.1%
175 statements
« prev ^ index » next coverage.py v7.14.0, created at 2026-05-12 04:49 +0000
1"""
2Frame Capture — High-FPS cross-platform screen capture with circuit breaker fallback.
4Tiered capture backends:
5 Tier 1: dxcam (Windows GPU-accelerated, 240+ FPS) — optional
6 Tier 2: mss (cross-platform, 30-60 FPS) — primary
7 Tier 3: pyautogui.screenshot() (existing fallback)
9Reuses:
10 - integrations/vision/frame_store.py → compute_frame_difference() for skip-unchanged
11 - integrations/vlm/vlm_adapter.py:34 → circuit breaker pattern
12"""
14import io
15import logging
16import threading
17import time
18from dataclasses import dataclass, field
19from typing import Generator, Optional, Tuple
21logger = logging.getLogger('hevolve.remote_desktop')
23# ── Optional dependencies (guarded imports) ─────────────────────
25_mss = None
26_dxcam = None
27_pyautogui = None
28_PIL_Image = None
30try:
31 import mss as _mss_module
32 _mss = _mss_module
33except ImportError:
34 pass
36try:
37 import dxcam as _dxcam_module
38 _dxcam = _dxcam_module
39except ImportError:
40 pass
42try:
43 import pyautogui as _pyautogui_module
44 _pyautogui = _pyautogui_module
45except ImportError:
46 pass
48try:
49 from PIL import Image as _PIL_Image_module
50 _PIL_Image = _PIL_Image_module
51except ImportError:
52 pass
54# Frame difference utility (from vision/frame_store.py)
55try:
56 from integrations.vision.frame_store import compute_frame_difference
57except ImportError:
58 def compute_frame_difference(frame1, frame2):
59 """Fallback: byte-level comparison."""
60 if len(frame1) != len(frame2):
61 return 1.0
62 diff = sum(abs(a - b) for a, b in zip(frame1[:1000], frame2[:1000]))
63 return min(diff / (255 * min(len(frame1), 1000)), 1.0)
66# ── Configuration ───────────────────────────────────────────────
68@dataclass
69class FrameConfig:
70 max_fps: int = 30
71 quality: int = 80 # JPEG quality (1-100)
72 scale_factor: float = 1.0 # Downscale factor (0.5 = half size)
73 min_change_threshold: float = 0.01 # Skip frame if < 1% changed
74 keyframe_interval: int = 30 # Force keyframe every N frames
75 adaptive_interval: bool = True # Backoff for static scenes
76 max_backoff_seconds: float = 2.0 # Max interval between frames
79# ── Circuit Breaker (vlm_adapter.py:34 pattern) ────────────────
81class _CaptureCircuitBreaker:
82 """Track failures per backend, open circuit after threshold."""
84 def __init__(self, threshold: int = 5):
85 self.threshold = threshold
86 self._failures: dict = {} # backend_name → count
87 self._open: set = set()
89 def record_failure(self, backend: str) -> None:
90 self._failures[backend] = self._failures.get(backend, 0) + 1
91 if self._failures[backend] >= self.threshold:
92 self._open.add(backend)
93 logger.warning(f"Circuit breaker OPEN for {backend}")
95 def record_success(self, backend: str) -> None:
96 self._failures[backend] = 0
97 self._open.discard(backend)
99 def is_open(self, backend: str) -> bool:
100 return backend in self._open
102 def reset(self, backend: str) -> None:
103 self._failures.pop(backend, None)
104 self._open.discard(backend)
107# ── Frame Capture ───────────────────────────────────────────────
109class FrameCapture:
110 """Cross-platform screen capture with tiered fallback."""
112 def __init__(self, config: Optional[FrameConfig] = None):
113 self.config = config or FrameConfig()
114 self._circuit = _CaptureCircuitBreaker()
115 self._lock = threading.Lock()
116 self._running = False
117 self._last_frame: Optional[bytes] = None
118 self._frame_count = 0
119 self._dxcam_instance = None
120 self._mss_instance = None
122 def get_screen_size(self) -> Tuple[int, int]:
123 """Get primary screen resolution (width, height)."""
124 if _mss:
125 try:
126 with _mss.mss() as sct:
127 monitor = sct.monitors[1] # Primary monitor
128 return monitor['width'], monitor['height']
129 except Exception:
130 pass
131 if _pyautogui:
132 try:
133 size = _pyautogui.size()
134 return size.width, size.height
135 except Exception:
136 pass
137 return 1920, 1080 # Default fallback
139 def capture_frame(self) -> Optional[bytes]:
140 """Capture single frame as JPEG bytes.
142 Uses circuit breaker pattern — tries backends in order,
143 skips backends with open circuits.
145 Returns:
146 JPEG bytes or None if all backends failed.
147 """
148 # Tier 1: DXCam (Windows GPU)
149 if _dxcam and not self._circuit.is_open('dxcam'):
150 try:
151 frame = self._capture_dxcam()
152 if frame:
153 self._circuit.record_success('dxcam')
154 return frame
155 except Exception as e:
156 self._circuit.record_failure('dxcam')
157 logger.debug(f"DXCam capture failed: {e}")
159 # Tier 2: mss (cross-platform)
160 if _mss and not self._circuit.is_open('mss'):
161 try:
162 frame = self._capture_mss()
163 if frame:
164 self._circuit.record_success('mss')
165 return frame
166 except Exception as e:
167 self._circuit.record_failure('mss')
168 logger.debug(f"MSS capture failed: {e}")
170 # Tier 3: pyautogui (existing fallback)
171 if _pyautogui and not self._circuit.is_open('pyautogui'):
172 try:
173 frame = self._capture_pyautogui()
174 if frame:
175 self._circuit.record_success('pyautogui')
176 return frame
177 except Exception as e:
178 self._circuit.record_failure('pyautogui')
179 logger.debug(f"PyAutoGUI capture failed: {e}")
181 logger.error("All capture backends failed")
182 return None
184 def _capture_dxcam(self) -> Optional[bytes]:
185 """DXCam GPU-accelerated capture (Windows only)."""
186 if self._dxcam_instance is None:
187 self._dxcam_instance = _dxcam.create()
188 frame = self._dxcam_instance.grab()
189 if frame is None:
190 return None
191 return self._encode_numpy_frame(frame)
193 def _capture_mss(self) -> Optional[bytes]:
194 """MSS cross-platform capture."""
195 if self._mss_instance is None:
196 self._mss_instance = _mss.mss()
197 monitor = self._mss_instance.monitors[1]
198 sct_img = self._mss_instance.grab(monitor)
199 # mss returns BGRA; convert to RGB JPEG
200 if _PIL_Image:
201 img = _PIL_Image.frombytes('RGB', sct_img.size,
202 sct_img.bgra, 'raw', 'BGRX')
203 return self._encode_pil_image(img)
204 # Fallback: raw PNG from mss
205 return _mss.tools.to_png(sct_img.rgb, sct_img.size)
207 def _capture_pyautogui(self) -> Optional[bytes]:
208 """PyAutoGUI screenshot fallback."""
209 screenshot = _pyautogui.screenshot()
210 return self._encode_pil_image(screenshot)
212 def _encode_pil_image(self, img) -> bytes:
213 """Encode PIL Image to JPEG bytes with configured quality and scale."""
214 if self.config.scale_factor != 1.0:
215 new_size = (
216 int(img.width * self.config.scale_factor),
217 int(img.height * self.config.scale_factor),
218 )
219 img = img.resize(new_size, _PIL_Image.LANCZOS if _PIL_Image else 1)
221 buf = io.BytesIO()
222 img.save(buf, format='JPEG', quality=self.config.quality, optimize=True)
223 return buf.getvalue()
225 def _encode_numpy_frame(self, frame) -> bytes:
226 """Encode numpy array (RGB/BGR) to JPEG bytes."""
227 if _PIL_Image:
228 img = _PIL_Image.fromarray(frame)
229 return self._encode_pil_image(img)
230 # Fallback: try cv2
231 try:
232 import cv2
233 _, buf = cv2.imencode('.jpg', frame,
234 [cv2.IMWRITE_JPEG_QUALITY, self.config.quality])
235 return buf.tobytes()
236 except ImportError:
237 return None
239 def capture_loop(self) -> Generator[bytes, None, None]:
240 """Yield JPEG frames at configured FPS, skipping unchanged frames.
242 Uses compute_frame_difference() from vision/frame_store.py.
243 Adaptive interval: backs off for static scenes (vision_service.py:36-37 pattern).
244 """
245 self._running = True
246 interval = 1.0 / self.config.max_fps
247 adaptive_interval = interval
248 self._frame_count = 0
250 try:
251 while self._running:
252 start = time.monotonic()
254 frame = self.capture_frame()
255 if frame is None:
256 time.sleep(interval)
257 continue
259 self._frame_count += 1
261 # Skip unchanged frames (unless keyframe)
262 is_keyframe = (self._frame_count % self.config.keyframe_interval == 0)
263 if self._last_frame and not is_keyframe:
264 try:
265 diff = compute_frame_difference(
266 self._last_frame[:4096], frame[:4096])
267 if diff < self.config.min_change_threshold:
268 # Static scene → adaptive backoff
269 if self.config.adaptive_interval:
270 adaptive_interval = min(
271 adaptive_interval * 1.5,
272 self.config.max_backoff_seconds,
273 )
274 elapsed = time.monotonic() - start
275 sleep_time = max(0, adaptive_interval - elapsed)
276 time.sleep(sleep_time)
277 continue
278 except Exception:
279 pass # On error, send the frame anyway
281 # Scene changed → reset adaptive interval
282 adaptive_interval = interval
283 self._last_frame = frame
285 yield frame
287 elapsed = time.monotonic() - start
288 sleep_time = max(0, interval - elapsed)
289 time.sleep(sleep_time)
290 finally:
291 self._running = False
292 self._cleanup()
294 def stop(self) -> None:
295 """Stop the capture loop."""
296 self._running = False
298 def is_running(self) -> bool:
299 return self._running
301 def get_stats(self) -> dict:
302 """Get capture statistics."""
303 return {
304 'running': self._running,
305 'frame_count': self._frame_count,
306 'config': {
307 'max_fps': self.config.max_fps,
308 'quality': self.config.quality,
309 'scale_factor': self.config.scale_factor,
310 },
311 'backends': {
312 'dxcam': _dxcam is not None and not self._circuit.is_open('dxcam'),
313 'mss': _mss is not None and not self._circuit.is_open('mss'),
314 'pyautogui': _pyautogui is not None and not self._circuit.is_open('pyautogui'),
315 },
316 }
318 def _cleanup(self) -> None:
319 """Release capture resources."""
320 if self._dxcam_instance:
321 try:
322 self._dxcam_instance.stop()
323 except Exception:
324 pass
325 self._dxcam_instance = None
326 if self._mss_instance:
327 try:
328 self._mss_instance.close()
329 except Exception:
330 pass
331 self._mss_instance = None