diff --git a/package.json b/package.json
index ab63a37..49c8833 100644
--- a/package.json
+++ b/package.json
@@ -25,6 +25,7 @@
"@types/validator": "^13.7.17",
"axios": "^1.4.0",
"bson-objectid": "^2.0.4",
+ "clsx": "^2.1.1",
"cookies-next": "^4.1.1",
"cpr": "^3.0.1",
"date-fns": "^3.3.1",
@@ -55,6 +56,7 @@
"react-scripts": "5.0.1",
"react-use-websocket": "^4.7.0",
"strftime": "0.10.1",
+ "tailwind-merge": "^3.0.1",
"typescript": "^4.4.2",
"validator": "^13.9.0",
"video.js": "^8.10.0",
diff --git a/src/pages/LiveCameraPage.tsx b/src/pages/LiveCameraPage.tsx
index be2fd17..0ac3264 100644
--- a/src/pages/LiveCameraPage.tsx
+++ b/src/pages/LiveCameraPage.tsx
@@ -27,7 +27,11 @@ const LiveCameraPage = () => {
return (
-
+
);
}
diff --git a/src/services/frigate.proxy/frigate.schema.ts b/src/services/frigate.proxy/frigate.schema.ts
index 68c7d74..8b88b59 100644
--- a/src/services/frigate.proxy/frigate.schema.ts
+++ b/src/services/frigate.proxy/frigate.schema.ts
@@ -131,7 +131,7 @@ export type GetFrigateHost = z.infer
export type GetFrigateHostWConfig = GetFrigateHost & { config: FrigateConfig }
export type GetCamera = z.infer
export type GetCameraWHost = z.infer
-export type GetCameraWHostWConfig = GetCameraWHost & { config?: CameraConfig }
+export type GetCameraWHostWConfig = GetCameraWHost & { config: CameraConfig }
export type PutFrigateHost = z.infer
export type DeleteFrigateHost = z.infer
export type GetRole = z.infer
diff --git a/src/shared/components/players/JSMpegPlayer.tsx b/src/shared/components/players/JSMpegPlayer.tsx
index b880ccb..85a3635 100644
--- a/src/shared/components/players/JSMpegPlayer.tsx
+++ b/src/shared/components/players/JSMpegPlayer.tsx
@@ -1,74 +1,249 @@
// @ts-ignore we know this doesn't have types
import JSMpeg from "@cycjimmy/jsmpeg-player";
import { useViewportSize } from "@mantine/hooks";
-import { useEffect, useRef, useState } from "react";
+import { useEffect, useMemo, useRef, useState } from "react";
import { useTranslation } from "react-i18next";
+import { cn } from "../../utils/class.merge";
+import { PlayerStatsType } from "../../../types/live";
type JSMpegPlayerProps = {
- wsUrl: string;
- cameraHeight?: number,
- cameraWidth?: number,
+ url: string;
+ camera: string
+ className?: string;
+ width: number;
+ height: number;
+ containerRef: React.MutableRefObject;
+ playbackEnabled: boolean;
+ useWebGL: boolean;
+ setStats?: (stats: PlayerStatsType) => void;
+ onPlaying?: () => void;
};
const JSMpegPlayer = (
{
- wsUrl,
- cameraWidth = 1200,
- cameraHeight = 800,
+ url,
+ camera,
+ width,
+ height,
+ className,
+ containerRef,
+ playbackEnabled,
+ useWebGL = false,
+ setStats,
+ onPlaying,
}: JSMpegPlayerProps
) => {
- const { t } = useTranslation()
- const playerRef = useRef(null);
- const [playerInitialized, setPlayerInitialized] = useState(false)
+ const videoRef = useRef(null);
+ const canvasRef = useRef(null);
+ const internalContainerRef = useRef(null);
+ const onPlayingRef = useRef(onPlaying);
+ const [showCanvas, setShowCanvas] = useState(false);
+ const [hasData, setHasData] = useState(false);
+ const hasDataRef = useRef(hasData);
+ const [dimensionsReady, setDimensionsReady] = useState(false);
+ const bytesReceivedRef = useRef(0);
+ const lastTimestampRef = useRef(Date.now());
+ const statsIntervalRef = useRef(null);
- const { height: maxHeight, width: maxWidth } = useViewportSize()
+ const selectedContainerRef = useMemo(
+ () => (containerRef.current ? containerRef : internalContainerRef),
+ // we know that these deps are correct
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ [containerRef, containerRef.current, internalContainerRef],
+ );
+
+ const { height: containerHeight, width: containerWidth } = useViewportSize()
+
+ const stretch = true;
+ const aspectRatio = width / height;
+
+ const fitAspect = useMemo(
+ () => containerWidth / containerHeight,
+ [containerWidth, containerHeight],
+ );
+
+ const scaledHeight = useMemo(() => {
+ if (selectedContainerRef?.current && width && height) {
+ const scaledHeight =
+ aspectRatio < (fitAspect ?? 0)
+ ? Math.floor(
+ Math.min(
+ containerHeight,
+ selectedContainerRef.current?.clientHeight,
+ ),
+ )
+ : aspectRatio >= fitAspect
+ ? Math.floor(containerWidth / aspectRatio)
+ : Math.floor(containerWidth / aspectRatio) / 1.5;
+ const finalHeight = stretch
+ ? scaledHeight
+ : Math.min(scaledHeight, height);
+
+ if (finalHeight > 0) {
+ return finalHeight;
+ }
+ }
+ return undefined;
+ }, [
+ aspectRatio,
+ containerWidth,
+ containerHeight,
+ fitAspect,
+ height,
+ width,
+ stretch,
+ selectedContainerRef,
+ ]);
+
+ const scaledWidth = useMemo(() => {
+ if (aspectRatio && scaledHeight) {
+ return Math.ceil(scaledHeight * aspectRatio);
+ }
+ return undefined;
+ }, [scaledHeight, aspectRatio]);
useEffect(() => {
- const video = new JSMpeg.VideoElement(
- playerRef.current,
- wsUrl,
- {},
- { protocols: [], audio: false, videoBufferSize: 1024 * 1024 * 4 }
- );
+ if (scaledWidth && scaledHeight) {
+ setDimensionsReady(true);
+ }
+ }, [scaledWidth, scaledHeight]);
- const toggleFullscreen = () => {
- const canvas = video.els.canvas;
- if (!document.fullscreenElement && !(document as any).webkitFullscreenElement) { // Use bracket notation for webkit
- // Enter fullscreen
- if (canvas.requestFullscreen) {
- canvas.requestFullscreen();
- } else if ((canvas as any).webkitRequestFullScreen) { // Use bracket notation for webkit
- (canvas as any).webkitRequestFullScreen();
- } else if (canvas.mozRequestFullScreen) {
- canvas.mozRequestFullScreen();
+ useEffect(() => {
+ onPlayingRef.current = onPlaying;
+ }, [onPlaying]);
+
+ useEffect(() => {
+ if (!selectedContainerRef?.current || !url) {
+ return;
+ }
+
+ const videoWrapper = videoRef.current;
+ const canvas = canvasRef.current;
+ let videoElement: JSMpeg.VideoElement | null = null;
+
+ let frameCount = 0;
+
+ setHasData(false);
+
+ if (videoWrapper && playbackEnabled) {
+ // Delayed init to avoid issues with react strict mode
+ const initPlayer = setTimeout(() => {
+ videoElement = new JSMpeg.VideoElement(
+ videoWrapper,
+ url,
+ { canvas: canvas },
+ {
+ protocols: [],
+ audio: false,
+ disableGl: !useWebGL,
+ disableWebAssembly: !useWebGL,
+ videoBufferSize: 1024 * 1024 * 4,
+ onVideoDecode: () => {
+ if (!hasDataRef.current) {
+ setHasData(true);
+ onPlayingRef.current?.();
+ }
+ frameCount++;
+ },
+ },
+ );
+
+ // Set up WebSocket message handler
+ if (
+ videoElement.player &&
+ videoElement.player.source &&
+ videoElement.player.source.socket
+ ) {
+ const socket = videoElement.player.source.socket;
+ socket.addEventListener("message", (event: MessageEvent) => {
+ if (event.data instanceof ArrayBuffer) {
+ bytesReceivedRef.current += event.data.byteLength;
+ }
+ });
}
- } else {
- // Exit fullscreen
- if (document.exitFullscreen) {
- document.exitFullscreen();
- } else if ((document as any).webkitExitFullscreen) { // Use bracket notation for webkit
- (document as any).webkitExitFullscreen();
- } else if ((document as any).mozCancelFullScreen) {
- (document as any).mozCancelFullScreen();
+
+ // Update stats every second
+ statsIntervalRef.current = setInterval(() => {
+ const currentTimestamp = Date.now();
+ const timeDiff = (currentTimestamp - lastTimestampRef.current) / 1000; // in seconds
+ const bitrate = (bytesReceivedRef.current * 8) / timeDiff / 1000; // in kbps
+
+ setStats?.({
+ streamType: "jsmpeg",
+ bandwidth: Math.round(bitrate),
+ totalFrames: frameCount,
+ latency: undefined,
+ droppedFrames: undefined,
+ decodedFrames: undefined,
+ droppedFrameRate: undefined,
+ });
+
+ bytesReceivedRef.current = 0;
+ lastTimestampRef.current = currentTimestamp;
+ }, 1000);
+
+ return () => {
+ if (statsIntervalRef.current) {
+ clearInterval(statsIntervalRef.current);
+ frameCount = 0;
+ statsIntervalRef.current = null;
+ }
+ };
+ }, 0);
+
+ return () => {
+ clearTimeout(initPlayer);
+ if (statsIntervalRef.current) {
+ clearInterval(statsIntervalRef.current);
+ statsIntervalRef.current = null;
}
- }
- };
+ if (videoElement) {
+ try {
+ // this causes issues in react strict mode
+ // https://stackoverflow.com/questions/76822128/issue-with-cycjimmy-jsmpeg-player-in-react-18-cannot-read-properties-of-null-o
+ videoElement.destroy();
+ // eslint-disable-next-line no-empty
+ } catch (e) {}
+ }
+ };
+ }
+ // we know that these deps are correct
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [playbackEnabled, url]);
- video.els.canvas.addEventListener('dblclick', toggleFullscreen);
+ useEffect(() => {
+ setShowCanvas(hasData && dimensionsReady);
+ }, [hasData, dimensionsReady]);
- return () => {
- video.destroy();
- video.els.canvas.removeEventListener('dblclick', toggleFullscreen);
- };
- }, [wsUrl]);
+ useEffect(() => {
+ hasDataRef.current = hasData;
+ }, [hasData]);
return (
-
- )
-};
+
+ );
+}
export default JSMpegPlayer
\ No newline at end of file
diff --git a/src/shared/components/players/WebRTCPlayer.tsx b/src/shared/components/players/WebRTCPlayer.tsx
index 5f6f56f..aeefdaf 100644
--- a/src/shared/components/players/WebRTCPlayer.tsx
+++ b/src/shared/components/players/WebRTCPlayer.tsx
@@ -1,25 +1,46 @@
-import { useCallback, useEffect, useRef } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
+import { LivePlayerError, PlayerStatsType } from "../../../types/live";
type WebRtcPlayerProps = {
- className?: string;
camera: string;
+ wsURI: string;
+ className?: string;
playbackEnabled?: boolean;
- onPlaying?: () => void,
- wsUrl: string
+ audioEnabled?: boolean;
+ volume?: number;
+ microphoneEnabled?: boolean;
+ iOSCompatFullScreen?: boolean; // ios doesn't support fullscreen divs so we must support the video element
+ pip?: boolean;
+ getStats?: boolean;
+ setStats?: (stats: PlayerStatsType) => void;
+ onPlaying?: () => void;
+ onError?: (error: LivePlayerError) => void;
+
};
export default function WebRtcPlayer({
- className,
camera,
+ wsURI,
+ className,
playbackEnabled = true,
+ audioEnabled = false,
+ volume,
+ microphoneEnabled = false,
+ iOSCompatFullScreen = false,
+ pip = false,
+ getStats = false,
+ setStats,
onPlaying,
- wsUrl
+ onError,
}: WebRtcPlayerProps) {
// camera states
const pcRef = useRef();
const videoRef = useRef(null);
+ const [bufferTimeout, setBufferTimeout] = useState();
+ const videoLoadTimeoutRef = useRef();
+
const PeerConnection = useCallback(
async (media: string) => {
if (!videoRef.current) {
@@ -27,6 +48,7 @@ export default function WebRtcPlayer({
}
const pc = new RTCPeerConnection({
+ bundlePolicy: "max-bundle",
iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
});
@@ -59,7 +81,7 @@ export default function WebRtcPlayer({
.filter((kind) => media.indexOf(kind) >= 0)
.map(
(kind) =>
- pc.addTransceiver(kind, { direction: "recvonly" }).receiver.track
+ pc.addTransceiver(kind, { direction: "recvonly" }).receiver.track,
);
localTracks.push(...tracks);
}
@@ -67,12 +89,12 @@ export default function WebRtcPlayer({
videoRef.current.srcObject = new MediaStream(localTracks);
return pc;
},
- [videoRef]
+ [videoRef],
);
async function getMediaTracks(
media: string,
- constraints: MediaStreamConstraints
+ constraints: MediaStreamConstraints,
) {
try {
const stream =
@@ -86,12 +108,13 @@ export default function WebRtcPlayer({
}
const connect = useCallback(
- async (ws: WebSocket, aPc: Promise) => {
+ async (aPc: Promise) => {
if (!aPc) {
return;
}
pcRef.current = await aPc;
+ const ws = new WebSocket(wsURI);
ws.addEventListener("open", () => {
pcRef.current?.addEventListener("icecandidate", (ev) => {
@@ -127,7 +150,7 @@ export default function WebRtcPlayer({
}
});
},
- []
+ [wsURI],
);
useEffect(() => {
@@ -139,13 +162,10 @@ export default function WebRtcPlayer({
return;
}
- // const url = `$baseUrl{.replace(
- // /^http/,
- // "ws"
- // )}live/webrtc/api/ws?src=${camera}`;
- const ws = new WebSocket(wsUrl);
- const aPc = PeerConnection("video+audio");
- connect(ws, aPc);
+ const aPc = PeerConnection(
+ microphoneEnabled ? "video+audio+microphone" : "video+audio",
+ );
+ connect(aPc);
return () => {
if (pcRef.current) {
@@ -153,16 +173,174 @@ export default function WebRtcPlayer({
pcRef.current = undefined;
}
};
- }, [camera, connect, PeerConnection, pcRef, videoRef, playbackEnabled, wsUrl]);
+ }, [
+ camera,
+ wsURI,
+ connect,
+ PeerConnection,
+ pcRef,
+ videoRef,
+ playbackEnabled,
+ microphoneEnabled,
+ ]);
+
+ // ios compat
+
+ const [iOSCompatControls, setiOSCompatControls] = useState(false);
+
+ useEffect(() => {
+ if (!videoRef.current || !pip) {
+ return;
+ }
+
+ videoRef.current.requestPictureInPicture();
+ }, [pip, videoRef]);
+
+ // control volume
+
+ useEffect(() => {
+ if (!videoRef.current || volume == undefined) {
+ return;
+ }
+
+ videoRef.current.volume = volume;
+ }, [volume, videoRef]);
+
+ useEffect(() => {
+ videoLoadTimeoutRef.current = setTimeout(() => {
+ onError?.("stalled");
+ }, 5000);
+
+ return () => {
+ if (videoLoadTimeoutRef.current) {
+ clearTimeout(videoLoadTimeoutRef.current);
+ }
+ };
+ // we know that these deps are correct
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, []);
+
+ const handleLoadedData = () => {
+ if (videoLoadTimeoutRef.current) {
+ clearTimeout(videoLoadTimeoutRef.current);
+ }
+ onPlaying?.();
+ };
+
+ useEffect(() => {
+ if (!pcRef.current || !getStats) return;
+
+ let lastBytesReceived = 0;
+ let lastTimestamp = 0;
+
+ const interval = setInterval(async () => {
+ if (pcRef.current && videoRef.current && !videoRef.current.paused) {
+ const report = await pcRef.current.getStats();
+ let bytesReceived = 0;
+ let timestamp = 0;
+ let roundTripTime = 0;
+ let framesReceived = 0;
+ let framesDropped = 0;
+ let framesDecoded = 0;
+
+ report.forEach((stat) => {
+ if (stat.type === "inbound-rtp" && stat.kind === "video") {
+ bytesReceived = stat.bytesReceived;
+ timestamp = stat.timestamp;
+ framesReceived = stat.framesReceived;
+ framesDropped = stat.framesDropped;
+ framesDecoded = stat.framesDecoded;
+ }
+ if (stat.type === "candidate-pair" && stat.state === "succeeded") {
+ roundTripTime = stat.currentRoundTripTime;
+ }
+ });
+
+ const timeDiff = (timestamp - lastTimestamp) / 1000; // in seconds
+ const bitrate =
+ timeDiff > 0
+ ? (bytesReceived - lastBytesReceived) / timeDiff / 1000
+ : 0; // in kbps
+
+ setStats?.({
+ streamType: "WebRTC",
+ bandwidth: Math.round(bitrate),
+ latency: roundTripTime,
+ totalFrames: framesReceived,
+ droppedFrames: framesDropped,
+ decodedFrames: framesDecoded,
+ droppedFrameRate:
+ framesReceived > 0 ? (framesDropped / framesReceived) * 100 : 0,
+ });
+
+ lastBytesReceived = bytesReceived;
+ lastTimestamp = timestamp;
+ }
+ }, 1000);
+
+ return () => {
+ clearInterval(interval);
+ setStats?.({
+ streamType: "-",
+ bandwidth: 0,
+ latency: undefined,
+ totalFrames: 0,
+ droppedFrames: undefined,
+ decodedFrames: 0,
+ droppedFrameRate: 0,
+ });
+ };
+ // we need to listen on the value of the ref
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [pcRef, pcRef.current, getStats]);
return (