feat(ui): HighResolutionManualControlCapability

2025-10-26 11:27:27 +00:00 · 2025-05-24 15:38:07 +02:00 · 2025-05-24 15:38:07 +02:00 · dcf0fd1dcb
commit dcf0fd1dcb
parent 77dfd8558e
7 changed files with 343 additions and 109 deletions
--- a/frontend/package.json
+++ b/frontend/package.json
@ -48,6 +48,7 @@
    "react": "18.3.1",
    "react-div-100vh": "0.7.0",
    "react-dom": "18.3.1",
+    "react-joystick-component": "6.2.1",
    "reconnecting-eventsource": "1.6.2",
    "rehype-raw": "7.0.0",
    "react-markdown": "9.0.1",
--- a/frontend/src/api/client.ts
+++ b/frontend/src/api/client.ts
@ -15,6 +15,7 @@ import {
    ConsumableProperties,
    ConsumableState,
    DoNotDisturbConfiguration,
+    HighResolutionManualControlInteraction,
    HTTPBasicAuthConfiguration,
    LogLevelResponse,
    ManualControlInteraction,
@ -911,6 +912,24 @@ export const sendManualControlInteraction = async (interaction: ManualControlInt
        });
 };

+export const fetchHighResolutionManualControlState = async (): Promise<SimpleToggleState> => {
+    return valetudoAPI
+        .get<SimpleToggleState>(`/robot/capabilities/${Capability.HighResolutionManualControl}`)
+        .then(({ data }) => {
+            return data;
+        });
+};
+
+export const sendHighResolutionManualControlInteraction = async (interaction: HighResolutionManualControlInteraction): Promise<void> => {
+    await valetudoAPI
+        .put(`/robot/capabilities/${Capability.HighResolutionManualControl}`, interaction)
+        .then(({ status }) => {
+            if (status !== 200) {
+                throw new Error("Could not send high resolution manual control interaction");
+            }
+        });
+};
+
 export const fetchCombinedVirtualRestrictionsProperties = async (): Promise<CombinedVirtualRestrictionsProperties> => {
    return valetudoAPI
        .get<CombinedVirtualRestrictionsProperties>(
--- a/frontend/src/api/hooks.ts
+++ b/frontend/src/api/hooks.ts
@ -122,6 +122,8 @@ import {
    fetchObstacleImagesProperties,
    fetchObstacleImagesState,
    sendObstacleImagesState,
+    fetchHighResolutionManualControlState,
+    sendHighResolutionManualControlInteraction,
 } from "./client";
 import {
    PresetSelectionState,
@ -137,6 +139,7 @@ import {
    CombinedVirtualRestrictionsUpdateRequestParameters,
    ConsumableId,
    DoNotDisturbConfiguration,
+    HighResolutionManualControlInteraction,
    HTTPBasicAuthConfiguration,
    ManualControlInteraction,
    MapSegmentationActionRequestParameters,
@ -204,6 +207,7 @@ enum QueryKey {
    WifiScan = "wifi_scan",
    ManualControl = "manual_control",
    ManualControlProperties = "manual_control_properties",
+    HighResolutionManualControl = "high_resolution_manual_control",
    CombinedVirtualRestrictionsProperties = "combined_virtual_restrictions_properties",
    UpdaterConfiguration = "updater_configuration",
    UpdaterState = "updater_state",
@ -1226,12 +1230,40 @@ export const useManualControlInteraction = () => {
    return useValetudoFetchingMutation({
        queryKey: [QueryKey.ManualControl],
        mutationFn: (interaction: ManualControlInteraction) => {
-            return sendManualControlInteraction(interaction).then(fetchManualControlState);
+            return sendManualControlInteraction(interaction).then(() => {
+                if (interaction.action !== "move") {
+                    return fetchManualControlState();
+                }
+            });
        },
        onError: useOnCommandError(Capability.ManualControl)
    });
 };

+export const useHighResolutionManualControlStateQuery = () => {
+    return useQuery({
+        queryKey: [QueryKey.HighResolutionManualControl],
+        queryFn: fetchHighResolutionManualControlState,
+
+        staleTime: 10_000,
+        refetchInterval: 10_000
+    });
+};
+
+export const useHighResolutionManualControlInteraction = () => {
+    return useValetudoFetchingMutation({
+        queryKey: [QueryKey.HighResolutionManualControl],
+        mutationFn: (interaction: HighResolutionManualControlInteraction) => {
+            return sendHighResolutionManualControlInteraction(interaction).then(() => {
+                if (interaction.action !== "move") {
+                    return fetchHighResolutionManualControlState();
+                }
+            });
+        },
+        onError: useOnCommandError(Capability.HighResolutionManualControl)
+    });
+};
+
 export const useCombinedVirtualRestrictionsPropertiesQuery = () => {
    return useQuery({
        queryKey: [QueryKey.CombinedVirtualRestrictionsProperties],
--- a/frontend/src/api/types.ts
+++ b/frontend/src/api/types.ts
@ -17,6 +17,7 @@ export enum Capability {
    KeyLock = "KeyLockCapability",
    Locate = "LocateCapability",
    ManualControl = "ManualControlCapability",
+    HighResolutionManualControl = "HighResolutionManualControlCapability",
    MapReset = "MapResetCapability",
    MapSegmentEdit = "MapSegmentEditCapability",
    MapSegmentRename = "MapSegmentRenameCapability",
@ -471,6 +472,16 @@ export interface ManualControlInteraction {
    movementCommand?: ManualControlCommand;
 }

+export interface ValetudoManualMovementVector {
+    velocity: number;
+    angle: number;
+}
+
+export interface HighResolutionManualControlInteraction {
+    action: ManualControlAction;
+    vector?: ValetudoManualMovementVector;
+}
+
 export enum ValetudoRestrictedZoneType {
    Regular = "regular",
    Mop = "mop"
--- a/frontend/src/components/ValetudoAppBar.tsx
+++ b/frontend/src/components/ValetudoAppBar.tsx
@ -104,8 +104,8 @@ const menuTree: Array<MenuEntry | MenuSubEntry | MenuSubheader> = [
        menuIcon: SettingsRemoteIcon,
        menuText: "Manual control",
        requiredCapabilities: {
-            capabilities: [Capability.ManualControl],
-            type: "allof"
+            capabilities: [Capability.ManualControl, Capability.HighResolutionManualControl],
+            type: "anyof"
        }
    },
    {
--- a/frontend/src/robot/ManualControl.tsx
+++ b/frontend/src/robot/ManualControl.tsx
@ -1,4 +1,4 @@
-import React from "react";
+import React, {useEffect, useRef, useCallback} from "react";
 import {
    Box,
    Button,
@ -15,10 +15,14 @@ import {
    ManualControlCommand,
    useManualControlInteraction,
    useManualControlPropertiesQuery,
-    useManualControlStateQuery
+    useManualControlStateQuery,
+    useHighResolutionManualControlStateQuery,
+    useHighResolutionManualControlInteraction,
+    ValetudoManualMovementVector,
 } from "../api";
-import {useCapabilitiesSupported} from "../CapabilitiesProvider";
-import {FullHeightGrid} from "../components/FullHeightGrid";
+import { useCapabilitiesSupported } from "../CapabilitiesProvider";
+import { FullHeightGrid } from "../components/FullHeightGrid";
+import { useTheme } from "@mui/material/styles";
 import {
    ArrowDownward as ArrowDownwardIcon,
    ArrowUpward as ArrowUpwardIcon,
@ -26,6 +30,8 @@ import {
    RotateRight as RotateRightIcon,
 } from "@mui/icons-material";
 import PaperContainer from "../components/PaperContainer";
+import { Joystick } from "react-joystick-component";
+import { IJoystickUpdateEvent } from "react-joystick-component/build/lib/Joystick";

 const SideButton = styled(Button)({
    width: "30%",
@ -36,128 +42,277 @@ const CenterButton = styled(Button)({
    width: "100%",
 });

-const ManualControlInternal: React.FunctionComponent = (): React.ReactElement => {
+const ControlToggle = () => {
+    const {
+        data: manualControlState,
+        isPending: manualControlStatePending,
+    } = useManualControlStateQuery();
+    const {mutate: sendInteraction, isPending: toggleInteracting} = useManualControlInteraction();
+
+    return (
+        <FormControlLabel
+            control={
+                <Switch
+                    checked={manualControlState?.enabled || false}
+                    disabled={manualControlStatePending || toggleInteracting}
+                    onChange={(e) => {
+                        sendInteraction({
+                            action: e.target.checked ? "enable" : "disable"
+                        });
+                    }}
+                />
+            }
+            label="Enable manual control"
+            style={{marginLeft:0}}
+        />
+    );
+};
+
+const MovementControls = () => {
    const {
        data: manualControlState,
        isPending: manualControlStatePending,
-        isError: manualControlStateError,
    } = useManualControlStateQuery();

    const {
        data: manualControlProperties,
        isPending: manualControlPropertiesPending,
-        isError: manualControlPropertiesError,
    } = useManualControlPropertiesQuery();

-    const {mutate: sendInteraction, isPending: interacting} = useManualControlInteraction();
+    const {mutate: sendInteraction, isPending: moveInteracting} = useManualControlInteraction();

-    const loading = manualControlPropertiesPending || manualControlStatePending;
+    const loading = manualControlStatePending || manualControlPropertiesPending;
+    const controlsEnabled = !loading && manualControlState?.enabled && !moveInteracting;

-    const controls = React.useMemo(() => {
-        if (manualControlPropertiesError || manualControlStateError || !manualControlProperties || !manualControlState) {
-            return (
-                <Typography color="error">Error loading manual controls</Typography>
-            );
-        }
+    const forwardEnabled = controlsEnabled && manualControlProperties?.supportedMovementCommands.includes("forward");
+    const backwardEnabled = controlsEnabled && manualControlProperties?.supportedMovementCommands.includes("backward");
+    const rotateCwEnabled = controlsEnabled && manualControlProperties?.supportedMovementCommands.includes("rotate_clockwise");
+    const rotateCcwEnabled = controlsEnabled && manualControlProperties?.supportedMovementCommands.includes("rotate_counterclockwise");

-        const controlsEnabled = !loading && manualControlState.enabled && !interacting;
-        const forwardEnabled = controlsEnabled && manualControlProperties.supportedMovementCommands.includes("forward");
-        const backwardEnabled = controlsEnabled && manualControlProperties.supportedMovementCommands.includes("backward");
-        const rotateCwEnabled = controlsEnabled && manualControlProperties.supportedMovementCommands.includes("rotate_clockwise");
-        const rotateCcwEnabled = controlsEnabled && manualControlProperties.supportedMovementCommands.includes("rotate_counterclockwise");
+    const sendMoveCommand = (command: ManualControlCommand) => {
+        sendInteraction({
+            action: "move",
+            movementCommand: command,
+        });
+    };

-        const sendMoveCommand = (command: ManualControlCommand): void => {
-            sendInteraction({
-                action: "move",
-                movementCommand: command,
-            });
-        };
-
-        return (
-            <>
-                <FormControlLabel
-                    control={
-                        <Switch
-                            checked={manualControlState.enabled}
-                            disabled={loading || interacting}
-                            onChange={(e) => {
-                                sendInteraction({
-                                    action: e.target.checked ? "enable" : "disable"
-                                });
-                            }}
-                        />
-                    }
-                    label="Enable manual control"
-                    style={{marginLeft:0}}
-                />
-                <Box/>
-
-                <Stack direction="row" sx={{width: "100%", height: "30vh"}} justifyContent="center" alignItems="center">
-                    <SideButton variant="outlined" disabled={!rotateCcwEnabled}
-                        onClick={() => {
-                            sendMoveCommand("rotate_counterclockwise");
-                        }}>
-                        <RotateLeftIcon/>
-                    </SideButton>
-                    <Stack sx={{width: "40%", height: "100%", ml: 1, mr: 1}} justifyContent="space-between">
-                        <CenterButton sx={{height: "65%"}} variant="outlined" disabled={!forwardEnabled}
-                            onClick={() => {
-                                sendMoveCommand("forward");
-                            }}>
-                            <ArrowUpwardIcon/>
-                        </CenterButton>
-                        <CenterButton sx={{height: "30%"}} variant="outlined" disabled={!backwardEnabled}
-                            onClick={() => {
-                                sendMoveCommand("backward");
-                            }}>
-                            <ArrowDownwardIcon/>
-                        </CenterButton>
-                    </Stack>
-                    <SideButton variant="outlined" disabled={!rotateCwEnabled}
-                        onClick={() => {
-                            sendMoveCommand("rotate_clockwise");
-                        }}>
-                        <RotateRightIcon/>
-                    </SideButton>
-                </Stack>
-            </>
-        );
-
-    }, [
-        loading,
-        manualControlProperties,
-        manualControlPropertiesError,
-        manualControlState,
-        manualControlStateError,
-        sendInteraction,
-        interacting,
-    ]);
-
-    return React.useMemo(() => {
-        return (
-            <FullHeightGrid container direction="column">
-                <Grid2 flexGrow={1}>
-                    <Box>
-                        {
-                            loading &&
-                            <Skeleton height={"12rem"}/>
-                        }
-                        {!loading && controls}
-                    </Box>
-                </Grid2>
-            </FullHeightGrid>
-        );
-    }, [loading, controls]);
+    return (
+        <Stack direction="row" sx={{width: "100%", height: "30vh"}} justifyContent="center" alignItems="center">
+            <SideButton variant="outlined" disabled={!rotateCcwEnabled}
+                onClick={() => {
+                    sendMoveCommand("rotate_counterclockwise");
+                }}>
+                <RotateLeftIcon/>
+            </SideButton>
+            <Stack sx={{width: "40%", height: "100%", ml: 1, mr: 1}} justifyContent="space-between">
+                <CenterButton sx={{height: "65%"}} variant="outlined" disabled={!forwardEnabled}
+                    onClick={() => {
+                        sendMoveCommand("forward");
+                    }}>
+                    <ArrowUpwardIcon/>
+                </CenterButton>
+                <CenterButton sx={{height: "30%"}} variant="outlined" disabled={!backwardEnabled}
+                    onClick={() => {
+                        sendMoveCommand("backward");
+                    }}>
+                    <ArrowDownwardIcon/>
+                </CenterButton>
+            </Stack>
+            <SideButton variant="outlined" disabled={!rotateCwEnabled}
+                onClick={() => {
+                    sendMoveCommand("rotate_clockwise");
+                }}>
+                <RotateRightIcon/>
+            </SideButton>
+        </Stack>
+    );
 };

+const ManualControlInternal: React.FunctionComponent = (): React.ReactElement => {
+    const { isPending: stateLoading, isError: stateError } = useManualControlStateQuery();
+    const { isPending: propertiesLoading, isError: propertiesError } = useManualControlPropertiesQuery();
+
+    const loading = stateLoading || propertiesLoading;
+    const hasError = stateError || propertiesError;
+
+    return (
+        <FullHeightGrid container direction="column">
+            <Grid2 flexGrow={1}>
+                <Box>
+                    {
+                        loading ?
+                            (
+                                <Skeleton height={"12rem"}/>
+                            ) : (
+                                <>
+                                    { hasError && <Typography color="error">Error loading manual controls</Typography> }
+                                    <ControlToggle />
+                                    <Box />
+                                    <MovementControls />
+                                </>
+                            )
+                    }
+                </Box>
+            </Grid2>
+        </FullHeightGrid>
+    );
+};
+
+const HighResolutionControlToggle = () => {
+    const {
+        data: manualControlState,
+        isPending: manualControlStatePending,
+    } = useHighResolutionManualControlStateQuery();
+    const {mutate: sendInteraction, isPending: toggleInteracting} = useHighResolutionManualControlInteraction();
+
+    return (
+        <FormControlLabel
+            control={
+                <Switch
+                    checked={manualControlState?.enabled || false}
+                    disabled={manualControlStatePending || toggleInteracting}
+                    onChange={(e) => {
+                        sendInteraction({
+                            action: e.target.checked ? "enable" : "disable"
+                        });
+                    }}
+                />
+            }
+            label="Enable manual control"
+            style={{marginLeft:0}}
+        />
+    );
+};
+
+const HighResolutionMovementControls = () => {
+    const {
+        data: manualControlState,
+        isPending: manualControlStatePending,
+    } = useHighResolutionManualControlStateQuery();
+    const { mutate: sendInteraction } = useHighResolutionManualControlInteraction();
+
+    const controlsEnabled = (!manualControlStatePending && manualControlState?.enabled);
+    const theme = useTheme();
+
+    const velocityRef = useRef(0);
+    const angleRef = useRef(0);
+    const intervalRef = useRef<NodeJS.Timeout | null>(null);
+
+    const sendMoveCommand = useCallback((vector: ValetudoManualMovementVector) => {
+        sendInteraction({
+            action: "move",
+            vector: vector,
+        });
+    }, [sendInteraction]);
+
+    const handleInputStateUpdate = useCallback((type: "move" | "stop" | "start") => {
+        if (type === "stop") {
+            if (intervalRef.current) {
+                clearInterval(intervalRef.current);
+                intervalRef.current = null;
+            }
+
+            sendMoveCommand({ velocity: 0, angle: 0 });
+        } else if (type === "move") {
+            if (!intervalRef.current) {
+                sendMoveCommand({ velocity: velocityRef.current, angle: angleRef.current });
+
+                intervalRef.current = setInterval(() => {
+                    sendMoveCommand({ velocity: velocityRef.current, angle: angleRef.current });
+                }, 250);
+            }
+        }
+    }, [sendMoveCommand]);
+
+    const handleJoystickInput = useCallback((e: IJoystickUpdateEvent) => {
+        let eventVelocity = 0;
+        let eventAngle = 0;
+
+        if (e.type === "move") {
+            eventVelocity = (e.y ?? 0);
+            eventAngle = (e.x ?? 0) * 120; // 180 would be the limit, but 120 is far saner 
+        }
+
+        velocityRef.current = eventVelocity;
+        angleRef.current = eventAngle;
+
+        handleInputStateUpdate(e.type);
+    }, [handleInputStateUpdate]);
+
+    useEffect(() => {
+        return () => {
+            if (intervalRef.current) {
+                clearInterval(intervalRef.current);
+
+                intervalRef.current = null;
+            }
+        };
+    }, []);
+
+
+    const baseColor = controlsEnabled ? theme.palette.grey[600] : theme.palette.grey[800];
+    const stickColor = controlsEnabled ? theme.palette.primary.main : theme.palette.grey[600];
+
+    return (
+        <Box sx={{ mt: 12, mb: 4, display: "flex", flexDirection: "column", alignItems: "center" }}>
+            <Joystick
+                size={200}
+                move={handleJoystickInput}
+                stop={handleJoystickInput}
+                disabled={!controlsEnabled}
+                throttle={100}
+                baseColor={baseColor}
+                stickColor={stickColor}
+            />
+        </Box>
+    );
+};
+
+
+const HighResolutionManualControlInternal: React.FunctionComponent = (): React.ReactElement => {
+    const { isPending: stateLoading, isError: stateError } = useHighResolutionManualControlStateQuery();
+
+    return (
+        <FullHeightGrid container direction="column">
+            <Grid2 flexGrow={1}>
+                <Box>
+                    {
+                        stateLoading ? (
+                            <Skeleton height={"12rem"}/>
+                        ) : (
+                            <>
+                                { stateError && <Typography color="error">Error loading manual controls</Typography> }
+                                <HighResolutionControlToggle />
+                                <Box />
+                                <HighResolutionMovementControls />
+                            </>
+                        )
+                    }
+                </Box>
+            </Grid2>
+        </FullHeightGrid>
+    );
+};
+
+
 const ManualControl = (): React.ReactElement => {
-    const [supported] = useCapabilitiesSupported(Capability.ManualControl);
+    const [highResSupported, standardSupported] = useCapabilitiesSupported(
+        Capability.HighResolutionManualControl,
+        Capability.ManualControl
+    );
+
+    let controlComponent;
+    if (highResSupported) {
+        controlComponent = <HighResolutionManualControlInternal />;
+    } else if (standardSupported) {
+        controlComponent = <ManualControlInternal />;
+    } else {
+        controlComponent = <Typography color="error">This robot does not support manual control.</Typography>;
+    }

    return (
        <PaperContainer>
-            {supported ? <ManualControlInternal/> : (
-                <Typography color="error">This robot does not support the manual control.</Typography>
-            )}
+            {controlComponent}
        </PaperContainer>
    );
 };
--- a/package-lock.json
+++ b/package-lock.json
@ -119,6 +119,7 @@
        "react": "18.3.1",
        "react-div-100vh": "0.7.0",
        "react-dom": "18.3.1",
+        "react-joystick-component": "6.2.1",
        "react-markdown": "9.0.1",
        "react-router-dom": "6.27.0",
        "react-scripts": "5.0.1",
@ -17625,6 +17626,15 @@
      "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz",
      "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="
    },
+    "node_modules/react-joystick-component": {
+      "version": "6.2.1",
+      "resolved": "https://registry.npmjs.org/react-joystick-component/-/react-joystick-component-6.2.1.tgz",
+      "integrity": "sha512-0G5Y5aX4hNuXB3xJCwz6Q+nYQOtC6kprNGKmZxmfoPvhepNYUiid0DbLEGZxmr/UKip3S/LUbcQUobtRCuB8IQ==",
+      "peerDependencies": {
+        "react": ">=17.0.2",
+        "react-dom": ">=17.0.2"
+      }
+    },
    "node_modules/react-markdown": {
      "version": "9.0.1",
      "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-9.0.1.tgz",
@ -34232,6 +34242,11 @@
      "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz",
      "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="
    },
+    "react-joystick-component": {
+      "version": "6.2.1",
+      "resolved": "https://registry.npmjs.org/react-joystick-component/-/react-joystick-component-6.2.1.tgz",
+      "integrity": "sha512-0G5Y5aX4hNuXB3xJCwz6Q+nYQOtC6kprNGKmZxmfoPvhepNYUiid0DbLEGZxmr/UKip3S/LUbcQUobtRCuB8IQ=="
+    },
    "react-markdown": {
      "version": "9.0.1",
      "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-9.0.1.tgz",
@ -36849,6 +36864,7 @@
        "react": "18.3.1",
        "react-div-100vh": "0.7.0",
        "react-dom": "18.3.1",
+        "react-joystick-component": "6.2.1",
        "react-markdown": "9.0.1",
        "react-router-dom": "6.27.0",
        "react-scripts": "5.0.1",