diff --git a/godot/scripts/avatar/gaze_controller.gd b/godot/scripts/avatar/gaze_controller.gd index 9d68bc3..294aca0 100644 --- a/godot/scripts/avatar/gaze_controller.gd +++ b/godot/scripts/avatar/gaze_controller.gd @@ -1,6 +1,7 @@ extends Node -## Dual-mode gaze system: Desktop Gaze (cursor) and Face-to-Face (webcam). -## Smoothly blends between targets based on conversation state. +## Dual-mode gaze: Desktop Gaze (cursor) and Face-to-Face (webcam). +## Uses VRM look animations to extract correct eye rotations, +## then blends between them based on 2D gaze direction. enum GazeMode { DESKTOP, @@ -11,23 +12,18 @@ const NodeUtilsScript = preload( "res://scripts/util/node_utils.gd" ) -const MODE_BLEND_SPEED: float = 2.0 -const GAZE_SMOOTHING: float = 10.0 +const GAZE_SMOOTHING: float = 8.0 const LOOK_AWAY_INTERVAL_MIN: float = 4.0 const LOOK_AWAY_INTERVAL_MAX: float = 8.0 const LOOK_AWAY_DURATION: float = 0.8 -const MAX_YAW: float = 0.4 -const MAX_PITCH: float = 0.3 var _skeleton: Skeleton3D -var _head_idx: int = -1 var _left_eye_idx: int = -1 var _right_eye_idx: int = -1 -var _head_rest: Quaternion var _left_eye_rest: Quaternion var _right_eye_rest: Quaternion +var _look_targets: Dictionary = {} var _current_mode: GazeMode = GazeMode.DESKTOP -var _mode_weight: float = 0.0 var _current_gaze: Vector2 = Vector2.ZERO var _target_gaze: Vector2 = Vector2.ZERO var _look_away_timer: float = 0.0 @@ -35,12 +31,11 @@ var _next_look_away: float = 5.0 var _is_looking_away: bool = false var _look_away_progress: float = 0.0 var _look_away_target: Vector2 = Vector2.ZERO -var _camera: Camera3D var _viewport: Viewport +var _has_look_anims: bool = false func setup(model: Node3D, camera: Camera3D) -> void: - _camera = camera _viewport = camera.get_viewport() _skeleton = NodeUtilsScript.find_child_of_type( model, "Skeleton3D" @@ -50,9 +45,6 @@ func setup(model: Node3D, camera: Camera3D) -> void: push_warning("GazeController: No Skeleton3D found") return - _head_idx = NodeUtilsScript.find_bone_case_insensitive( - _skeleton, "Head" - ) _left_eye_idx = NodeUtilsScript.find_bone_case_insensitive( _skeleton, "LeftEye" ) @@ -60,10 +52,6 @@ func setup(model: Node3D, camera: Camera3D) -> void: _skeleton, "RightEye" ) - if _head_idx != -1: - _head_rest = _skeleton.get_bone_rest( - _head_idx - ).basis.get_rotation_quaternion() if _left_eye_idx != -1: _left_eye_rest = _skeleton.get_bone_rest( _left_eye_idx @@ -73,6 +61,8 @@ func setup(model: Node3D, camera: Camera3D) -> void: _right_eye_idx ).basis.get_rotation_quaternion() + _extract_look_targets(model) + _next_look_away = randf_range( LOOK_AWAY_INTERVAL_MIN, LOOK_AWAY_INTERVAL_MAX ) @@ -80,28 +70,81 @@ func setup(model: Node3D, camera: Camera3D) -> void: EventBus.state_changed.connect(_on_state_changed) -func _process(delta: float) -> void: - if _skeleton == null or _head_idx == -1: +func _extract_look_targets(model: Node3D) -> void: + var anim_player := _find_animation_player(model) + if anim_player == null: + return + + var directions: Array[String] = [ + "lookLeft", "lookRight", "lookUp", "lookDown", + ] + + for dir_name: String in directions: + if not anim_player.has_animation(dir_name): + continue + + var anim := anim_player.get_animation(dir_name) + var rotations := _extract_eye_rotations(anim) + if not rotations.is_empty(): + _look_targets[dir_name] = rotations + + _has_look_anims = not _look_targets.is_empty() + + if not _has_look_anims: + push_warning("GazeController: No VRM look animations") + + +func _extract_eye_rotations( + anim: Animation, +) -> Dictionary: + var result: Dictionary = {} + + for i: int in range(anim.get_track_count()): + if anim.track_get_type(i) != Animation.TYPE_ROTATION_3D: + continue + + var path_str := str(anim.track_get_path(i)) + var bone_name := path_str.get_slice(":", 1) + + if bone_name != "LeftEye" and bone_name != "RightEye": + continue + + var key_count := anim.track_get_key_count(i) + if key_count == 0: + continue + + var rot: Quaternion = anim.track_get_key_value( + i, key_count - 1 + ) + result[bone_name] = rot + + return result + + +func _find_animation_player(node: Node) -> AnimationPlayer: + if node is AnimationPlayer: + return node as AnimationPlayer + for child: Node in node.get_children(): + var found := _find_animation_player(child) + if found != null: + return found + return null + + +func _process(delta: float) -> void: + if _skeleton == null: + return + if _left_eye_idx == -1 and _right_eye_idx == -1: return - _update_mode_weight(delta) _update_desktop_gaze() _update_look_away(delta) _smooth_gaze(delta) - _apply_gaze() - - -func _update_mode_weight(delta: float) -> void: - var target: float = ( - 0.0 if _current_mode == GazeMode.DESKTOP else 1.0 - ) - _mode_weight = lerpf( - _mode_weight, target, MODE_BLEND_SPEED * delta - ) + _apply_eye_rotation() func _update_desktop_gaze() -> void: - if _viewport == null: + if _viewport == null or _is_looking_away: return var mouse_pos := _viewport.get_mouse_position() @@ -110,15 +153,11 @@ func _update_desktop_gaze() -> void: if vp_size.x < 1.0 or vp_size.y < 1.0: return - var normalized := Vector2( + _target_gaze = Vector2( (mouse_pos.x / vp_size.x - 0.5) * 2.0, -(mouse_pos.y / vp_size.y - 0.5) * 2.0, ) - - _target_gaze = Vector2( - clampf(normalized.x * MAX_YAW, -MAX_YAW, MAX_YAW), - clampf(normalized.y * MAX_PITCH, -MAX_PITCH, MAX_PITCH), - ) + _target_gaze = _target_gaze.clampf(-1.0, 1.0) func _update_look_away(delta: float) -> void: @@ -143,8 +182,8 @@ func _update_look_away(delta: float) -> void: _is_looking_away = true _look_away_progress = 0.0 _look_away_target = Vector2( - randf_range(-MAX_YAW, MAX_YAW), - randf_range(-MAX_PITCH * 0.5, MAX_PITCH), + randf_range(-1.0, 1.0), + randf_range(-0.5, 0.5), ) @@ -154,24 +193,67 @@ func _smooth_gaze(delta: float) -> void: ) -func _apply_gaze() -> void: - var yaw := _current_gaze.x - var pitch := _current_gaze.y +func _apply_eye_rotation() -> void: + if _has_look_anims: + _apply_via_animations() + else: + _apply_via_fallback() - var head_yaw := yaw * 0.6 - var head_pitch := pitch * 0.6 - var eye_yaw := yaw * 0.4 - var eye_pitch := pitch * 0.4 - if _head_idx != -1: - var head_rot := _head_rest * Quaternion( - Vector3.UP, head_yaw - ) * Quaternion(Vector3.RIGHT, head_pitch) - _skeleton.set_bone_pose_rotation(_head_idx, head_rot) +func _apply_via_animations() -> void: + var gx := _current_gaze.x + var gy := _current_gaze.y + for bone_name: String in ["LeftEye", "RightEye"]: + var bone_idx := ( + _left_eye_idx if bone_name == "LeftEye" + else _right_eye_idx + ) + if bone_idx == -1: + continue + + var rest := ( + _left_eye_rest if bone_name == "LeftEye" + else _right_eye_rest + ) + + var result := rest + + if gx > 0.0 and _look_targets.has("lookRight"): + var t: Dictionary = _look_targets["lookRight"] + if t.has(bone_name): + result = rest.slerp(t[bone_name], gx) + elif gx < 0.0 and _look_targets.has("lookLeft"): + var t: Dictionary = _look_targets["lookLeft"] + if t.has(bone_name): + result = rest.slerp(t[bone_name], -gx) + + if gy > 0.0 and _look_targets.has("lookUp"): + var t: Dictionary = _look_targets["lookUp"] + if t.has(bone_name): + var up_rot: Quaternion = rest.slerp( + t[bone_name], gy + ) + var up_delta := rest.inverse() * up_rot + result = result * up_delta + elif gy < 0.0 and _look_targets.has("lookDown"): + var t: Dictionary = _look_targets["lookDown"] + if t.has(bone_name): + var down_rot: Quaternion = rest.slerp( + t[bone_name], -gy + ) + var down_delta := rest.inverse() * down_rot + result = result * down_delta + + _skeleton.set_bone_pose_rotation(bone_idx, result) + + +func _apply_via_fallback() -> void: var eye_rot := Quaternion( - Vector3.UP, eye_yaw - ) * Quaternion(Vector3.RIGHT, eye_pitch) + Vector3.UP, _current_gaze.x * 0.3 + ) * Quaternion( + Vector3.RIGHT, _current_gaze.y * 0.2 + ) if _left_eye_idx != -1: _skeleton.set_bone_pose_rotation( @@ -183,16 +265,9 @@ func _apply_gaze() -> void: ) -func set_face_target(_face_position: Vector2) -> void: +func set_face_target(face_position: Vector2) -> void: if _current_mode == GazeMode.FACE_TO_FACE: - _target_gaze = Vector2( - clampf( - _face_position.x, -MAX_YAW, MAX_YAW - ), - clampf( - _face_position.y, -MAX_PITCH, MAX_PITCH - ), - ) + _target_gaze = face_position.clampf(-1.0, 1.0) func _on_state_changed( diff --git a/godot/scripts/companion/companion.gd b/godot/scripts/companion/companion.gd index 09e6d3c..aeb02ed 100644 --- a/godot/scripts/companion/companion.gd +++ b/godot/scripts/companion/companion.gd @@ -34,6 +34,9 @@ const LLMClientScript = preload( const OrchestratorScript = preload( "res://scripts/companion/conversation_orchestrator.gd" ) +const WindowDragScript = preload( + "res://scripts/companion/window_drag.gd" +) const NodeUtilsScript = preload( "res://scripts/util/node_utils.gd" ) @@ -47,6 +50,8 @@ const SPEECH_SERVICE_URL: String = "http://localhost:8000" const LLM_URL: String = "http://localhost:11434" const LLM_MODEL: String = "llama3" +var _drag: Node + @onready var _avatar_root: Node3D = $AvatarRoot @onready var _camera: Camera3D = $Camera3D @onready var _audio_player: AudioStreamPlayer = ( @@ -56,10 +61,18 @@ const LLM_MODEL: String = "llama3" func _ready() -> void: _setup_window() + _setup_drag() _load_avatar() _setup_backend() +func _setup_drag() -> void: + _drag = WindowDragScript.new() + _drag.name = "WindowDrag" + add_child(_drag) + _drag.load_position() + + func _setup_window() -> void: var ds := DisplayServer ds.window_set_flag(ds.WINDOW_FLAG_TRANSPARENT, true) diff --git a/godot/scripts/companion/window_drag.gd b/godot/scripts/companion/window_drag.gd new file mode 100644 index 0000000..d5c9d97 --- /dev/null +++ b/godot/scripts/companion/window_drag.gd @@ -0,0 +1,55 @@ +extends Node +## Click and drag anywhere on the window to move it. +## Persists position to user://window_position.cfg. + +const SAVE_PATH: String = "user://window_position.cfg" + +var _dragging: bool = false +var _drag_start_window: Vector2i = Vector2i.ZERO +var _drag_start_mouse: Vector2i = Vector2i.ZERO + + +func _unhandled_input(event: InputEvent) -> void: + if event is InputEventMouseButton: + var mb: InputEventMouseButton = event as InputEventMouseButton + if mb.button_index == MOUSE_BUTTON_LEFT: + if mb.pressed: + _dragging = true + _drag_start_mouse = ( + DisplayServer.mouse_get_position() + ) + _drag_start_window = ( + DisplayServer.window_get_position() + ) + else: + _dragging = false + save_position() + + elif event is InputEventMouseMotion and _dragging: + var current_mouse := DisplayServer.mouse_get_position() + var delta := current_mouse - _drag_start_mouse + DisplayServer.window_set_position( + _drag_start_window + delta + ) + + +func load_position() -> bool: + var cfg := ConfigFile.new() + if cfg.load(SAVE_PATH) != OK: + return false + + var x: int = cfg.get_value("window", "x", -1) + var y: int = cfg.get_value("window", "y", -1) + if x < 0 or y < 0: + return false + + DisplayServer.window_set_position(Vector2i(x, y)) + return true + + +func save_position() -> void: + var pos := DisplayServer.window_get_position() + var cfg := ConfigFile.new() + cfg.set_value("window", "x", pos.x) + cfg.set_value("window", "y", pos.y) + cfg.save(SAVE_PATH) diff --git a/run b/run new file mode 100755 index 0000000..0949fe5 --- /dev/null +++ b/run @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# Launch the Chobit companion app +exec flatpak run --user org.godotengine.Godot --path "$(dirname "$0")/godot" "$@"