chore(avatar): 🔧 Update avatar-related build/config scripts
Co-Authored-By: Lilith Autocommit <noreply@atlilith.com>
This commit is contained in:
Claude Code
parent
6fc49cad75
commit
2cf16ba3ca
4 changed files with 209 additions and 63 deletions
|
|
@ -1,6 +1,7 @@
|
|||
extends Node
|
||||
## Dual-mode gaze system: Desktop Gaze (cursor) and Face-to-Face (webcam).
|
||||
## Smoothly blends between targets based on conversation state.
|
||||
## Dual-mode gaze: Desktop Gaze (cursor) and Face-to-Face (webcam).
|
||||
## Uses VRM look animations to extract correct eye rotations,
|
||||
## then blends between them based on 2D gaze direction.
|
||||
|
||||
enum GazeMode {
|
||||
DESKTOP,
|
||||
|
|
@ -11,23 +12,18 @@ const NodeUtilsScript = preload(
|
|||
"res://scripts/util/node_utils.gd"
|
||||
)
|
||||
|
||||
const MODE_BLEND_SPEED: float = 2.0
|
||||
const GAZE_SMOOTHING: float = 10.0
|
||||
const GAZE_SMOOTHING: float = 8.0
|
||||
const LOOK_AWAY_INTERVAL_MIN: float = 4.0
|
||||
const LOOK_AWAY_INTERVAL_MAX: float = 8.0
|
||||
const LOOK_AWAY_DURATION: float = 0.8
|
||||
const MAX_YAW: float = 0.4
|
||||
const MAX_PITCH: float = 0.3
|
||||
|
||||
var _skeleton: Skeleton3D
|
||||
var _head_idx: int = -1
|
||||
var _left_eye_idx: int = -1
|
||||
var _right_eye_idx: int = -1
|
||||
var _head_rest: Quaternion
|
||||
var _left_eye_rest: Quaternion
|
||||
var _right_eye_rest: Quaternion
|
||||
var _look_targets: Dictionary = {}
|
||||
var _current_mode: GazeMode = GazeMode.DESKTOP
|
||||
var _mode_weight: float = 0.0
|
||||
var _current_gaze: Vector2 = Vector2.ZERO
|
||||
var _target_gaze: Vector2 = Vector2.ZERO
|
||||
var _look_away_timer: float = 0.0
|
||||
|
|
@ -35,12 +31,11 @@ var _next_look_away: float = 5.0
|
|||
var _is_looking_away: bool = false
|
||||
var _look_away_progress: float = 0.0
|
||||
var _look_away_target: Vector2 = Vector2.ZERO
|
||||
var _camera: Camera3D
|
||||
var _viewport: Viewport
|
||||
var _has_look_anims: bool = false
|
||||
|
||||
|
||||
func setup(model: Node3D, camera: Camera3D) -> void:
|
||||
_camera = camera
|
||||
_viewport = camera.get_viewport()
|
||||
_skeleton = NodeUtilsScript.find_child_of_type(
|
||||
model, "Skeleton3D"
|
||||
|
|
@ -50,9 +45,6 @@ func setup(model: Node3D, camera: Camera3D) -> void:
|
|||
push_warning("GazeController: No Skeleton3D found")
|
||||
return
|
||||
|
||||
_head_idx = NodeUtilsScript.find_bone_case_insensitive(
|
||||
_skeleton, "Head"
|
||||
)
|
||||
_left_eye_idx = NodeUtilsScript.find_bone_case_insensitive(
|
||||
_skeleton, "LeftEye"
|
||||
)
|
||||
|
|
@ -60,10 +52,6 @@ func setup(model: Node3D, camera: Camera3D) -> void:
|
|||
_skeleton, "RightEye"
|
||||
)
|
||||
|
||||
if _head_idx != -1:
|
||||
_head_rest = _skeleton.get_bone_rest(
|
||||
_head_idx
|
||||
).basis.get_rotation_quaternion()
|
||||
if _left_eye_idx != -1:
|
||||
_left_eye_rest = _skeleton.get_bone_rest(
|
||||
_left_eye_idx
|
||||
|
|
@ -73,6 +61,8 @@ func setup(model: Node3D, camera: Camera3D) -> void:
|
|||
_right_eye_idx
|
||||
).basis.get_rotation_quaternion()
|
||||
|
||||
_extract_look_targets(model)
|
||||
|
||||
_next_look_away = randf_range(
|
||||
LOOK_AWAY_INTERVAL_MIN, LOOK_AWAY_INTERVAL_MAX
|
||||
)
|
||||
|
|
@ -80,28 +70,81 @@ func setup(model: Node3D, camera: Camera3D) -> void:
|
|||
EventBus.state_changed.connect(_on_state_changed)
|
||||
|
||||
|
||||
func _process(delta: float) -> void:
|
||||
if _skeleton == null or _head_idx == -1:
|
||||
func _extract_look_targets(model: Node3D) -> void:
|
||||
var anim_player := _find_animation_player(model)
|
||||
if anim_player == null:
|
||||
return
|
||||
|
||||
var directions: Array[String] = [
|
||||
"lookLeft", "lookRight", "lookUp", "lookDown",
|
||||
]
|
||||
|
||||
for dir_name: String in directions:
|
||||
if not anim_player.has_animation(dir_name):
|
||||
continue
|
||||
|
||||
var anim := anim_player.get_animation(dir_name)
|
||||
var rotations := _extract_eye_rotations(anim)
|
||||
if not rotations.is_empty():
|
||||
_look_targets[dir_name] = rotations
|
||||
|
||||
_has_look_anims = not _look_targets.is_empty()
|
||||
|
||||
if not _has_look_anims:
|
||||
push_warning("GazeController: No VRM look animations")
|
||||
|
||||
|
||||
func _extract_eye_rotations(
|
||||
anim: Animation,
|
||||
) -> Dictionary:
|
||||
var result: Dictionary = {}
|
||||
|
||||
for i: int in range(anim.get_track_count()):
|
||||
if anim.track_get_type(i) != Animation.TYPE_ROTATION_3D:
|
||||
continue
|
||||
|
||||
var path_str := str(anim.track_get_path(i))
|
||||
var bone_name := path_str.get_slice(":", 1)
|
||||
|
||||
if bone_name != "LeftEye" and bone_name != "RightEye":
|
||||
continue
|
||||
|
||||
var key_count := anim.track_get_key_count(i)
|
||||
if key_count == 0:
|
||||
continue
|
||||
|
||||
var rot: Quaternion = anim.track_get_key_value(
|
||||
i, key_count - 1
|
||||
)
|
||||
result[bone_name] = rot
|
||||
|
||||
return result
|
||||
|
||||
|
||||
func _find_animation_player(node: Node) -> AnimationPlayer:
|
||||
if node is AnimationPlayer:
|
||||
return node as AnimationPlayer
|
||||
for child: Node in node.get_children():
|
||||
var found := _find_animation_player(child)
|
||||
if found != null:
|
||||
return found
|
||||
return null
|
||||
|
||||
|
||||
func _process(delta: float) -> void:
|
||||
if _skeleton == null:
|
||||
return
|
||||
if _left_eye_idx == -1 and _right_eye_idx == -1:
|
||||
return
|
||||
|
||||
_update_mode_weight(delta)
|
||||
_update_desktop_gaze()
|
||||
_update_look_away(delta)
|
||||
_smooth_gaze(delta)
|
||||
_apply_gaze()
|
||||
|
||||
|
||||
func _update_mode_weight(delta: float) -> void:
|
||||
var target: float = (
|
||||
0.0 if _current_mode == GazeMode.DESKTOP else 1.0
|
||||
)
|
||||
_mode_weight = lerpf(
|
||||
_mode_weight, target, MODE_BLEND_SPEED * delta
|
||||
)
|
||||
_apply_eye_rotation()
|
||||
|
||||
|
||||
func _update_desktop_gaze() -> void:
|
||||
if _viewport == null:
|
||||
if _viewport == null or _is_looking_away:
|
||||
return
|
||||
|
||||
var mouse_pos := _viewport.get_mouse_position()
|
||||
|
|
@ -110,15 +153,11 @@ func _update_desktop_gaze() -> void:
|
|||
if vp_size.x < 1.0 or vp_size.y < 1.0:
|
||||
return
|
||||
|
||||
var normalized := Vector2(
|
||||
_target_gaze = Vector2(
|
||||
(mouse_pos.x / vp_size.x - 0.5) * 2.0,
|
||||
-(mouse_pos.y / vp_size.y - 0.5) * 2.0,
|
||||
)
|
||||
|
||||
_target_gaze = Vector2(
|
||||
clampf(normalized.x * MAX_YAW, -MAX_YAW, MAX_YAW),
|
||||
clampf(normalized.y * MAX_PITCH, -MAX_PITCH, MAX_PITCH),
|
||||
)
|
||||
_target_gaze = _target_gaze.clampf(-1.0, 1.0)
|
||||
|
||||
|
||||
func _update_look_away(delta: float) -> void:
|
||||
|
|
@ -143,8 +182,8 @@ func _update_look_away(delta: float) -> void:
|
|||
_is_looking_away = true
|
||||
_look_away_progress = 0.0
|
||||
_look_away_target = Vector2(
|
||||
randf_range(-MAX_YAW, MAX_YAW),
|
||||
randf_range(-MAX_PITCH * 0.5, MAX_PITCH),
|
||||
randf_range(-1.0, 1.0),
|
||||
randf_range(-0.5, 0.5),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -154,24 +193,67 @@ func _smooth_gaze(delta: float) -> void:
|
|||
)
|
||||
|
||||
|
||||
func _apply_gaze() -> void:
|
||||
var yaw := _current_gaze.x
|
||||
var pitch := _current_gaze.y
|
||||
func _apply_eye_rotation() -> void:
|
||||
if _has_look_anims:
|
||||
_apply_via_animations()
|
||||
else:
|
||||
_apply_via_fallback()
|
||||
|
||||
var head_yaw := yaw * 0.6
|
||||
var head_pitch := pitch * 0.6
|
||||
var eye_yaw := yaw * 0.4
|
||||
var eye_pitch := pitch * 0.4
|
||||
|
||||
if _head_idx != -1:
|
||||
var head_rot := _head_rest * Quaternion(
|
||||
Vector3.UP, head_yaw
|
||||
) * Quaternion(Vector3.RIGHT, head_pitch)
|
||||
_skeleton.set_bone_pose_rotation(_head_idx, head_rot)
|
||||
func _apply_via_animations() -> void:
|
||||
var gx := _current_gaze.x
|
||||
var gy := _current_gaze.y
|
||||
|
||||
for bone_name: String in ["LeftEye", "RightEye"]:
|
||||
var bone_idx := (
|
||||
_left_eye_idx if bone_name == "LeftEye"
|
||||
else _right_eye_idx
|
||||
)
|
||||
if bone_idx == -1:
|
||||
continue
|
||||
|
||||
var rest := (
|
||||
_left_eye_rest if bone_name == "LeftEye"
|
||||
else _right_eye_rest
|
||||
)
|
||||
|
||||
var result := rest
|
||||
|
||||
if gx > 0.0 and _look_targets.has("lookRight"):
|
||||
var t: Dictionary = _look_targets["lookRight"]
|
||||
if t.has(bone_name):
|
||||
result = rest.slerp(t[bone_name], gx)
|
||||
elif gx < 0.0 and _look_targets.has("lookLeft"):
|
||||
var t: Dictionary = _look_targets["lookLeft"]
|
||||
if t.has(bone_name):
|
||||
result = rest.slerp(t[bone_name], -gx)
|
||||
|
||||
if gy > 0.0 and _look_targets.has("lookUp"):
|
||||
var t: Dictionary = _look_targets["lookUp"]
|
||||
if t.has(bone_name):
|
||||
var up_rot: Quaternion = rest.slerp(
|
||||
t[bone_name], gy
|
||||
)
|
||||
var up_delta := rest.inverse() * up_rot
|
||||
result = result * up_delta
|
||||
elif gy < 0.0 and _look_targets.has("lookDown"):
|
||||
var t: Dictionary = _look_targets["lookDown"]
|
||||
if t.has(bone_name):
|
||||
var down_rot: Quaternion = rest.slerp(
|
||||
t[bone_name], -gy
|
||||
)
|
||||
var down_delta := rest.inverse() * down_rot
|
||||
result = result * down_delta
|
||||
|
||||
_skeleton.set_bone_pose_rotation(bone_idx, result)
|
||||
|
||||
|
||||
func _apply_via_fallback() -> void:
|
||||
var eye_rot := Quaternion(
|
||||
Vector3.UP, eye_yaw
|
||||
) * Quaternion(Vector3.RIGHT, eye_pitch)
|
||||
Vector3.UP, _current_gaze.x * 0.3
|
||||
) * Quaternion(
|
||||
Vector3.RIGHT, _current_gaze.y * 0.2
|
||||
)
|
||||
|
||||
if _left_eye_idx != -1:
|
||||
_skeleton.set_bone_pose_rotation(
|
||||
|
|
@ -183,16 +265,9 @@ func _apply_gaze() -> void:
|
|||
)
|
||||
|
||||
|
||||
func set_face_target(_face_position: Vector2) -> void:
|
||||
func set_face_target(face_position: Vector2) -> void:
|
||||
if _current_mode == GazeMode.FACE_TO_FACE:
|
||||
_target_gaze = Vector2(
|
||||
clampf(
|
||||
_face_position.x, -MAX_YAW, MAX_YAW
|
||||
),
|
||||
clampf(
|
||||
_face_position.y, -MAX_PITCH, MAX_PITCH
|
||||
),
|
||||
)
|
||||
_target_gaze = face_position.clampf(-1.0, 1.0)
|
||||
|
||||
|
||||
func _on_state_changed(
|
||||
|
|
|
|||
|
|
@ -34,6 +34,9 @@ const LLMClientScript = preload(
|
|||
const OrchestratorScript = preload(
|
||||
"res://scripts/companion/conversation_orchestrator.gd"
|
||||
)
|
||||
const WindowDragScript = preload(
|
||||
"res://scripts/companion/window_drag.gd"
|
||||
)
|
||||
const NodeUtilsScript = preload(
|
||||
"res://scripts/util/node_utils.gd"
|
||||
)
|
||||
|
|
@ -47,6 +50,8 @@ const SPEECH_SERVICE_URL: String = "http://localhost:8000"
|
|||
const LLM_URL: String = "http://localhost:11434"
|
||||
const LLM_MODEL: String = "llama3"
|
||||
|
||||
var _drag: Node
|
||||
|
||||
@onready var _avatar_root: Node3D = $AvatarRoot
|
||||
@onready var _camera: Camera3D = $Camera3D
|
||||
@onready var _audio_player: AudioStreamPlayer = (
|
||||
|
|
@ -56,10 +61,18 @@ const LLM_MODEL: String = "llama3"
|
|||
|
||||
func _ready() -> void:
|
||||
_setup_window()
|
||||
_setup_drag()
|
||||
_load_avatar()
|
||||
_setup_backend()
|
||||
|
||||
|
||||
func _setup_drag() -> void:
|
||||
_drag = WindowDragScript.new()
|
||||
_drag.name = "WindowDrag"
|
||||
add_child(_drag)
|
||||
_drag.load_position()
|
||||
|
||||
|
||||
func _setup_window() -> void:
|
||||
var ds := DisplayServer
|
||||
ds.window_set_flag(ds.WINDOW_FLAG_TRANSPARENT, true)
|
||||
|
|
|
|||
55
godot/scripts/companion/window_drag.gd
Normal file
55
godot/scripts/companion/window_drag.gd
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
extends Node
|
||||
## Click and drag anywhere on the window to move it.
|
||||
## Persists position to user://window_position.cfg.
|
||||
|
||||
const SAVE_PATH: String = "user://window_position.cfg"
|
||||
|
||||
var _dragging: bool = false
|
||||
var _drag_start_window: Vector2i = Vector2i.ZERO
|
||||
var _drag_start_mouse: Vector2i = Vector2i.ZERO
|
||||
|
||||
|
||||
func _unhandled_input(event: InputEvent) -> void:
|
||||
if event is InputEventMouseButton:
|
||||
var mb: InputEventMouseButton = event as InputEventMouseButton
|
||||
if mb.button_index == MOUSE_BUTTON_LEFT:
|
||||
if mb.pressed:
|
||||
_dragging = true
|
||||
_drag_start_mouse = (
|
||||
DisplayServer.mouse_get_position()
|
||||
)
|
||||
_drag_start_window = (
|
||||
DisplayServer.window_get_position()
|
||||
)
|
||||
else:
|
||||
_dragging = false
|
||||
save_position()
|
||||
|
||||
elif event is InputEventMouseMotion and _dragging:
|
||||
var current_mouse := DisplayServer.mouse_get_position()
|
||||
var delta := current_mouse - _drag_start_mouse
|
||||
DisplayServer.window_set_position(
|
||||
_drag_start_window + delta
|
||||
)
|
||||
|
||||
|
||||
func load_position() -> bool:
|
||||
var cfg := ConfigFile.new()
|
||||
if cfg.load(SAVE_PATH) != OK:
|
||||
return false
|
||||
|
||||
var x: int = cfg.get_value("window", "x", -1)
|
||||
var y: int = cfg.get_value("window", "y", -1)
|
||||
if x < 0 or y < 0:
|
||||
return false
|
||||
|
||||
DisplayServer.window_set_position(Vector2i(x, y))
|
||||
return true
|
||||
|
||||
|
||||
func save_position() -> void:
|
||||
var pos := DisplayServer.window_get_position()
|
||||
var cfg := ConfigFile.new()
|
||||
cfg.set_value("window", "x", pos.x)
|
||||
cfg.set_value("window", "y", pos.y)
|
||||
cfg.save(SAVE_PATH)
|
||||
3
run
Executable file
3
run
Executable file
|
|
@ -0,0 +1,3 @@
|
|||
#!/usr/bin/env bash
|
||||
# Launch the Chobit companion app
|
||||
exec flatpak run --user org.godotengine.Godot --path "$(dirname "$0")/godot" "$@"
|
||||
Loading…
Add table
Reference in a new issue