Skip to content

Commit 0e57daa

Browse files
rdmclin2rdmclin2
committed
✨ feat: use viewer dance
1 parent fccf0b6 commit 0e57daa

File tree

8 files changed

+52
-33
lines changed

8 files changed

+52
-33
lines changed

‎src/features/AgentViewer/ToolBar/index.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ const ToolBar = (props: ToolBarProps) => {
6969
break;
7070
}
7171
case 'power': {
72-
viewer.model?.resetToIdle();
72+
viewer.resetToIdle();
7373
break;
7474
}
7575
case 'screenShot': {

‎src/features/DanceList/Item/index.tsx

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,18 +46,17 @@ const DanceItem = (props: DanceItemProps) => {
4646
const viewer = useGlobalStore((s) => s.viewer);
4747

4848
const handlePlayPause = async () => {
49-
viewer.model?.disposeAll();
5049
if (isPlaying && isCurrentPlay) {
5150
setIsPlaying(false);
52-
viewer.model?.loadIdleAnimation();
51+
viewer?.resetToIdle();
5352
} else {
5453
setCurrentPlayId(danceItem.danceId);
5554
setIsPlaying(true);
5655
const audioPromise = fetchAudioUrl(danceItem.danceId, danceItem.audio);
5756
const dancePromise = fetchDanceUrl(danceItem.danceId, danceItem.src);
5857
const [danceUrl, audioUrl] = await Promise.all([dancePromise, audioPromise]);
5958
if (danceUrl && audioUrl)
60-
viewer.model?.dance(danceUrl, audioUrl, () => {
59+
viewer?.dance(danceUrl, audioUrl, () => {
6160
setIsPlaying(false);
6261
});
6362
}

‎src/features/audioPlayer/audioPlayer.ts renamed to ‎src/libs/audioPlayer/audioPlayer.ts

File renamed without changes.

‎src/features/messages/speakCharacter.ts renamed to ‎src/libs/messages/speakCharacter.ts

File renamed without changes.

‎src/libs/vrmViewer/model.ts

Lines changed: 4 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import { AnimationAction, AnimationClip } from 'three';
44
import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader';
55
import { LoopOnce } from 'three/src/constants';
66

7-
import { AudioPlayer } from '@/features/audioPlayer/audioPlayer';
87
import { loadMixamoAnimation } from '@/libs/FBXAnimation/loadMixamoAnimation';
98
import { loadVMDAnimation } from '@/libs/VMDAnimation/loadVMDAnimation';
109
import { convert } from '@/libs/VMDAnimation/vmd2vrmanim';
@@ -13,6 +12,7 @@ import IKHandler from '@/libs/VMDAnimation/vrm-ik-handler';
1312
import { VRMAnimation } from '@/libs/VRMAnimation/VRMAnimation';
1413
import { loadVRMAnimation } from '@/libs/VRMAnimation/loadVRMAnimation';
1514
import { VRMLookAtSmootherLoaderPlugin } from '@/libs/VRMLookAtSmootherLoaderPlugin/VRMLookAtSmootherLoaderPlugin';
15+
import { AudioPlayer } from '@/libs/audioPlayer/audioPlayer';
1616
import { EmoteController } from '@/libs/emoteController/emoteController';
1717
import { LipSync } from '@/libs/lipSync/lipSync';
1818
import { Screenplay } from '@/types/touch';
@@ -28,14 +28,13 @@ export class Model {
2828

2929
private _lookAtTargetParent: THREE.Object3D;
3030
private _lipSync?: LipSync;
31-
private _audioPlayer?: AudioPlayer;
31+
3232
private _action: AnimationAction | undefined;
3333
private _clip: AnimationClip | undefined;
3434

3535
constructor(lookAtTargetParent: THREE.Object3D) {
3636
this._lookAtTargetParent = lookAtTargetParent;
3737
this._lipSync = new LipSync(new AudioContext());
38-
this._audioPlayer = new AudioPlayer(new AudioContext());
3938
this._action = undefined;
4039
this._clip = undefined;
4140
}
@@ -93,8 +92,6 @@ export class Model {
9392
this._action.stop();
9493
this._action = undefined;
9594
}
96-
97-
this._audioPlayer?.stopPlay();
9895
}
9996

10097
/**
@@ -135,39 +132,20 @@ export class Model {
135132
}
136133
}
137134

138-
public async loadVMD(animationUrl: string) {
135+
public async loadVMD(animationUrl: string, loop: boolean = true) {
139136
const { vrm, mixer } = this;
140137

141138
if (vrm && mixer) {
142139
this.disposeAll();
143140
const clip = await loadVMDAnimation(animationUrl, vrm);
144141
const action = mixer.clipAction(clip);
142+
if (!loop) action.setLoop(LoopOnce, 1);
145143
action.play();
146144
this._action = action;
147145
this._clip = clip;
148146
}
149147
}
150148

151-
/**
152-
* 播放舞蹈，以音乐文件的播放作为结束标志。
153-
*/
154-
public async dance(danceUrl: string, audioUrl: string, onEnd?: () => void) {
155-
const { vrm, mixer } = this;
156-
if (vrm && mixer) {
157-
this.disposeAll();
158-
const clip = await loadVMDAnimation(danceUrl, vrm);
159-
const action = mixer.clipAction(clip);
160-
action.setLoop(LoopOnce, 1).play(); // play animation
161-
this._audioPlayer?.playFromURL(audioUrl, () => {
162-
this.disposeAll();
163-
onEnd?.();
164-
});
165-
166-
this._action = action;
167-
this._clip = clip;
168-
}
169-
}
170-
171149
public async resetToIdle() {
172150
this.disposeAll();
173151

‎src/libs/vrmViewer/viewer.ts

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
import { Parser } from 'mmd-parser';
22
import * as THREE from 'three';
3-
import { GridHelper, Mesh, MeshLambertMaterial, PlaneGeometry } from 'three';
3+
import { Audio, GridHelper, Mesh, MeshLambertMaterial, PlaneGeometry } from 'three';
44
import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls';
5+
import { LoopOnce } from 'three/src/constants';
6+
7+
import { loadVMDAnimation } from '@/libs/VMDAnimation/loadVMDAnimation';
8+
import { AudioPlayer } from '@/libs/audioPlayer/audioPlayer';
59

610
import { Model } from './model';
711

@@ -12,6 +16,7 @@ export class Viewer {
1216
private _renderer?: THREE.WebGLRenderer;
1317
private _clock: THREE.Clock;
1418
private _scene: THREE.Scene;
19+
private _sound?: Audio;
1520
private _cameraHelper?: THREE.CameraHelper;
1621
private _camera?: THREE.PerspectiveCamera;
1722
private _cameraControls?: OrbitControls;
@@ -45,6 +50,36 @@ export class Viewer {
4550
this._clock.start();
4651
}
4752

53+
/**
54+
* 播放舞蹈，以音乐文件的播放作为结束标志。
55+
*/
56+
public async dance(danceUrl: string, audioUrl: string, onEnd?: () => void) {
57+
if (!this._sound || !this.model) {
58+
console.error('Audio Object or Model Object Not Existed');
59+
return null;
60+
}
61+
this._sound.stop();
62+
this.model?.disposeAll();
63+
const audioLoader = new THREE.AudioLoader();
64+
// 监听音频播放结束事件
65+
this._sound.onEnded = () => {
66+
onEnd?.();
67+
this.model?.loadIdleAnimation();
68+
};
69+
const buffer = await audioLoader.loadAsync(audioUrl);
70+
this._sound.setBuffer(buffer);
71+
this._sound.setVolume(0.5);
72+
this._sound.play();
73+
74+
this.model?.loadVMD(danceUrl, false);
75+
}
76+
77+
public resetToIdle() {
78+
this._sound?.stop();
79+
this.model?.disposeAll();
80+
this.model?.loadIdleAnimation();
81+
}
82+
4883
/**
4984
* 加载舞台
5085
* @param buffer
@@ -111,6 +146,13 @@ export class Viewer {
111146
this._cameraControls?.target.set(0, 0, 0);
112147
this._cameraControls.update();
113148

149+
// Audio 音频播放
150+
const listener = new THREE.AudioListener();
151+
this._camera.add(listener);
152+
153+
// 创建一个全局 audio 源
154+
this._sound = new THREE.Audio(listener);
155+
114156
const resizeObserver = new ResizeObserver(() => {
115157
setTimeout(() => this.resize(), 0);
116158
});

‎src/panels/RolePanel/RoleEdit/Touch/ActionList/Actions/Play.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { memo, useState } from 'react';
55
import { useTranslation } from 'react-i18next';
66

77
import { DEFAULT_MOTION_ANIMATION } from '@/constants/touch';
8-
import { speakCharacter } from '@/features/messages/speakCharacter';
8+
import { speakCharacter } from '@/libs/messages/speakCharacter';
99
import { agentSelectors, useAgentStore } from '@/store/agent';
1010
import { useGlobalStore } from '@/store/global';
1111
import { TouchAction } from '@/types/touch';

‎src/services/chat.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { OPENAI_API_KEY, OPENAI_END_POINT } from '@/constants/openai';
2-
import { speakCharacter } from '@/features/messages/speakCharacter';
2+
import { speakCharacter } from '@/libs/messages/speakCharacter';
33
import { useGlobalStore } from '@/store/global';
44
import { sessionSelectors, useSessionStore } from '@/store/session';
55
import { configSelectors, useSettingStore } from '@/store/setting';

0 commit comments

Comments
 (0)