Skip to content

Commit b629c6e

Browse files
rdmclin2rdmclin2
committed
✨ feat: 添加 loadVMDAnimation.ts
1 parent 1907623 commit b629c6e

File tree

10 files changed

+129
-49
lines changed

10 files changed

+129
-49
lines changed

‎src/features/AgentViewer/index.tsx

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,9 @@ function AgentViewer(props: Props) {
8585
break;
8686
}
8787
case 'vmd': {
88-
file.arrayBuffer().then((data) => {
89-
viewer.model?.dance(data);
90-
});
91-
88+
const blob = new Blob([file], { type: 'application/octet-stream' });
89+
const url = window.URL.createObjectURL(blob);
90+
viewer.model?.loadVMD(url);
9291
break;
9392
}
9493
// No default

‎src/features/DanceList/Item/index.tsx

Lines changed: 38 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
1-
import { Avatar } from '@lobehub/ui';
1+
import { Avatar, Icon } from '@lobehub/ui';
22
import { useHover } from 'ahooks';
33
import { Progress, Typography } from 'antd';
4+
import { Pause, Play } from 'lucide-react';
45
import React, { memo, useRef, useState } from 'react';
6+
import { useTranslation } from 'react-i18next';
57

68
import ListItem from '@/components/ListItem';
79
import Actions from '@/features/DanceList/Item/Actions';
@@ -24,28 +26,40 @@ const DanceItem = (props: DanceItemProps) => {
2426
const [open, setOpen] = useState(false);
2527

2628
const { styles } = useStyles();
27-
const [currentPlayId, setCurrentPlayId] = useDanceStore((s) => [
29+
const [currentPlayId, currentIdentifier, activateDance, setCurrentPlayId] = useDanceStore((s) => [
2830
s.currentPlayId,
31+
s.currentIdentifier,
32+
s.activateDance,
2933
s.setCurrentPlayId,
3034
]);
3135

36+
const [isPlaying, setIsPlaying] = useGlobalStore((s) => [s.isPlaying, s.setIsPlaying]);
37+
3238
const isCurrentPlay = currentPlayId ? currentPlayId === danceItem.danceId : false;
39+
const isSelected = currentIdentifier === danceItem.danceId;
3340
const hoverRef = useRef(null);
3441
const isHovered = useHover(hoverRef);
42+
const { t } = useTranslation('common');
3543

3644
const { downloading: audioDownloading, percent: audioPercent, fetchAudioBuffer } = useLoadAudio();
3745
const { downloading: danceDownloading, percent: dancePercent, fetchDanceBuffer } = useLoadDance();
3846
const viewer = useGlobalStore((s) => s.viewer);
3947

4048
const handlePlayPause = () => {
41-
const audioPromise = fetchAudioBuffer(danceItem.danceId, danceItem.audio);
42-
const dancePromise = fetchDanceBuffer(danceItem.danceId, danceItem.src);
43-
Promise.all([dancePromise, audioPromise]).then((res) => {
44-
if (!res) return;
45-
const [danceBuffer, audioBuffer] = res;
46-
viewer.model?.dance(danceBuffer, audioBuffer);
47-
});
48-
setCurrentPlayId(danceItem.danceId);
49+
viewer.model?.resetToIdle();
50+
if (isPlaying && isCurrentPlay) {
51+
setIsPlaying(false);
52+
} else {
53+
setCurrentPlayId(danceItem.danceId);
54+
setIsPlaying(true);
55+
const audioPromise = fetchAudioBuffer(danceItem.danceId, danceItem.audio);
56+
const dancePromise = fetchDanceBuffer(danceItem.danceId, danceItem.src);
57+
Promise.all([dancePromise, audioPromise]).then((res) => {
58+
if (!res) return;
59+
const [danceBuffer, audioBuffer] = res;
60+
viewer.model?.dance(danceBuffer, { data: audioBuffer });
61+
});
62+
}
4963
};
5064

5165
return (
@@ -64,11 +78,23 @@ const DanceItem = (props: DanceItemProps) => {
6478
) : null,
6579
<Actions danceItem={danceItem} setOpen={setOpen} key={`actions-${danceItem.danceId}`} />,
6680
]}
67-
onClick={handlePlayPause}
81+
onClick={() => {
82+
activateDance(danceItem.danceId);
83+
}}
84+
onDoubleClick={handlePlayPause}
6885
className={styles.listItem}
6986
avatar={
7087
<div style={{ position: 'relative' }}>
7188
<Avatar src={danceItem?.thumb} shape={'square'} size={48} />
89+
{isHovered || isCurrentPlay ? (
90+
<div className={styles.mask} onClick={handlePlayPause}>
91+
<Icon
92+
icon={isCurrentPlay && isPlaying ? Pause : Play}
93+
title={isCurrentPlay && isPlaying ? t('actions.pause') : t('actions.play')}
94+
className={styles.playIcon}
95+
/>
96+
</div>
97+
) : null}
7298
</div>
7399
}
74100
title={danceItem?.name}
@@ -77,7 +103,7 @@ const DanceItem = (props: DanceItemProps) => {
77103
{danceItem?.author}
78104
</Text>
79105
}
80-
active={isCurrentPlay || isHovered}
106+
active={isSelected || isHovered}
81107
/>
82108
);
83109
};

‎src/features/audioPlayer/audioPlayer.ts

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,11 @@ export class AudioPlayer {
77
this.bufferSource = undefined;
88
}
99

10-
public async playFromArrayBuffer(buffer: ArrayBuffer, onEnded?: () => void) {
10+
public async playFromArrayBuffer(
11+
buffer: ArrayBuffer,
12+
onEnded?: () => void,
13+
onProgress?: () => void,
14+
) {
1115
const audioBuffer = await this.audio.decodeAudioData(buffer);
1216

1317
this.bufferSource = this.audio.createBufferSource();
@@ -18,10 +22,16 @@ export class AudioPlayer {
1822
if (onEnded) {
1923
this.bufferSource.addEventListener('ended', onEnded);
2024
}
25+
if (onProgress) {
26+
this.bufferSource.addEventListener('time', onProgress);
27+
}
2128
}
2229

2330
public stopPlay() {
24-
if (this.bufferSource) this.bufferSource.stop();
31+
if (this.bufferSource) {
32+
this.bufferSource.stop();
33+
this.bufferSource = undefined;
34+
}
2535
}
2636

2737
public async playFromURL(url: string, onEnded?: () => void) {

‎src/features/lipSync/lipSync.ts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,10 @@ export class LipSync {
4848
}
4949

5050
public stopPlay() {
51-
if (this.bufferSource) this.bufferSource.stop();
51+
if (this.bufferSource) {
52+
this.bufferSource.stop();
53+
this.bufferSource = undefined;
54+
}
5255
}
5356

5457
public async playFromURL(url: string, onEnded?: () => void) {

‎src/features/vrmViewer/model.ts

Lines changed: 41 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,15 @@ import { VRM, VRMLoaderPlugin, VRMUtils } from '@pixiv/three-vrm';
22
import * as THREE from 'three';
33
import { AnimationAction, AnimationClip } from 'three';
44
import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader';
5+
import { LoopOnce } from 'three/src/constants';
56

67
import { AudioPlayer } from '@/features/audioPlayer/audioPlayer';
8+
import { loadMixamoAnimation } from '@/libs/FBXAnimation/loadMixamoAnimation';
9+
import { loadVMDAnimation } from '@/libs/VMDAnimation/loadVMDAnimation';
710
import { convert } from '@/libs/VMDAnimation/vmd2vrmanim';
811
import { bindToVRM, toOffset } from '@/libs/VMDAnimation/vmd2vrmanim.binding';
912
import IKHandler from '@/libs/VMDAnimation/vrm-ik-handler';
1013
import { VRMAnimation } from '@/libs/VRMAnimation/VRMAnimation';
11-
import { loadMixamoAnimation } from '@/libs/VRMAnimation/loadMixamoAnimation';
1214
import { loadVRMAnimation } from '@/libs/VRMAnimation/loadVRMAnimation';
1315
import { VRMLookAtSmootherLoaderPlugin } from '@/libs/VRMLookAtSmootherLoaderPlugin/VRMLookAtSmootherLoaderPlugin';
1416
import { Screenplay } from '@/types/touch';
@@ -78,14 +80,17 @@ export class Model {
7880
}
7981

8082
public disposeAll() {
81-
const { vrm, mixer } = this;
82-
83-
if (!vrm || !mixer) {
84-
console.error('You have to load VRM first');
85-
return;
83+
const { mixer } = this;
84+
85+
if (mixer) {
86+
mixer.stopAllAction();
87+
if (this._clip) {
88+
mixer.uncacheAction(this._clip);
89+
mixer.uncacheClip(this._clip);
90+
this._clip = undefined;
91+
}
8692
}
8793

88-
mixer.stopAllAction();
8994
this.ikHandler?.disableAll();
9095
if (this._action) {
9196
this._action.stop();
@@ -96,12 +101,6 @@ export class Model {
96101
this._audioPlayer?.stopPlay();
97102
this._audio = undefined;
98103
}
99-
100-
if (this._clip) {
101-
mixer.uncacheAction(this._clip);
102-
mixer.uncacheClip(this._clip);
103-
this._clip = undefined;
104-
}
105104
}
106105

107106
/**
@@ -124,7 +123,7 @@ export class Model {
124123

125124
public async loadIdleAnimation() {
126125
const vrma = await loadVRMAnimation('/idle_loop.vrma');
127-
if (vrma) this.loadAnimation(vrma);
126+
if (vrma) await this.loadAnimation(vrma);
128127
}
129128

130129
public async loadFBX(animationUrl: string) {
@@ -142,22 +141,44 @@ export class Model {
142141
}
143142
}
144143

144+
public async loadVMD(animationUrl: string) {
145+
const { vrm, mixer } = this;
146+
147+
if (vrm && mixer) {
148+
this.disposeAll();
149+
const clip = await loadVMDAnimation(animationUrl, vrm);
150+
const action = mixer.clipAction(clip);
151+
action.play();
152+
this._action = action;
153+
this._clip = clip;
154+
}
155+
}
156+
145157
/**
146158
* 播放舞蹈
147-
* @param audio ArrayBuffer
148-
* @param dance ArrayBuffer
159+
* @param audio
160+
* @param dance
149161
*/
150-
public async dance(dance: ArrayBuffer, audio?: ArrayBuffer) {
162+
public async dance(
163+
dance: ArrayBuffer,
164+
audio?: {
165+
data: ArrayBuffer;
166+
onEnd?: () => void;
167+
},
168+
) {
151169
const { vrm, mixer } = this;
152170
if (vrm && mixer) {
153171
this.disposeAll();
154172
const animation = convert(dance, toOffset(vrm));
155173
const clip = bindToVRM(animation, vrm);
156174
const action = mixer.clipAction(clip);
157-
action.play(); // play animation
175+
action.setLoop(LoopOnce, 1).play(); // play animation
158176
if (audio) {
159-
this._audioPlayer?.playFromArrayBuffer(audio);
160-
this._audio = audio;
177+
this._audioPlayer?.playFromArrayBuffer(audio.data, () => {
178+
this.resetToIdle();
179+
audio.onEnd?.();
180+
});
181+
this._audio = audio.data;
161182
}
162183

163184
this._action = action;

‎src/libs/VRMAnimation/loadMixamoAnimation.ts renamed to ‎src/libs/FBXAnimation/loadMixamoAnimation.ts

File renamed without changes.

‎src/libs/VRMAnimation/mixamoVRMRigMap.ts renamed to ‎src/libs/FBXAnimation/mixamoVRMRigMap.ts

File renamed without changes.

‎src/libs/VMDAnimation/loadVMDAnimation.ts

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,14 @@
1-
// export async function loadVMDAnimation(url: string): Promise {
2-
// const gltf = await loader.loadAsync(url);
1+
import { VRM } from '@pixiv/three-vrm';
32

4-
// const vrmAnimations: VRMAnimation[] = gltf.userData.vrmAnimations;
5-
// const vrmAnimation: VRMAnimation | undefined = vrmAnimations[0];
3+
import { convert } from '@/libs/VMDAnimation/vmd2vrmanim';
4+
import { bindToVRM, toOffset } from '@/libs/VMDAnimation/vmd2vrmanim.binding';
65

7-
// return vrmAnimation ?? null;
8-
// }
6+
export async function loadVMDAnimation(url: string, vrm: VRM) {
7+
const res = await fetch(url);
8+
const buffer = await res.arrayBuffer();
9+
10+
const animation = convert(buffer, toOffset(vrm));
11+
const clip = bindToVRM(animation, vrm);
12+
13+
return clip ?? null;
14+
}

‎src/libs/VMDAnimation/vrm-ik-handler.ts

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ export default class VRMIKHandler {
100100

101101
private targets = new Map<number, Object3D>();
102102
private iks = new Map<number, IKS>();
103+
// private vector = new Vector3();
103104
private bones: Bone[];
104105
private root: Object3D;
105106

@@ -151,7 +152,7 @@ export default class VRMIKHandler {
151152
effector: leftToeId,
152153
target: leftToeId,
153154
iteration: 3,
154-
maxAngle: 1,
155+
maxAngle: 4,
155156
links: [
156157
{
157158
enabled: false,
@@ -164,7 +165,7 @@ export default class VRMIKHandler {
164165
effector: rightToeId,
165166
target: rightToeId,
166167
iteration: 3,
167-
maxAngle: 1,
168+
maxAngle: 4,
168169
links: [
169170
{
170171
enabled: false,
@@ -223,7 +224,17 @@ export default class VRMIKHandler {
223224
if (ik.maxAngle != null && angle > ik.maxAngle) angle = ik.maxAngle;
224225
axis.crossVectors(effectorVec, targetVec).normalize();
225226
link.quaternion.multiply(quaternion.setFromAxisAngle(axis, angle));
227+
226228
clampVector3ByRadian(link.rotation, rotationMin, rotationMax);
229+
230+
// if (rotationMin) {
231+
// link.rotation.setFromVector3(this.vector.setFromEuler(link.rotation).max(rotationMin));
232+
// }
233+
//
234+
// if (rotationMax) {
235+
// link.rotation.setFromVector3(this.vector.setFromEuler(link.rotation).min(rotationMax));
236+
// }
237+
227238
link.updateMatrixWorld(true);
228239
rotated = true;
229240
}

‎src/store/global/index.ts

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ export interface GlobalStore {
2323
* @param key
2424
*/
2525
focusPanel: (key: PanelKey) => void;
26+
isPlaying: boolean;
2627
/**
2728
* Open panel
2829
* @param key
@@ -31,6 +32,7 @@ export interface GlobalStore {
3132
panel: PanelConfig;
3233
setChatDialog: (show: boolean) => void;
3334
setChatSidebar: (show: boolean) => void;
35+
setIsPlaying: (isPlaying: boolean) => void;
3436
/**
3537
* Set panel config
3638
* @param panel
@@ -43,7 +45,6 @@ export interface GlobalStore {
4345
showChatDialog: boolean;
4446
showChatSidebar: boolean;
4547
showRoleList: boolean;
46-
4748
showSessionList: boolean;
4849
/**
4950
* 主题模式
@@ -62,6 +63,7 @@ const initialState = {
6263
* 主题模式
6364
*/
6465
themeMode: 'auto' as ThemeMode,
66+
isPlaying: false,
6567
showChatSidebar: false,
6668
showSessionList: true,
6769
showChatDialog: true,
@@ -90,7 +92,9 @@ const initialState = {
9092
export const useGlobalStore = createWithEqualityFn<GlobalStore>()(
9193
(set, get) => ({
9294
...initialState,
93-
95+
setIsPlaying: (isPlaying: boolean) => {
96+
set({ isPlaying: isPlaying });
97+
},
9498
closePanel: (key: PanelKey) => {
9599
const { setPanel, focusList } = get();
96100
setPanel(key, { open: false });

0 commit comments

Comments
 (0)