Skip to content

Commit 4f66c31

Browse files
rdmclin2rdmclin2
committed
✨ feat: add model playing
1 parent 7d1f049 commit 4f66c31

File tree

10 files changed

+196
-33
lines changed

10 files changed

+196
-33
lines changed

‎scripts/mixamo/Motion/Female/Greeting/input.json

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
[
2+
{
3+
"id": "c9c7996a-b96c-11e4-a802-0aaa78deedf9",
4+
"type": "Motion",
5+
"description": "Greeting While Standing",
6+
"category": "",
7+
"character_type": "human",
8+
"name": "Standing Greeting",
9+
"thumbnail": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/113350901/static.png",
10+
"thumbnail_animated": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/113350901/animated.gif",
11+
"motion_id": "c9c7996a-b96c-11e4-a802-0aaa78deedf9",
12+
"motions": null,
13+
"source": "system"
14+
},
15+
{
16+
"id": "c9c9f343-b96c-11e4-a802-0aaa78deedf9",
17+
"type": "Motion",
18+
"description": "2 People Shaking Hands Part 1 - Female",
19+
"category": "",
20+
"character_type": "human",
21+
"name": "Shaking Hands 1",
22+
"thumbnail": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119070901/static.png",
23+
"thumbnail_animated": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119070901/animated.gif",
24+
"motion_id": "c9c9f343-b96c-11e4-a802-0aaa78deedf9",
25+
"motions": null,
26+
"source": "system"
27+
}
28+
]

‎scripts/mixamo/Motion/Male/Greeting/input.json

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
[
2+
{
3+
"id": "c9c9f403-b96c-11e4-a802-0aaa78deedf9",
4+
"type": "Motion",
5+
"description": "2 People Shaking Hands Part 2 - Male",
6+
"category": "",
7+
"character_type": "human",
8+
"name": "Shaking Hands 2",
9+
"thumbnail": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119070902/static.png",
10+
"thumbnail_animated": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119070902/animated.gif",
11+
"motion_id": "c9c9f403-b96c-11e4-a802-0aaa78deedf9",
12+
"motions": null,
13+
"source": "system"
14+
},
15+
{
16+
"id": "c9c9f4bc-b96c-11e4-a802-0aaa78deedf9",
17+
"type": "Motion",
18+
"description": "Quick Formal Bow",
19+
"category": "",
20+
"character_type": "human",
21+
"name": "Quick Formal Bow",
22+
"thumbnail": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119110901/static.png",
23+
"thumbnail_animated": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119110901/animated.gif",
24+
"motion_id": "c9c9f4bc-b96c-11e4-a802-0aaa78deedf9",
25+
"motions": null,
26+
"source": "system"
27+
},
28+
{
29+
"id": "c9cb0ab6-b96c-11e4-a802-0aaa78deedf9",
30+
"type": "Motion",
31+
"description": "Formal Military Salute",
32+
"category": "",
33+
"character_type": "human",
34+
"name": "Salute",
35+
"thumbnail": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/120580901/static.png",
36+
"thumbnail_animated": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/120580901/animated.gif",
37+
"motion_id": "c9cb0ab6-b96c-11e4-a802-0aaa78deedf9",
38+
"motions": null,
39+
"source": "system"
40+
},
41+
{
42+
"id": "c9c9f57a-b96c-11e4-a802-0aaa78deedf9",
43+
"type": "Motion",
44+
"description": "Quick Informal Bow",
45+
"category": "",
46+
"character_type": "human",
47+
"name": "Quick Informal Bow",
48+
"thumbnail": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119110902/static.png",
49+
"thumbnail_animated": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119110902/animated.gif",
50+
"motion_id": "c9c9f57a-b96c-11e4-a802-0aaa78deedf9",
51+
"motions": null,
52+
"source": "system"
53+
}
54+
]

‎scripts/mixamo/index.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/**
22
* https://www.mixamo.com/
3-
* Mixamo Animations 脚本处理
3+
* Mixamo Animations 脚本处理, `bun index.ts`
44
* @author rdmclin2
55
*/
66
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';

‎src/animations/Motion/index.json

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -529,6 +529,46 @@
529529
"url": "https://r2.vidol.chat/animations/c9c7438e-b96c-11e4-a802-0aaa78deedf9.fbx",
530530
"avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/108840901/animated.gif"
531531
},
532+
{
533+
"id": "c9c9f403-b96c-11e4-a802-0aaa78deedf9",
534+
"name": "Shaking Hands 2",
535+
"type": "Motion",
536+
"gender": "Male",
537+
"category": "Greeting",
538+
"description": "2 People Shaking Hands Part 2 - Male",
539+
"url": "https://r2.vidol.chat/animations/c9c9f403-b96c-11e4-a802-0aaa78deedf9.fbx",
540+
"avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119070902/animated.gif"
541+
},
542+
{
543+
"id": "c9c9f4bc-b96c-11e4-a802-0aaa78deedf9",
544+
"name": "Quick Formal Bow",
545+
"type": "Motion",
546+
"gender": "Male",
547+
"category": "Greeting",
548+
"description": "Quick Formal Bow",
549+
"url": "https://r2.vidol.chat/animations/c9c9f4bc-b96c-11e4-a802-0aaa78deedf9.fbx",
550+
"avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119110901/animated.gif"
551+
},
552+
{
553+
"id": "c9cb0ab6-b96c-11e4-a802-0aaa78deedf9",
554+
"name": "Salute",
555+
"type": "Motion",
556+
"gender": "Male",
557+
"category": "Greeting",
558+
"description": "Formal Military Salute",
559+
"url": "https://r2.vidol.chat/animations/c9cb0ab6-b96c-11e4-a802-0aaa78deedf9.fbx",
560+
"avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/120580901/animated.gif"
561+
},
562+
{
563+
"id": "c9c9f57a-b96c-11e4-a802-0aaa78deedf9",
564+
"name": "Quick Informal Bow",
565+
"type": "Motion",
566+
"gender": "Male",
567+
"category": "Greeting",
568+
"description": "Quick Informal Bow",
569+
"url": "https://r2.vidol.chat/animations/c9c9f57a-b96c-11e4-a802-0aaa78deedf9.fbx",
570+
"avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119110902/animated.gif"
571+
},
532572
{
533573
"id": "c9c609a9-b96c-11e4-a802-0aaa78deedf9",
534574
"name": "Walking",
@@ -1019,6 +1059,26 @@
10191059
"url": "https://r2.vidol.chat/animations/c9c6e172-b96c-11e4-a802-0aaa78deedf9.fbx",
10201060
"avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/108660901/animated.gif"
10211061
},
1062+
{
1063+
"id": "c9c7996a-b96c-11e4-a802-0aaa78deedf9",
1064+
"name": "Standing Greeting",
1065+
"type": "Motion",
1066+
"gender": "Female",
1067+
"category": "Greeting",
1068+
"description": "Greeting While Standing",
1069+
"url": "https://r2.vidol.chat/animations/c9c7996a-b96c-11e4-a802-0aaa78deedf9.fbx",
1070+
"avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/113350901/animated.gif"
1071+
},
1072+
{
1073+
"id": "c9c9f343-b96c-11e4-a802-0aaa78deedf9",
1074+
"name": "Shaking Hands 1",
1075+
"type": "Motion",
1076+
"gender": "Female",
1077+
"category": "Greeting",
1078+
"description": "2 People Shaking Hands Part 1 - Female",
1079+
"url": "https://r2.vidol.chat/animations/c9c9f343-b96c-11e4-a802-0aaa78deedf9.fbx",
1080+
"avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119070901/animated.gif"
1081+
},
10221082
{
10231083
"id": "c9c98a38-b96c-11e4-a802-0aaa78deedf9",
10241084
"name": "Happy",

‎src/animations/Posture/index.json

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -639,6 +639,16 @@
639639
"url": "https://r2.vidol.chat/animations/27fe7228-487a-4951-af65-30cef1a9183d.fbx",
640640
"avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/140000905/animated.gif"
641641
},
642+
{
643+
"id": "8c95cac8-2068-410f-9564-d0a2a04ea0a5",
644+
"name": "Female Action Pose",
645+
"type": "Posture",
646+
"gender": "Female",
647+
"category": "Action",
648+
"description": "Sliding",
649+
"url": "https://r2.vidol.chat/animations/8c95cac8-2068-410f-9564-d0a2a04ea0a5.fbx",
650+
"avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/140000902/animated.gif"
651+
},
642652
{
643653
"id": "eaf51ff4-0fb7-4759-aadf-edb8fd3ae6a0",
644654
"name": "Female Action Pose",
@@ -719,16 +729,6 @@
719729
"url": "https://r2.vidol.chat/animations/e3543986-d43d-4ec1-bac8-5cf7f5ec003b.fbx",
720730
"avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/140100902/animated.gif"
721731
},
722-
{
723-
"id": "8c95cac8-2068-410f-9564-d0a2a04ea0a5",
724-
"name": "Female Action Pose",
725-
"type": "Posture",
726-
"gender": "Female",
727-
"category": "Crouch",
728-
"description": "Sliding",
729-
"url": "https://r2.vidol.chat/animations/8c95cac8-2068-410f-9564-d0a2a04ea0a5.fbx",
730-
"avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/140000902/animated.gif"
731-
},
732732
{
733733
"id": "31dca3e7-dba8-4536-a70e-34d7d5976fa9",
734734
"name": "Female Dance Pose",

‎src/app/role/RoleEdit/Touch/ActionList/Actions/Play.tsx

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,6 @@ export default memo((props: Props) => {
5454
...currentAgentTTS,
5555
message: touchAction.text,
5656
},
57-
motion: touchAction.motion,
5857
},
5958
viewer,
6059
() => {},

‎src/constants/touch.tsx

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import {
1212
export const HAPPY_MOTION_ID = 'c9c98a38-b96c-11e4-a802-0aaa78deedf9';
1313
export const ANGRY_MOTION_ID = 'c9c98b02-b96c-11e4-a802-0aaa78deedf9';
1414
export const RELAX_MOTION_ID = 'c9c98361-b96c-11e4-a802-0aaa78deedf9';
15+
export const GREETING_MOTION_ID = 'c9c7996a-b96c-11e4-a802-0aaa78deedf9';
1516

1617
export const DEFAULT_TOUCH_ACTION_CONFIG_FEMALE: TouchActionConfig = {
1718
[TouchAreaEnum.Head]: [

‎src/features/AgentViewer/index.tsx

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
1+
import { VRMExpressionPresetName } from '@pixiv/three-vrm';
12
import { Progress } from 'antd';
23
import classNames from 'classnames';
34
import React, { memo, useCallback, useRef } from 'react';
45
import { useTranslation } from 'react-i18next';
56

67
import PageLoading from '@/components/PageLoading';
8+
import { DEFAULT_MOTION_ANIMATION, GREETING_MOTION_ID } from '@/constants/touch';
79
import { useLoadModel } from '@/hooks/useLoadModel';
10+
import { speakCharacter } from '@/libs/messages/speakCharacter';
811
import { useGlobalStore } from '@/store/global';
912
import { Agent } from '@/types/agent';
13+
import { fetchWithProgress } from '@/utils/fetch';
1014

1115
import ToolBar from './ToolBar';
1216
import { useStyles } from './style';
@@ -43,14 +47,38 @@ function AgentViewer(props: Props) {
4347
loadingScreen.append(loader);
4448
agentViewer.append(loadingScreen);
4549

46-
// load
50+
// load vrm
4751
await viewer.loadVrm(modelUrl);
4852

49-
// remove loading dom
50-
loadingScreen.classList.add('fade-out');
51-
loadingScreen.addEventListener('transitionend', () => {
52-
loadingScreen.remove();
53-
});
53+
// load motion
54+
let motionUrl = undefined;
55+
const item = DEFAULT_MOTION_ANIMATION.find((item) => item.id === GREETING_MOTION_ID);
56+
if (item) {
57+
const blob = await fetchWithProgress(item.url);
58+
motionUrl = window.URL.createObjectURL(blob);
59+
}
60+
61+
speakCharacter(
62+
{
63+
emotion: VRMExpressionPresetName.Neutral,
64+
tts: {
65+
...agent.tts,
66+
message: agent.greeting,
67+
},
68+
motion: motionUrl,
69+
},
70+
viewer,
71+
() => {
72+
// remove loading dom
73+
loadingScreen.classList.add('fade-out');
74+
loadingScreen.addEventListener('transitionend', () => {
75+
loadingScreen.remove();
76+
});
77+
},
78+
() => {
79+
viewer.model?.loadIdleAnimation();
80+
},
81+
);
5482
}
5583
});
5684

‎src/features/AgentViewer/style.ts

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,10 @@ export const useStyles = createStyles(({ css, token }) => ({
2525
opacity: 0;
2626
}
2727
28+
#loading-screen.fade-in {
29+
opacity: 0;
30+
}
31+
2832
#loader {
2933
position: relative;
3034
top: 50%;
@@ -39,8 +43,6 @@ export const useStyles = createStyles(({ css, token }) => ({
3943
border: 3px solid transparent;
4044
border-top-color: #9370db;
4145
border-radius: 50%;
42-
43-
animation: spin 2s linear infinite;
4446
animation: spin 2s linear infinite;
4547
}
4648
@@ -53,8 +55,6 @@ export const useStyles = createStyles(({ css, token }) => ({
5355
border: 3px solid transparent;
5456
border-top-color: #ba55d3;
5557
border-radius: 50%;
56-
57-
animation: spin 3s linear infinite;
5858
animation: spin 3s linear infinite;
5959
}
6060
@@ -67,36 +67,26 @@ export const useStyles = createStyles(({ css, token }) => ({
6767
border: 3px solid transparent;
6868
border-top-color: #f0f;
6969
border-radius: 50%;
70-
71-
animation: spin 1.5s linear infinite;
7270
animation: spin 1.5s linear infinite;
7371
}
7472
7573
@keyframes spin {
7674
0% {
7775
transform: rotate(0deg);
78-
transform: rotate(0deg);
79-
transform: rotate(0deg);
8076
}
8177
8278
100% {
8379
transform: rotate(360deg);
84-
transform: rotate(360deg);
85-
transform: rotate(360deg);
8680
}
8781
}
8882
8983
@keyframes spin {
9084
0% {
9185
transform: rotate(0deg);
92-
transform: rotate(0deg);
93-
transform: rotate(0deg);
9486
}
9587
9688
100% {
9789
transform: rotate(360deg);
98-
transform: rotate(360deg);
99-
transform: rotate(360deg);
10090
}
10191
}
10292
`,

‎src/libs/vrmViewer/model.ts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ export class Model {
116116
if (vrma) await this.loadAnimation(vrma);
117117
}
118118

119-
public async loadFBX(animationUrl: string) {
119+
public async loadFBX(animationUrl: string, loop: boolean = true) {
120120
const { vrm, mixer } = this;
121121

122122
if (vrm && mixer) {
@@ -125,6 +125,8 @@ export class Model {
125125
const clip = await loadMixamoAnimation(animationUrl, vrm);
126126
// Apply the loaded animation to mixer and play
127127
const action = mixer.clipAction(clip);
128+
if (!loop) action.setLoop(LoopOnce, 1);
129+
128130
action.play();
129131
this._action = action;
130132
this._clip = clip;
@@ -157,6 +159,7 @@ export class Model {
157159
*/
158160
public async speak(buffer: ArrayBuffer, screenplay: Screenplay) {
159161
this.emoteController?.playEmotion(screenplay.emotion);
162+
if (screenplay.motion) this.loadFBX(screenplay.motion);
160163
await new Promise((resolve) => {
161164
this._lipSync?.playFromArrayBuffer(buffer, () => {
162165
resolve(true);

0 commit comments

Comments
 (0)