Skip to content

Commit eab17e9

Browse files
AlmasBAlmasB
committed
feat: added intial impl hand tracking service
1 parent 1ad4d43 commit eab17e9

File tree

3 files changed

+187
-0
lines changed

3 files changed

+187
-0
lines changed

‎fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/gesturerecog/Hand.kt

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* FXGL - JavaFX Game Library. The MIT License (MIT).
3+
* Copyright (c) AlmasB (almaslvl@gmail.com).
4+
* See LICENSE for details.
5+
*/
6+
7+
package com.almasb.fxgl.gesturerecog
8+
9+
import javafx.geometry.Point3D
10+
11+
/**
12+
* Each hand has an id and a list of 21 hand landmarks points.
13+
* Each point can be mapped into 2D app (or screen where appropriate) space via:
14+
* appX = (1 - pointX) * appWidth
15+
* appY = pointY * appHeight
16+
*
17+
* @author Almas Baim (https://github.com/AlmasB)
18+
*/
19+
data class Hand(
20+
val id: Int,
21+
val points: List<Point3D>
22+
) {
23+
24+
fun getPoint(landmark: HandLandmark) = points[landmark.ordinal]
25+
}

‎fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/gesturerecog/HandLandmark.kt

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* FXGL - JavaFX Game Library. The MIT License (MIT).
3+
* Copyright (c) AlmasB (almaslvl@gmail.com).
4+
* See LICENSE for details.
5+
*/
6+
7+
package com.almasb.fxgl.gesturerecog
8+
9+
/**
10+
* The ordinal of each item matches the format defined at:
11+
* https://developers.google.com/mediapipe/solutions/vision/hand_landmarker
12+
*
13+
* @author Almas Baim (https://github.com/AlmasB)
14+
*/
15+
enum class HandLandmark {
16+
WRIST,
17+
18+
THUMB_CMC,
19+
THUMB_MCP,
20+
THUMB_IP,
21+
THUMB_TIP,
22+
23+
INDEX_FINGER_MCP,
24+
INDEX_FINGER_PIP,
25+
INDEX_FINGER_DIP,
26+
INDEX_FINGER_TIP,
27+
28+
MIDDLE_FINGER_MCP,
29+
MIDDLE_FINGER_PIP,
30+
MIDDLE_FINGER_DIP,
31+
MIDDLE_FINGER_TIP,
32+
33+
RING_FINGER_MCP,
34+
RING_FINGER_PIP,
35+
RING_FINGER_DIP,
36+
RING_FINGER_TIP,
37+
38+
PINKY_MCP,
39+
PINKY_PIP,
40+
PINKY_DIP,
41+
PINKY_TIP
42+
}

‎fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/gesturerecog/HandTrackingService.kt

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
/*
2+
* FXGL - JavaFX Game Library. The MIT License (MIT).
3+
* Copyright (c) AlmasB (almaslvl@gmail.com).
4+
* See LICENSE for details.
5+
*/
6+
7+
package com.almasb.fxgl.gesturerecog
8+
9+
import com.almasb.fxgl.core.EngineService
10+
import com.almasb.fxgl.core.concurrent.Async
11+
import com.almasb.fxgl.intelligence.WebAPI
12+
import com.almasb.fxgl.logging.Logger
13+
import com.almasb.fxgl.speechrecog.SpeechRecognitionService
14+
import com.almasb.fxgl.ws.LocalWebSocketServer
15+
import javafx.geometry.Point3D
16+
import org.openqa.selenium.WebDriver
17+
import org.openqa.selenium.chrome.ChromeDriver
18+
import org.openqa.selenium.chrome.ChromeOptions
19+
import java.util.function.Consumer
20+
21+
/**
22+
* TODO: remove duplicate code
23+
*
24+
* @author Almas Baim (https://github.com/AlmasB)
25+
*/
26+
class HandTrackingService : EngineService() {
27+
28+
private val log = Logger.get(SpeechRecognitionService::class.java)
29+
private val server = LocalWebSocketServer("HandTrackingServer", WebAPI.GESTURE_RECOGNITION_PORT)
30+
31+
private var webDriver: WebDriver? = null
32+
33+
private val handDataHandlers = arrayListOf<Consumer<Hand>>()
34+
35+
override fun onInit() {
36+
server.addMessageHandler { message ->
37+
try {
38+
val rawData = message.split(",").filter { it.isNotEmpty() }
39+
40+
val id = rawData[0].toInt()
41+
val points = ArrayList<Point3D>()
42+
43+
var i = 1
44+
while (i < rawData.size) {
45+
val x = rawData[i + 0].toDouble()
46+
val y = rawData[i + 1].toDouble()
47+
val z = rawData[i + 2].toDouble()
48+
49+
points.add(Point3D(x, y, z))
50+
51+
i += 3
52+
}
53+
54+
Async.startAsyncFX {
55+
handDataHandlers.forEach { it.accept(Hand(id, points)) }
56+
}
57+
58+
} catch (e: Exception) {
59+
log.warning("Failed to parse message.", e)
60+
}
61+
}
62+
63+
server.start()
64+
}
65+
66+
/**
67+
* Starts this service in a background thread.
68+
* Can be called after stop() to restart the service.
69+
* If the service has already started, then calls stop() and restarts it.
70+
*/
71+
fun start() {
72+
Async.startAsync {
73+
try {
74+
if (webDriver != null) {
75+
stop()
76+
}
77+
78+
val options = ChromeOptions()
79+
options.addArguments("--headless=new")
80+
options.addArguments("--use-fake-ui-for-media-stream")
81+
82+
webDriver = ChromeDriver(options)
83+
webDriver!!.get(WebAPI.GESTURE_RECOGNITION_API)
84+
85+
// we are ready to use the web api service
86+
} catch (e: Exception) {
87+
log.warning("Failed to start Chrome web driver. Ensure Chrome is installed in default location")
88+
log.warning("Error data", e)
89+
}
90+
}
91+
}
92+
93+
/**
94+
* Stops this service.
95+
* No-op if it has not started via start() before.
96+
*/
97+
fun stop() {
98+
try {
99+
if (webDriver != null) {
100+
webDriver!!.quit()
101+
webDriver = null
102+
}
103+
} catch (e: Exception) {
104+
log.warning("Failed to quit web driver", e)
105+
}
106+
}
107+
108+
fun addInputHandler(handler: Consumer<Hand>) {
109+
handDataHandlers += handler
110+
}
111+
112+
fun removeInputHandler(handler: Consumer<Hand>) {
113+
handDataHandlers -= handler
114+
}
115+
116+
override fun onExit() {
117+
stop()
118+
server.stop()
119+
}
120+
}

0 commit comments

Comments
 (0)