Skip to content

Commit 4d65061

Browse files
mhaemingmhaeming
committed
feat: hand detection mvp
1 parent cd1d5b6 commit 4d65061

9 files changed

Lines changed: 471 additions & 50 deletions

File tree

‎app/build.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,8 @@ dependencies {
8080
implementation("org.tensorflow:tensorflow-lite-task-vision:0.4.0")
8181
implementation("org.tensorflow:tensorflow-lite-gpu-delegate-plugin:0.4.0")
8282
implementation("org.tensorflow:tensorflow-lite-gpu:2.9.0")
83+
implementation("org.tensorflow:tensorflow-lite-support:0.4.4")
84+
implementation("org.tensorflow:tensorflow-lite-metadata:0.4.4")
8385

8486
// Camera UI
8587
implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.7.0")

‎app/src/main/assets/hand-fp16.tflite‎

13.5 MB
Binary file not shown.

‎app/src/main/assets/hand_labels.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
myleft
2+
myright
3+
yourleft
4+
yourright

‎app/src/main/java/com/nlinterface/fragments/CameraFragment.kt‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import android.content.res.Configuration
44
import android.graphics.Bitmap
55
import android.os.Bundle
66
import android.util.Log
7+
import android.util.Size
78
import android.view.LayoutInflater
89
import android.view.View
910
import android.view.ViewGroup
@@ -22,10 +23,14 @@ import androidx.fragment.app.Fragment
2223
import com.nlinterface.databinding.FragmentCameraBinding
2324
import com.nlinterface.interfaces.DetectorListener
2425
import com.nlinterface.utility.ObjectDetectorHelper
26+
import com.nlinterface.utility.Recognition
27+
import com.nlinterface.utility.Yolov5TFLiteDetector
2528
import org.tensorflow.lite.task.vision.detector.Detection
2629
import java.util.LinkedList
2730
import java.util.concurrent.ExecutorService
2831
import java.util.concurrent.Executors
32+
import kotlin.math.log
33+
2934
class CameraFragment : Fragment(), DetectorListener{
3035

3136
private val TAG = "ObjectDetection"
@@ -171,15 +176,15 @@ class CameraFragment : Fragment(), DetectorListener{
171176
// Update UI after objects have been detected. Extracts original image height/width
172177
// to scale and place bounding boxes properly through OverlayView
173178
override fun onResults(
174-
results: MutableList<Detection>?,
179+
results: ArrayList<Recognition>?,
175180
inferenceTime: Long,
176181
imageHeight: Int,
177182
imageWidth: Int
178183
) {
179184
activity?.runOnUiThread {
180185
// Pass necessary information to OverlayView for drawing on the canvas
181186
fragmentCameraBinding.overlay.setResults(
182-
results ?: LinkedList<Detection>(),
187+
results ?: ArrayList<Recognition>(),
183188
imageHeight,
184189
imageWidth
185190
)

‎app/src/main/java/com/nlinterface/interfaces/DetectorListener.kt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
package com.nlinterface.interfaces
22

3+
import com.nlinterface.utility.Recognition
34
import org.tensorflow.lite.task.vision.detector.Detection
45

56
interface DetectorListener {
67
fun onError(error: String)
7-
fun onResults(results: MutableList<Detection>?, inferenceTime: Long, imageHeight: Int, imageWidth: Int)
8+
fun onResults(results: ArrayList<Recognition>?, inferenceTime: Long, imageHeight: Int, imageWidth: Int)
89
}

‎app/src/main/java/com/nlinterface/utility/ObjectDetectorHelper.kt‎

Lines changed: 9 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,10 @@ import org.tensorflow.lite.task.vision.detector.ObjectDetector
1919
* */
2020

2121
class ObjectDetectorHelper(
22-
var threshold: Float = 0.5f,
23-
var numThreads: Int = 2,
24-
var maxResults: Int = 3,
25-
var currentDelegate: Int = 0,
26-
var currentModel: Int = 0,
2722
val context: Context,
2823
val objectDetectorListener: DetectorListener?
2924
) {
30-
private var objectDetector: ObjectDetector? = null
25+
private var objectDetector: Yolov5TFLiteDetector? = null
3126

3227
init {
3328
setupObjectDetector()
@@ -38,37 +33,14 @@ class ObjectDetectorHelper(
3833
}
3934

4035
fun setupObjectDetector() {
41-
val optionsBuilder = ObjectDetector.ObjectDetectorOptions.builder().setScoreThreshold(threshold).setMaxResults(maxResults)
42-
43-
val baseOptionsBuilder = BaseOptions.builder().setNumThreads(numThreads)
44-
45-
when (currentDelegate) {
46-
DELEGATE_CPU -> {}
47-
DELEGATE_GPU -> {
48-
if (CompatibilityList().isDelegateSupportedOnThisDevice) {
49-
baseOptionsBuilder.useGpu()
50-
} else {
51-
objectDetectorListener?.onError("GPU is not supported on this device")
52-
}
53-
}
54-
DELEGATE_NNAPI -> {
55-
baseOptionsBuilder.useNnapi()
56-
}
57-
}
58-
59-
optionsBuilder.setBaseOptions(baseOptionsBuilder.build())
60-
61-
// Model selection
62-
63-
val modelName = when (currentModel) {
64-
MODEL_MOBILENETV1 -> "mobilenetv1.tflite"
65-
else -> "mobilenetv1.tflite"
66-
}
67-
6836
try {
69-
objectDetector = ObjectDetector.createFromFileAndOptions(context, modelName, optionsBuilder.build())
37+
val yolov5TFLiteDetector = Yolov5TFLiteDetector()
38+
yolov5TFLiteDetector.modelFile = "hand-fp16.tflite"
39+
yolov5TFLiteDetector.addGPUDelegate()
40+
yolov5TFLiteDetector.initialModel(context)
41+
objectDetector = yolov5TFLiteDetector
42+
7043
} catch (e: IllegalStateException) {
71-
objectDetectorListener?.onError("Object detector failed to initialize. See error logs for details")
7244
Log.e("NLI-Classification", "TFLite failed to load model with error: " + e.message)
7345
}
7446
}
@@ -80,12 +52,11 @@ class ObjectDetectorHelper(
8052

8153
var inferenceTime = SystemClock.uptimeMillis()
8254

83-
// Image Preprocessing
8455
val imageProcessor = ImageProcessor.Builder().add(Rot90Op(-imageRotation / 90)).build()
85-
8656
val tensorImage = imageProcessor.process(TensorImage.fromBitmap(image))
8757

88-
val results = objectDetector?.detect(tensorImage)
58+
val results = objectDetector?.detect(tensorImage.bitmap)
59+
8960
inferenceTime = SystemClock.uptimeMillis() - inferenceTime
9061
objectDetectorListener?.onResults(results, inferenceTime, tensorImage.height, tensorImage.width)
9162
}

‎app/src/main/java/com/nlinterface/utility/OverlayView.kt‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ import kotlin.math.max
1919
**/
2020

2121
class OverlayView(context: Context?, attrs: AttributeSet) : View(context, attrs) {
22-
private var results: List<Detection> = LinkedList<Detection>()
22+
private var results: ArrayList<Recognition> = ArrayList<Recognition>()
2323
private var boxPaint = Paint()
2424
private var textBackgroundPaint = Paint()
2525
private var textPaint = Paint()
@@ -58,7 +58,7 @@ class OverlayView(context: Context?, attrs: AttributeSet) : View(context, attrs)
5858
super.draw(canvas)
5959

6060
for (result in results) {
61-
val boundingBox = result.boundingBox
61+
val boundingBox = result.location
6262

6363
val top = boundingBox.top * scaleFactor
6464
val bottom = boundingBox.bottom * scaleFactor
@@ -68,22 +68,17 @@ class OverlayView(context: Context?, attrs: AttributeSet) : View(context, attrs)
6868
val drawableRect = RectF(left, top, right, bottom)
6969
canvas.drawRect(drawableRect, boxPaint)
7070

71-
val drawableText = result.categories[0].label + " " + String.format("%.2f", result.categories[0].score)
71+
val drawableText = result.labelName + " " + String.format("%.2f", result.labelScore)
7272

7373
textBackgroundPaint.getTextBounds(drawableText, 0, drawableText.length, bounds)
74-
val textWidth = bounds.width()
75-
val textHeight = bounds.height()
76-
77-
canvas.drawRect(left, top, left + textWidth + Companion.BOUNDING_RECT_TEXT_PADDING, top + textHeight + Companion.BOUNDING_RECT_TEXT_PADDING, textBackgroundPaint)
78-
7974
canvas.drawText(drawableText, left, top + bounds.height(), textPaint)
8075

8176

8277

8378
}
8479
}
8580
fun setResults(
86-
detectionResults: MutableList<Detection>,
81+
detectionResults: ArrayList<Recognition>,
8782
imageHeight: Int,
8883
imageWidth: Int
8984
) {

‎app/src/main/java/com/nlinterface/utility/Recognition.java‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
package com.nlinterface.utility;
2+
3+
import android.graphics.RectF;
4+
5+
public class Recognition {
6+
7+
/**
8+
* Display name for the recognition.
9+
*/
10+
private Integer labelId;
11+
private String labelName;
12+
private Float labelScore;
13+
14+
/**
15+
* A sortable score for how good the recognition is relative to others. Higher should be better.
16+
*/
17+
private Float confidence;
18+
19+
20+
/**
21+
* Optional location within the source image for the location of the recognized object.
22+
*/
23+
private RectF location;
24+
25+
public Recognition(
26+
final int labelId, final String labelName, final Float labelScore, final Float confidence, final RectF location) {
27+
this.labelId = labelId;
28+
this.labelScore = labelScore;
29+
this.labelName = labelName;
30+
this.confidence = confidence;
31+
this.location = location;
32+
}
33+
34+
public Integer getLabelId() {
35+
return labelId;
36+
}
37+
38+
public String getLabelName() {
39+
return labelName;
40+
}
41+
42+
public Float getLabelScore() {
43+
return labelScore;
44+
}
45+
46+
public Float getConfidence() {
47+
return confidence;
48+
}
49+
50+
public RectF getLocation() {
51+
return new RectF(location);
52+
}
53+
54+
public void setLocation(RectF location) {
55+
this.location = location;
56+
}
57+
58+
public void setLabelName(String labelName) {
59+
this.labelName = labelName;
60+
}
61+
62+
public void setLabelId(int labelId) {
63+
this.labelId = labelId;
64+
}
65+
66+
public void setLabelScore(Float labelScore) {
67+
this.labelScore = labelScore;
68+
}
69+
70+
public void setConfidence(Float confidence) {
71+
this.confidence = confidence;
72+
}
73+
74+
@Override
75+
public String toString() {
76+
String resultString = "";
77+
78+
resultString += labelId + " ";
79+
80+
if (labelName != null) {
81+
resultString += labelName + " ";
82+
}
83+
84+
if (confidence != null) {
85+
resultString += String.format("(%.1f%%) ", confidence * 100.0f);
86+
}
87+
88+
if (location != null) {
89+
resultString += location + " ";
90+
}
91+
92+
return resultString.trim();
93+
}
94+
}

0 commit comments

Comments
 (0)