1+ package com.nlinterface.fragments
2+
3+ import android.content.res.Configuration
4+ import android.graphics.Bitmap
5+ import android.os.Bundle
6+ import android.util.Log
7+ import android.view.LayoutInflater
8+ import android.view.View
9+ import android.view.ViewGroup
10+ import android.widget.AdapterView
11+ import android.widget.Toast
12+ import androidx.camera.core.AspectRatio
13+ import androidx.camera.core.Camera
14+ import androidx.camera.core.CameraSelector
15+ import androidx.camera.core.ImageAnalysis
16+ import androidx.camera.core.ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888
17+ import androidx.camera.core.ImageProxy
18+ import androidx.camera.core.Preview
19+ import androidx.camera.lifecycle.ProcessCameraProvider
20+ import androidx.core.content.ContextCompat
21+ import androidx.fragment.app.Fragment
22+ import com.nlinterface.databinding.FragmentCameraBinding
23+ import com.nlinterface.interfaces.DetectorListener
24+ import com.nlinterface.utility.ObjectDetectorHelper
25+ import org.tensorflow.lite.task.vision.detector.Detection
26+ import java.util.LinkedList
27+ import java.util.concurrent.ExecutorService
28+ import java.util.concurrent.Executors
29+ class CameraFragment : Fragment (), DetectorListener{
30+
31+ private val TAG = " ObjectDetection"
32+
33+ private var _fragmentCameraBinding : FragmentCameraBinding ? = null
34+
35+ private val fragmentCameraBinding
36+ get() = _fragmentCameraBinding !!
37+
38+ private lateinit var objectDetectorHelper: ObjectDetectorHelper
39+ private lateinit var bitmapBuffer: Bitmap
40+ private var preview: Preview ? = null
41+ private var imageAnalyzer: ImageAnalysis ? = null
42+ private var camera: Camera ? = null
43+ private var cameraProvider: ProcessCameraProvider ? = null
44+
45+ /* * Blocking camera operations are performed using this executor */
46+ private lateinit var cameraExecutor: ExecutorService
47+
48+ override fun onDestroyView () {
49+ _fragmentCameraBinding = null
50+ super .onDestroyView()
51+
52+ // Shut down our background executor
53+ cameraExecutor.shutdown()
54+ }
55+
56+ override fun onCreateView (
57+ inflater : LayoutInflater ,
58+ container : ViewGroup ? ,
59+ savedInstanceState : Bundle ?
60+ ): View {
61+ _fragmentCameraBinding = FragmentCameraBinding .inflate(inflater, container, false )
62+
63+ return fragmentCameraBinding.root
64+ }
65+
66+ override fun onViewCreated (view : View , savedInstanceState : Bundle ? ) {
67+ super .onViewCreated(view, savedInstanceState)
68+
69+ objectDetectorHelper = ObjectDetectorHelper (
70+ context = requireContext(),
71+ objectDetectorListener = this )
72+
73+ // Initialize our background executor
74+ cameraExecutor = Executors .newSingleThreadExecutor()
75+
76+ // Wait for the views to be properly laid out
77+ fragmentCameraBinding.viewFinder.post {
78+ // Set up the camera and its use cases
79+ setUpCamera()
80+ }
81+
82+ }
83+
84+ // Initialize CameraX, and prepare to bind the camera use cases
85+ private fun setUpCamera () {
86+ val cameraProviderFuture = ProcessCameraProvider .getInstance(requireContext())
87+ cameraProviderFuture.addListener(
88+ {
89+ // CameraProvider
90+ cameraProvider = cameraProviderFuture.get()
91+
92+ // Build and bind the camera use cases
93+ bindCameraUseCases()
94+ },
95+ ContextCompat .getMainExecutor(requireContext())
96+ )
97+ }
98+
99+ // Declare and bind preview, capture and analysis use cases
100+ private fun bindCameraUseCases () {
101+
102+ // CameraProvider
103+ val cameraProvider =
104+ cameraProvider ? : throw IllegalStateException (" Camera initialization failed."
105+
106+ // CameraSelector - makes assumption that we're only using the back camera
107+ val cameraSelector =
108+ CameraSelector .Builder ().requireLensFacing(CameraSelector .LENS_FACING_BACK ).build()
109+
110+ // Preview. Only using the 4:3 ratio because this is the closest to our models
111+ preview =
112+ Preview .Builder ()
113+ .setTargetAspectRatio(AspectRatio .RATIO_4_3 )
114+ .setTargetRotation(fragmentCameraBinding.viewFinder.display.rotation)
115+ .build()
116+
117+ // ImageAnalysis. Using RGBA 8888 to match how our models work
118+ imageAnalyzer =
119+ ImageAnalysis .Builder ()
120+ .setTargetAspectRatio(AspectRatio .RATIO_4_3 )
121+ .setTargetRotation(fragmentCameraBinding.viewFinder.display.rotation)
122+ .setBackpressureStrategy(ImageAnalysis .STRATEGY_KEEP_ONLY_LATEST )
123+ .setOutputImageFormat(OUTPUT_IMAGE_FORMAT_RGBA_8888 )
124+ .build()
125+ // The analyzer can then be assigned to the instance
126+ .also {
127+ it.setAnalyzer(cameraExecutor) { image ->
128+ if (! ::bitmapBuffer.isInitialized) {
129+ // The image rotation and RGB image buffer are initialized only once
130+ // the analyzer has started running
131+ bitmapBuffer = Bitmap .createBitmap(
132+ image.width,
133+ image.height,
134+ Bitmap .Config .ARGB_8888
135+ )
136+ }
137+
138+ detectObjects(image)
139+ }
140+ }
141+
142+ // Must unbind the use-cases before rebinding them
143+ cameraProvider.unbindAll()
144+
145+ try {
146+ // A variable number of use-cases can be passed here -
147+ // camera provides access to CameraControl & CameraInfo
148+ camera = cameraProvider.bindToLifecycle(this , cameraSelector, preview, imageAnalyzer)
149+
150+ // Attach the viewfinder's surface provider to preview use case
151+ preview?.setSurfaceProvider(fragmentCameraBinding.viewFinder.surfaceProvider)
152+ } catch (exc: Exception ) {
153+ Log .e(TAG , " Use case binding failed"
154+ }
155+ }
156+
157+ private fun detectObjects (image : ImageProxy ) {
158+ // Copy out RGB bits to the shared bitmap buffer
159+ image.use { bitmapBuffer.copyPixelsFromBuffer(image.planes[0 ].buffer) }
160+
161+ val imageRotation = image.imageInfo.rotationDegrees
162+ // Pass Bitmap and rotation to the object detector helper for processing and detection
163+ objectDetectorHelper.detect(bitmapBuffer, imageRotation)
164+ }
165+
166+ override fun onConfigurationChanged (newConfig : Configuration ) {
167+ super .onConfigurationChanged(newConfig)
168+ imageAnalyzer?.targetRotation = fragmentCameraBinding.viewFinder.display.rotation
169+ }
170+
171+ // Update UI after objects have been detected. Extracts original image height/width
172+ // to scale and place bounding boxes properly through OverlayView
173+ override fun onResults (
174+ results : MutableList <Detection >? ,
175+ inferenceTime : Long ,
176+ imageHeight : Int ,
177+ imageWidth : Int
178+ ) {
179+ activity?.runOnUiThread {
180+ // Pass necessary information to OverlayView for drawing on the canvas
181+ fragmentCameraBinding.overlay.setResults(
182+ results ? : LinkedList <Detection >(),
183+ imageHeight,
184+ imageWidth
185+ )
186+
187+ // Force a redraw
188+ fragmentCameraBinding.overlay.invalidate()
189+ }
190+ }
191+
192+ override fun onError (error : String ) {
193+ activity?.runOnUiThread {
194+ Toast .makeText(requireContext(), error, Toast .LENGTH_SHORT ).show()
195+ }
196+ }
197+ }
0 commit comments