Add cooking assistance with Gemini Live and grocery list feature

app/build.gradle.kts

@@ -1,6 +1,5 @@
 plugins {
     alias(libs.plugins.android.application)
-    alias(libs.plugins.kotlin.android)
     alias(libs.plugins.kotlin.compose)
     alias(libs.plugins.google.services)
     alias(libs.plugins.google.ksp)
@@ -11,12 +10,12 @@ plugins {
 
 android {
     namespace = "com.google.firebase.example.friendlymeals"
-    compileSdk = 36
+    compileSdk = 37
 
     defaultConfig {
         applicationId = "com.google.firebase.example.friendlymeals"
         minSdk = 26
-        targetSdk = 36
+        targetSdk = 37
         versionCode = 1
         versionName = "1.0"
 
@@ -36,9 +35,6 @@ android {
         sourceCompatibility = JavaVersion.VERSION_11
         targetCompatibility = JavaVersion.VERSION_11
     }
-    kotlinOptions {
-        jvmTarget = "11"
-    }
     buildFeatures {
         compose = true
     }
@@ -58,6 +54,10 @@ dependencies {
     implementation(libs.androidx.core.splashscreen)
     implementation(libs.androidx.constraintlayout.compose)
     implementation(libs.androidx.exifinterface)
+    implementation(libs.androidx.camera.core)
+    implementation(libs.androidx.camera.camera2)
+    implementation(libs.androidx.camera.lifecycle)
+    implementation(libs.androidx.camera.view)
     implementation(libs.kotlinx.serialization.json)
     implementation(libs.hilt.android)
     ksp(libs.hilt.android.compiler)

app/src/main/AndroidManifest.xml

@@ -2,6 +2,7 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
     xmlns:tools="http://schemas.android.com/tools">
     <uses-permission android:name="android.permission.CAMERA" />
+    <uses-permission android:name="android.permission.RECORD_AUDIO" />
     <uses-feature android:name="android.hardware.camera" android:required="false" />
     <application
         android:allowBackup="true"

app/src/main/java/com/google/firebase/example/friendlymeals/MainActivity.kt

@@ -23,6 +23,10 @@ import androidx.navigation.navigation
 import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel
 import com.google.firebase.example.friendlymeals.ui.generate.GenerateRoute
 import com.google.firebase.example.friendlymeals.ui.generate.GenerateScreen
+import com.google.firebase.example.friendlymeals.ui.groceryList.GroceryListRoute
+import com.google.firebase.example.friendlymeals.ui.groceryList.GroceryListScreen
+import com.google.firebase.example.friendlymeals.ui.live.LiveAssistantRoute
+import com.google.firebase.example.friendlymeals.ui.live.LiveAssistantScreen
 import com.google.firebase.example.friendlymeals.ui.recipe.RecipeRoute
 import com.google.firebase.example.friendlymeals.ui.recipe.RecipeScreen
 import com.google.firebase.example.friendlymeals.ui.recipeList.RecipeListGraph
@@ -121,9 +125,26 @@ class MainActivity : ComponentActivity() {
                             }
                             composable<RecipeRoute> {
                                 RecipeScreen(
-                                    navigateBack = { navController.popBackStack() }
+                                    navigateBack = { navController.popBackStack() },
+                                    navigateToLiveAssistant = { recipeId ->
+                                        navController.navigate(LiveAssistantRoute(recipeId)) {
+                                            launchSingleTop = true
+                                        }
+                                    }
                                 )
                             }
+                            composable<LiveAssistantRoute> {
+                                LiveAssistantScreen(
+                                    navigateBack = { navController.popBackStack() },
+                                    showError = {
+                                        val message = this@MainActivity.getString(R.string.camera_error_message)
+                                        scope.launch { snackbarHostState.showSnackbar(message) }
+                                    }
+                                )
+                            }
+                            composable<GroceryListRoute> {
+                                GroceryListScreen()
+                            }
                         }
                     }
                 }

app/src/main/java/com/google/firebase/example/friendlymeals/MainViewModel.kt

@@ -5,6 +5,7 @@ import androidx.lifecycle.ViewModel
 import androidx.lifecycle.viewModelScope
 import kotlinx.coroutines.CoroutineExceptionHandler
 import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.launch
 
 open class MainViewModel : ViewModel() {
@@ -15,4 +16,12 @@ open class MainViewModel : ViewModel() {
             },
             block = block
         )
+
+    fun launchCatchingIO(block: suspend CoroutineScope.() -> Unit) =
+        viewModelScope.launch(
+            Dispatchers.IO + CoroutineExceptionHandler { _, throwable ->
+                Log.e("MainViewModel", throwable.message ?: "Unknown error")
+            },
+            block = block
+        )
 }

app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/AIRemoteDataSource.kt

@@ -3,15 +3,18 @@ package com.google.firebase.example.friendlymeals.data.datasource
 import android.graphics.Bitmap
 import android.util.Log
 import com.google.firebase.Firebase
+import com.google.firebase.ai.DownloadStatus.DownloadCompleted
+import com.google.firebase.ai.DownloadStatus.DownloadFailed
+import com.google.firebase.ai.DownloadStatus.DownloadInProgress
+import com.google.firebase.ai.DownloadStatus.DownloadStarted
 import com.google.firebase.ai.FirebaseAI
 import com.google.firebase.ai.InferenceMode
 import com.google.firebase.ai.InferenceSource
 import com.google.firebase.ai.OnDeviceConfig
-import com.google.firebase.ai.TemplateGenerativeModel
-import com.google.firebase.ai.TemplateImagenModel
-import com.google.firebase.ai.ondevice.DownloadStatus
-import com.google.firebase.ai.ondevice.FirebaseAIOnDevice
-import com.google.firebase.ai.ondevice.OnDeviceModelStatus
+import com.google.firebase.ai.OnDeviceModelStatus.Companion.AVAILABLE
+import com.google.firebase.ai.OnDeviceModelStatus.Companion.DOWNLOADABLE
+import com.google.firebase.ai.OnDeviceModelStatus.Companion.DOWNLOADING
+import com.google.firebase.ai.OnDeviceModelStatus.Companion.UNAVAILABLE
 import com.google.firebase.ai.type.ImagePart
 import com.google.firebase.ai.type.PublicPreviewAPI
 import com.google.firebase.ai.type.content
@@ -25,9 +28,7 @@ import javax.inject.Inject
 
 @OptIn(PublicPreviewAPI::class)
 class AIRemoteDataSource @Inject constructor(
-    private val aiModel: FirebaseAI,
-    private val generativeModel: TemplateGenerativeModel,
-    private val imagenModel: TemplateImagenModel,
+    aiModel: FirebaseAI,
     private val remoteConfig: FirebaseRemoteConfig
 ) {
     private val json = Json { ignoreUnknownKeys = true }
@@ -36,7 +37,8 @@ class AIRemoteDataSource @Inject constructor(
         onDeviceConfig = OnDeviceConfig(mode = InferenceMode.PREFER_IN_CLOUD)
     )
 
-    @OptIn(PublicPreviewAPI::class)
+    private val templateGenerativeModel = aiModel.templateGenerativeModel()
+
     suspend fun generateIngredients(image: Bitmap): String {
         // Adding a Performance Monitoring trace is completely optional. Traces can help you
         // measure how long it takes to generate ingredients on device and in cloud.
@@ -63,7 +65,7 @@ class AIRemoteDataSource @Inject constructor(
     }
 
     suspend fun generateRecipe(ingredients: String, notes: String): RecipeSchema? {
-        val response = generativeModel.generateContent(
+        val response = templateGenerativeModel.generateContent(
             templateId = remoteConfig.getString(GENERATE_RECIPE_KEY),
             inputs = buildMap {
                 put(INGREDIENTS_FIELD, ingredients)
@@ -79,7 +81,7 @@ class AIRemoteDataSource @Inject constructor(
     }
 
     suspend fun generateRecipePhoto(recipeTitle: String): Bitmap? {
-        val response = generativeModel.generateContent(
+        val response = templateGenerativeModel.generateContent(
             templateId = remoteConfig.getString(GENERATE_RECIPE_PHOTO_GEMINI_KEY),
             inputs = mapOf(RECIPE_TITLE_FIELD to recipeTitle)
         )
@@ -88,17 +90,8 @@ class AIRemoteDataSource @Inject constructor(
             ?.filterIsInstance<ImagePart>()?.firstOrNull()?.image
     }
 
-    suspend fun generateRecipePhotoImagen(recipeTitle: String): Bitmap? {
-        val response = imagenModel.generateImages(
-            templateId = remoteConfig.getString(GENERATE_RECIPE_PHOTO_IMAGEN_KEY),
-            inputs = mapOf(RECIPE_TITLE_FIELD to recipeTitle)
-        )
-
-        return response.images.firstOrNull()?.asBitmap()
-    }
-
     suspend fun scanMeal(imageData: String): MealSchema? {
-        val response = generativeModel.generateContent(
+        val response = templateGenerativeModel.generateContent(
             templateId = remoteConfig.getString(SCAN_MEAL_KEY),
             inputs = mapOf(
                 MIME_TYPE_FIELD to MIME_TYPE_VALUE,
@@ -112,31 +105,31 @@ class AIRemoteDataSource @Inject constructor(
     }
 
     suspend fun loadOnDeviceModel() {
-        when (FirebaseAIOnDevice.checkStatus()) {
-            OnDeviceModelStatus.UNAVAILABLE -> {
+        when (hybridGenerativeModel.onDeviceExtension?.checkStatus()) {
+            UNAVAILABLE -> {
                 Log.i(TAG, "On-device model is unavailable")
             }
-            OnDeviceModelStatus.DOWNLOADABLE -> {
-                FirebaseAIOnDevice.download().collect { status ->
+            DOWNLOADABLE -> {
+                hybridGenerativeModel.onDeviceExtension?.download()?.collect { status ->
                     when (status) {
-                        is DownloadStatus.DownloadStarted ->
+                        is DownloadStarted ->
                             Log.i(TAG, "Starting download - ${status.bytesToDownload}")
 
-                        is DownloadStatus.DownloadInProgress ->
+                        is DownloadInProgress ->
                             Log.i(TAG, "Download in progress ${status.totalBytesDownloaded} bytes downloaded")
 
-                        is DownloadStatus.DownloadCompleted ->
+                        is DownloadCompleted ->
                             Log.i(TAG, "On-device model download complete")
 
-                        is DownloadStatus.DownloadFailed ->
+                        is DownloadFailed ->
                             Log.e(TAG, "Download failed $status")
                     }
                 }
             }
-            OnDeviceModelStatus.DOWNLOADING -> {
+            DOWNLOADING -> {
                 Log.i(TAG, "On-device model is being downloaded")
             }
-            OnDeviceModelStatus.AVAILABLE -> {
+            AVAILABLE -> {
                 Log.i(TAG, "On-device model is available")
             }
         }
@@ -146,7 +139,6 @@ class AIRemoteDataSource @Inject constructor(
         //Remote Config Keys
         private const val GENERATE_RECIPE_KEY = "generate_recipe"
         private const val GENERATE_RECIPE_PHOTO_GEMINI_KEY = "generate_recipe_photo_gemini"
-        private const val GENERATE_RECIPE_PHOTO_IMAGEN_KEY = "generate_recipe_photo_imagen"
         private const val SCAN_MEAL_KEY = "scan_meal"
         private const val HYBRID_CLOUD_MODEL_KEY = "hybrid_cloud_model"
         private const val HYBRID_INGREDIENTS_PROMPT_KEY = "hybrid_ingredients_prompt"

app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/DatabaseRemoteDataSource.kt

@@ -1,6 +1,7 @@
 package com.google.firebase.example.friendlymeals.data.datasource
 
 import android.util.Log
+import com.google.firebase.example.friendlymeals.data.model.GroceryItem
 import com.google.firebase.example.friendlymeals.data.model.Recipe
 import com.google.firebase.example.friendlymeals.data.model.Review
 import com.google.firebase.example.friendlymeals.data.model.Like
@@ -19,6 +20,10 @@ import com.google.firebase.firestore.pipeline.Expression.Companion.documentId
 import com.google.firebase.firestore.pipeline.Expression.Companion.field
 import com.google.firebase.firestore.pipeline.Expression.Companion.variable
 import com.google.firebase.firestore.pipeline.SearchStage
+import com.google.firebase.firestore.snapshots
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.flowOf
+import kotlinx.coroutines.flow.mapNotNull
 import kotlinx.coroutines.tasks.await
 import javax.inject.Inject
 import kotlin.collections.first
@@ -251,12 +256,63 @@ class DatabaseRemoteDataSource @Inject constructor(
         }
     }
 
+    fun getGroceriesFlow(userId: String): Flow<List<GroceryItem>> {
+        if (userId.isEmpty()) {
+            return flowOf(emptyList())
+        }
+
+        return firestore.collection(GROCERIES_COLLECTION)
+            .whereEqualTo(USER_ID_FIELD, userId)
+            .snapshots()
+            .mapNotNull { snapshot ->
+                snapshot.documents.mapNotNull { doc ->
+                    doc.toObject(GroceryItem::class.java)?.copy(id = doc.id)
+                }
+            }
+    }
+
+    suspend fun addGroceryItem(item: GroceryItem) {
+        val docRef = firestore.collection(GROCERIES_COLLECTION).document()
+        val itemWithId = item.copy(id = docRef.id)
+        docRef.set(itemWithId).await()
+    }
+
+    suspend fun updateGroceryItemChecked(itemId: String, checked: Boolean) {
+        firestore.collection(GROCERIES_COLLECTION).document(itemId)
+            .update(CHECKED_FIELD, checked).await()
+    }
+
+    suspend fun deleteGroceryItem(itemId: String) {
+        firestore.collection(GROCERIES_COLLECTION).document(itemId)
+            .delete().await()
+    }
+
+    suspend fun addIngredientsToGroceries(userId: String, ingredients: List<String>) {
+        if (userId.isEmpty() || ingredients.isEmpty()) return
+
+        val batch = firestore.batch()
+        val collection = firestore.collection(GROCERIES_COLLECTION)
+
+        for (ingredient in ingredients) {
+            val docRef = collection.document()
+            val item = GroceryItem(
+                id = docRef.id,
+                userId = userId,
+                name = ingredient,
+                checked = false
+            )
+            batch.set(docRef, item)
+        }
+        batch.commit().await()
+    }
+
     companion object {
         //Collections
         private const val USERS_COLLECTION = "users"
         private const val RECIPES_COLLECTION = "recipes"
         private const val LIKES_COLLECTION = "likes"
         private const val REVIEWS_SUBCOLLECTION = "reviews"
+        private const val GROCERIES_COLLECTION = "groceries"
 
         //Fields
         private const val RATING_FIELD = "rating"
@@ -272,6 +328,8 @@ class DatabaseRemoteDataSource @Inject constructor(
         private const val INSTRUCTIONS_FIELD = "instructions"
         private const val INGREDIENTS_FIELD = "ingredients"
         private const val RECIPE_ID_FIELD = "recipeId"
+        private const val USER_ID_FIELD = "userId"
+        private const val CHECKED_FIELD = "checked"
 
         //Field aliases
         private const val AVG_RATING_ALIAS = "avg_rating"