From ee19f156cde12bfc186e03692b82cc969abdc4ad Mon Sep 17 00:00:00 2001 From: Marina Coelho Date: Wed, 13 May 2026 15:30:29 +0100 Subject: [PATCH 1/8] Update libraries and AGP --- app/build.gradle.kts | 6 +---- build.gradle.kts | 1 - gradle.properties | 11 +++++++- gradle/gradle-daemon-jvm.properties | 13 ++++++++++ gradle/libs.versions.toml | 32 ++++++++++++------------ gradle/wrapper/gradle-wrapper.properties | 2 +- settings.gradle.kts | 3 +++ 7 files changed, 44 insertions(+), 24 deletions(-) create mode 100644 gradle/gradle-daemon-jvm.properties diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 2189d6f..ef8be09 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -1,6 +1,5 @@ plugins { alias(libs.plugins.android.application) - alias(libs.plugins.kotlin.android) alias(libs.plugins.kotlin.compose) alias(libs.plugins.google.services) alias(libs.plugins.google.ksp) @@ -11,7 +10,7 @@ plugins { android { namespace = "com.google.firebase.example.friendlymeals" - compileSdk = 36 + compileSdk = 37 defaultConfig { applicationId = "com.google.firebase.example.friendlymeals" @@ -36,9 +35,6 @@ android { sourceCompatibility = JavaVersion.VERSION_11 targetCompatibility = JavaVersion.VERSION_11 } - kotlinOptions { - jvmTarget = "11" - } buildFeatures { compose = true } diff --git a/build.gradle.kts b/build.gradle.kts index 074ea6b..89f8551 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,7 +1,6 @@ // Top-level build file where you can add configuration options common to all sub-projects/modules. plugins { alias(libs.plugins.android.application) apply false - alias(libs.plugins.kotlin.android) apply false alias(libs.plugins.kotlin.compose) apply false alias(libs.plugins.google.services) apply false alias(libs.plugins.google.hilt) apply false diff --git a/gradle.properties b/gradle.properties index 20e2a01..61e7198 100644 --- a/gradle.properties +++ b/gradle.properties @@ -20,4 +20,13 @@ kotlin.code.style=official # Enables namespacing of each library's R class so that its R class includes only the # resources declared in the library itself and none from the library's dependencies, # thereby reducing the size of the R class for that library -android.nonTransitiveRClass=true \ No newline at end of file +android.nonTransitiveRClass=true +android.defaults.buildfeatures.resvalues=true +android.sdk.defaultTargetSdkToCompileSdkIfUnset=false +android.enableAppCompileTimeRClass=false +android.usesSdkInManifest.disallowed=false +android.uniquePackageNames=false +android.dependency.useConstraints=true +android.r8.strictFullModeForKeepRules=false +android.r8.optimizedResourceShrinking=false +android.disallowKotlinSourceSets=false \ No newline at end of file diff --git a/gradle/gradle-daemon-jvm.properties b/gradle/gradle-daemon-jvm.properties new file mode 100644 index 0000000..5c34300 --- /dev/null +++ b/gradle/gradle-daemon-jvm.properties @@ -0,0 +1,13 @@ +#This file is generated by updateDaemonJvm +toolchainUrl.FREE_BSD.AARCH64=https\://api.foojay.io/disco/v3.0/ids/56a19bc915b9ba2eb62ba7554c61b919/redirect +toolchainUrl.FREE_BSD.X86_64=https\://api.foojay.io/disco/v3.0/ids/398ffe3949748bfb1d5636f023d228fd/redirect +toolchainUrl.LINUX.AARCH64=https\://api.foojay.io/disco/v3.0/ids/56a19bc915b9ba2eb62ba7554c61b919/redirect +toolchainUrl.LINUX.X86_64=https\://api.foojay.io/disco/v3.0/ids/398ffe3949748bfb1d5636f023d228fd/redirect +toolchainUrl.MAC_OS.AARCH64=https\://api.foojay.io/disco/v3.0/ids/e99bae143b75f9a10ead10248f02055e/redirect +toolchainUrl.MAC_OS.X86_64=https\://api.foojay.io/disco/v3.0/ids/04e088f8677de3b384108493cc9481d0/redirect +toolchainUrl.UNIX.AARCH64=https\://api.foojay.io/disco/v3.0/ids/56a19bc915b9ba2eb62ba7554c61b919/redirect +toolchainUrl.UNIX.X86_64=https\://api.foojay.io/disco/v3.0/ids/398ffe3949748bfb1d5636f023d228fd/redirect +toolchainUrl.WINDOWS.AARCH64=https\://api.foojay.io/disco/v3.0/ids/e55dccbfe27cb97945148c61a39c89c5/redirect +toolchainUrl.WINDOWS.X86_64=https\://api.foojay.io/disco/v3.0/ids/dbd05c4936d573642f94cd149e1356c8/redirect +toolchainVendor=JETBRAINS +toolchainVersion=21 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index be04753..0d5f938 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,28 +1,28 @@ [versions] -agp = "8.13.1" +agp = "9.2.1" coilCompose = "2.7.0" exifinterface = "1.4.2" -firebaseAi = "17.10.1" -firebaseAiOndevice = "16.0.0-beta01" -firebaseBom = "34.12.0" -kotlin = "2.2.21" -coreKtx = "1.17.0" +firebaseAi = "17.12.0" +firebaseAiOndevice = "16.0.0-beta02" +firebaseBom = "34.13.0" +kotlin = "2.3.21" +coreKtx = "1.18.0" junit = "4.13.2" -junitVersion = "1.2.1" -espressoCore = "3.6.1" +junitVersion = "1.3.0" +espressoCore = "3.7.0" lifecycleRuntimeKtx = "2.10.0" -activityCompose = "1.12.0" -composeBom = "2025.11.01" +activityCompose = "1.13.0" +composeBom = "2026.05.00" googleServices = "4.4.4" -googleHilt = "2.57.2" -googleKotlinKsp = "2.2.21-2.0.5" -hiltAndroidCompiler = "2.57.2" +googleHilt = "2.59.2" +googleKotlinKsp = "2.3.2" +hiltAndroidCompiler = "2.59.2" coreSplashscreen = "1.2.0" hiltNavigationCompose = "1.3.0" -navigationCompose = "2.9.6" +navigationCompose = "2.9.8" constraintlayoutCompose = "1.1.1" -kotlinxSerializationJson = "1.9.0" -richtextCommonmark = "1.0.0-alpha03" +kotlinxSerializationJson = "1.11.0" +richtextCommonmark = "1.0.0-alpha04" firebasePerf = "2.0.2" [libraries] diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 9787e6b..8144ab6 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ #Thu Feb 13 16:25:53 GMT 2025 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-9.4.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/settings.gradle.kts b/settings.gradle.kts index 82a9970..32a8873 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -11,6 +11,9 @@ pluginManagement { gradlePluginPortal() } } +plugins { + id("org.gradle.toolchains.foojay-resolver-convention") version "1.0.0" +} dependencyResolutionManagement { repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) repositories { From af6440f1c8a8270282d66d0bbe6800e7518629fd Mon Sep 17 00:00:00 2001 From: Marina Coelho Date: Wed, 13 May 2026 15:54:27 +0100 Subject: [PATCH 2/8] Removed deprecated Imagen models, updated Download Status for on device models --- .../data/datasource/AIRemoteDataSource.kt | 56 ++++++++----------- .../data/injection/FirebaseHiltModule.kt | 13 ----- .../data/repository/AIRepository.kt | 4 -- .../main/res/xml/remote_config_defaults.xml | 6 +- 4 files changed, 25 insertions(+), 54 deletions(-) diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/AIRemoteDataSource.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/AIRemoteDataSource.kt index ac3e97a..45e019c 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/AIRemoteDataSource.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/AIRemoteDataSource.kt @@ -3,15 +3,18 @@ package com.google.firebase.example.friendlymeals.data.datasource import android.graphics.Bitmap import android.util.Log import com.google.firebase.Firebase +import com.google.firebase.ai.DownloadStatus.DownloadCompleted +import com.google.firebase.ai.DownloadStatus.DownloadFailed +import com.google.firebase.ai.DownloadStatus.DownloadInProgress +import com.google.firebase.ai.DownloadStatus.DownloadStarted import com.google.firebase.ai.FirebaseAI import com.google.firebase.ai.InferenceMode import com.google.firebase.ai.InferenceSource import com.google.firebase.ai.OnDeviceConfig -import com.google.firebase.ai.TemplateGenerativeModel -import com.google.firebase.ai.TemplateImagenModel -import com.google.firebase.ai.ondevice.DownloadStatus -import com.google.firebase.ai.ondevice.FirebaseAIOnDevice -import com.google.firebase.ai.ondevice.OnDeviceModelStatus +import com.google.firebase.ai.OnDeviceModelStatus.Companion.AVAILABLE +import com.google.firebase.ai.OnDeviceModelStatus.Companion.DOWNLOADABLE +import com.google.firebase.ai.OnDeviceModelStatus.Companion.DOWNLOADING +import com.google.firebase.ai.OnDeviceModelStatus.Companion.UNAVAILABLE import com.google.firebase.ai.type.ImagePart import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.ai.type.content @@ -25,9 +28,7 @@ import javax.inject.Inject @OptIn(PublicPreviewAPI::class) class AIRemoteDataSource @Inject constructor( - private val aiModel: FirebaseAI, - private val generativeModel: TemplateGenerativeModel, - private val imagenModel: TemplateImagenModel, + aiModel: FirebaseAI, private val remoteConfig: FirebaseRemoteConfig ) { private val json = Json { ignoreUnknownKeys = true } @@ -36,7 +37,8 @@ class AIRemoteDataSource @Inject constructor( onDeviceConfig = OnDeviceConfig(mode = InferenceMode.PREFER_IN_CLOUD) ) - @OptIn(PublicPreviewAPI::class) + private val templateGenerativeModel = aiModel.templateGenerativeModel() + suspend fun generateIngredients(image: Bitmap): String { // Adding a Performance Monitoring trace is completely optional. Traces can help you // measure how long it takes to generate ingredients on device and in cloud. @@ -63,7 +65,7 @@ class AIRemoteDataSource @Inject constructor( } suspend fun generateRecipe(ingredients: String, notes: String): RecipeSchema? { - val response = generativeModel.generateContent( + val response = templateGenerativeModel.generateContent( templateId = remoteConfig.getString(GENERATE_RECIPE_KEY), inputs = buildMap { put(INGREDIENTS_FIELD, ingredients) @@ -79,7 +81,7 @@ class AIRemoteDataSource @Inject constructor( } suspend fun generateRecipePhoto(recipeTitle: String): Bitmap? { - val response = generativeModel.generateContent( + val response = templateGenerativeModel.generateContent( templateId = remoteConfig.getString(GENERATE_RECIPE_PHOTO_GEMINI_KEY), inputs = mapOf(RECIPE_TITLE_FIELD to recipeTitle) ) @@ -88,17 +90,8 @@ class AIRemoteDataSource @Inject constructor( ?.filterIsInstance()?.firstOrNull()?.image } - suspend fun generateRecipePhotoImagen(recipeTitle: String): Bitmap? { - val response = imagenModel.generateImages( - templateId = remoteConfig.getString(GENERATE_RECIPE_PHOTO_IMAGEN_KEY), - inputs = mapOf(RECIPE_TITLE_FIELD to recipeTitle) - ) - - return response.images.firstOrNull()?.asBitmap() - } - suspend fun scanMeal(imageData: String): MealSchema? { - val response = generativeModel.generateContent( + val response = templateGenerativeModel.generateContent( templateId = remoteConfig.getString(SCAN_MEAL_KEY), inputs = mapOf( MIME_TYPE_FIELD to MIME_TYPE_VALUE, @@ -112,31 +105,31 @@ class AIRemoteDataSource @Inject constructor( } suspend fun loadOnDeviceModel() { - when (FirebaseAIOnDevice.checkStatus()) { - OnDeviceModelStatus.UNAVAILABLE -> { + when (hybridGenerativeModel.onDeviceExtension?.checkStatus()) { + UNAVAILABLE -> { Log.i(TAG, "On-device model is unavailable") } - OnDeviceModelStatus.DOWNLOADABLE -> { - FirebaseAIOnDevice.download().collect { status -> + DOWNLOADABLE -> { + hybridGenerativeModel.onDeviceExtension?.download()?.collect { status -> when (status) { - is DownloadStatus.DownloadStarted -> + is DownloadStarted -> Log.i(TAG, "Starting download - ${status.bytesToDownload}") - is DownloadStatus.DownloadInProgress -> + is DownloadInProgress -> Log.i(TAG, "Download in progress ${status.totalBytesDownloaded} bytes downloaded") - is DownloadStatus.DownloadCompleted -> + is DownloadCompleted -> Log.i(TAG, "On-device model download complete") - is DownloadStatus.DownloadFailed -> + is DownloadFailed -> Log.e(TAG, "Download failed $status") } } } - OnDeviceModelStatus.DOWNLOADING -> { + DOWNLOADING -> { Log.i(TAG, "On-device model is being downloaded") } - OnDeviceModelStatus.AVAILABLE -> { + AVAILABLE -> { Log.i(TAG, "On-device model is available") } } @@ -146,7 +139,6 @@ class AIRemoteDataSource @Inject constructor( //Remote Config Keys private const val GENERATE_RECIPE_KEY = "generate_recipe" private const val GENERATE_RECIPE_PHOTO_GEMINI_KEY = "generate_recipe_photo_gemini" - private const val GENERATE_RECIPE_PHOTO_IMAGEN_KEY = "generate_recipe_photo_imagen" private const val SCAN_MEAL_KEY = "scan_meal" private const val HYBRID_CLOUD_MODEL_KEY = "hybrid_cloud_model" private const val HYBRID_INGREDIENTS_PROMPT_KEY = "hybrid_ingredients_prompt" diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/data/injection/FirebaseHiltModule.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/data/injection/FirebaseHiltModule.kt index a934497..595a0d5 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/data/injection/FirebaseHiltModule.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/data/injection/FirebaseHiltModule.kt @@ -3,11 +3,8 @@ package com.google.firebase.example.friendlymeals.data.injection import android.util.Log import com.google.firebase.Firebase import com.google.firebase.ai.FirebaseAI -import com.google.firebase.ai.TemplateGenerativeModel -import com.google.firebase.ai.TemplateImagenModel import com.google.firebase.ai.ai import com.google.firebase.ai.type.GenerativeBackend -import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.auth.FirebaseAuth import com.google.firebase.auth.auth import com.google.firebase.example.friendlymeals.R @@ -37,16 +34,6 @@ object FirebaseHiltModule { return Firebase.ai(backend = GenerativeBackend.googleAI()) } - @OptIn(PublicPreviewAPI::class) - @Provides fun generativeModel(ai: FirebaseAI): TemplateGenerativeModel { - return ai.templateGenerativeModel() - } - - @OptIn(PublicPreviewAPI::class) - @Provides fun imagenModel(ai: FirebaseAI): TemplateImagenModel { - return ai.templateImagenModel() - } - @Provides fun storage(): StorageReference { return Firebase.storage.reference } diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/data/repository/AIRepository.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/data/repository/AIRepository.kt index c1a6253..5a1a37e 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/data/repository/AIRepository.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/data/repository/AIRepository.kt @@ -21,10 +21,6 @@ class AIRepository @Inject constructor( return aiRemoteDataSource.generateRecipePhoto(recipeTitle) } - suspend fun generateRecipePhotoImagen(recipeTitle: String): Bitmap? { - return aiRemoteDataSource.generateRecipePhotoImagen(recipeTitle) - } - suspend fun scanMeal(imageData: String): MealSchema? { return aiRemoteDataSource.scanMeal(imageData) } diff --git a/app/src/main/res/xml/remote_config_defaults.xml b/app/src/main/res/xml/remote_config_defaults.xml index 1175602..c8fc36d 100644 --- a/app/src/main/res/xml/remote_config_defaults.xml +++ b/app/src/main/res/xml/remote_config_defaults.xml @@ -1,7 +1,7 @@ hybrid_cloud_model - gemini-3.1-flash-lite-preview + gemini-3.1-flash-lite hybrid_ingredients_prompt @@ -15,10 +15,6 @@ generate_recipe_photo_gemini generate-recipe-photo-gemini-template-v1-0-0 - - generate_recipe_photo_imagen - generate-recipe-photo-imagen-template-v1-0-0 - generate_recipe generate-recipe-template-v1-0-0 From d560c6c60276022d5ac1a9c958d8944480b12dbf Mon Sep 17 00:00:00 2001 From: Marina Coelho Date: Wed, 13 May 2026 19:00:34 +0100 Subject: [PATCH 3/8] Add Live Assistant Screen and functionality --- app/build.gradle.kts | 5 + app/src/main/AndroidManifest.xml | 1 + .../example/friendlymeals/MainActivity.kt | 12 ++ .../ui/live/LiveAssistantScreen.kt | 177 ++++++++++++++++++ .../ui/live/LiveAssistantUiState.kt | 9 + .../ui/live/LiveAssistantViewModel.kt | 157 ++++++++++++++++ .../friendlymeals/ui/recipe/RecipeScreen.kt | 70 ++++++- .../ui/recipe/RecipeViewModel.kt | 1 + .../ui/recipe/RecipeViewState.kt | 1 + gradle/libs.versions.toml | 7 + 10 files changed, 437 insertions(+), 3 deletions(-) create mode 100644 app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantScreen.kt create mode 100644 app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantUiState.kt create mode 100644 app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt diff --git a/app/build.gradle.kts b/app/build.gradle.kts index ef8be09..b4130f5 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -54,6 +54,11 @@ dependencies { implementation(libs.androidx.core.splashscreen) implementation(libs.androidx.constraintlayout.compose) implementation(libs.androidx.exifinterface) + implementation(libs.androidx.camera.core) + implementation(libs.androidx.camera.camera2) + implementation(libs.androidx.camera.lifecycle) + implementation(libs.androidx.camera.view) + implementation(libs.guava) implementation(libs.kotlinx.serialization.json) implementation(libs.hilt.android) ksp(libs.hilt.android.compiler) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 0e8567a..4ab4840 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -2,6 +2,7 @@ + { RecipeScreen( + navigateBack = { navController.popBackStack() }, + navigateToLiveAssistant = { recipeId -> + navController.navigate(LiveAssistantRoute(recipeId)) { + launchSingleTop = true + } + } + ) + } + composable { + LiveAssistantScreen( navigateBack = { navController.popBackStack() } ) } diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantScreen.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantScreen.kt new file mode 100644 index 0000000..ad9321e --- /dev/null +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantScreen.kt @@ -0,0 +1,177 @@ +package com.google.firebase.example.friendlymeals.ui.live + +import android.util.Log +import androidx.camera.core.CameraSelector +import androidx.camera.core.ImageAnalysis +import androidx.camera.core.Preview +import androidx.camera.lifecycle.ProcessCameraProvider +import androidx.camera.view.PreviewView +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.Scaffold +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.painterResource +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import androidx.compose.ui.viewinterop.AndroidView +import androidx.core.content.ContextCompat +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.compose.LocalLifecycleOwner +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.google.firebase.example.friendlymeals.R +import com.google.firebase.example.friendlymeals.ui.theme.Teal +import kotlinx.serialization.Serializable + +@Serializable +data class LiveAssistantRoute(val recipeId: String) + +@Composable +fun LiveAssistantScreen( + viewModel: LiveAssistantViewModel = hiltViewModel(), + navigateBack: () -> Unit +) { + val uiState by viewModel.uiState.collectAsStateWithLifecycle() + val lifecycleOwner = LocalLifecycleOwner.current + + Scaffold { innerPadding -> + Column( + modifier = Modifier + .fillMaxSize() + .padding(innerPadding) + .background(Color.Black), + horizontalAlignment = Alignment.CenterHorizontally + ) { + // Custom Top Bar + Row( + modifier = Modifier + .fillMaxWidth() + .padding(16.dp), + verticalAlignment = Alignment.CenterVertically + ) { + IconButton( + onClick = navigateBack, + modifier = Modifier + .background(Color.White.copy(alpha = 0.2f), CircleShape) + .size(40.dp) + ) { + Icon( + painter = painterResource(R.drawable.ic_arrow_back), + contentDescription = "Back", + tint = Color.White + ) + } + Spacer(modifier = Modifier.width(16.dp)) + Text( + text = "Live Cooking Assistant", + fontSize = 22.sp, + fontWeight = FontWeight.Bold, + color = Color.White + ) + } + + when (val state = uiState) { + is LiveAssistantUiState.Loading -> { + Box(modifier = Modifier.weight(1f), contentAlignment = Alignment.Center) { + CircularProgressIndicator(color = Teal) + } + } + is LiveAssistantUiState.Error -> { + Box(modifier = Modifier.weight(1f), contentAlignment = Alignment.Center) { + Text( + text = state.message, + color = Color.Red, + modifier = Modifier.padding(16.dp), + textAlign = TextAlign.Center + ) + } + } + is LiveAssistantUiState.Success -> { + Box( + modifier = Modifier + .weight(1f) + .fillMaxWidth() + .padding(horizontal = 16.dp, vertical = 8.dp) + .clip(RoundedCornerShape(24.dp)) + ) { + // CameraX Live Preview streaming to ViewModel + AndroidView( + factory = { ctx -> + val previewView = PreviewView(ctx) + val cameraProviderFuture = ProcessCameraProvider.getInstance(ctx) + cameraProviderFuture.addListener({ + val cameraProvider = cameraProviderFuture.get() + val preview = Preview.Builder().build().also { + it.surfaceProvider = previewView.surfaceProvider + } + val imageAnalyzer = ImageAnalysis.Builder() + .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) + .build() + .also { analysis -> + analysis.setAnalyzer(ContextCompat.getMainExecutor(ctx)) { imageProxy -> + val bitmap = imageProxy.toBitmap() + viewModel.sendVideoFrame(bitmap) + imageProxy.close() + } + } + val cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA + try { + cameraProvider.unbindAll() + cameraProvider.bindToLifecycle( + lifecycleOwner, + cameraSelector, + preview, + imageAnalyzer + ) + } catch (exc: Exception) { + Log.e("LiveAssistantScreen", "Use case binding failed", exc) + } + }, ContextCompat.getMainExecutor(ctx)) + previewView + }, + modifier = Modifier.fillMaxSize() + ) + + // Overlay instructions + Box( + modifier = Modifier + .align(Alignment.BottomCenter) + .fillMaxWidth() + .background(Color.Black.copy(alpha = 0.6f)) + .padding(16.dp), + contentAlignment = Alignment.Center + ) { + Text( + text = "Point your camera at your cooking and ask questions like:\n\"Is this the expected texture?\"", + color = Color.White, + fontSize = 14.sp, + textAlign = TextAlign.Center + ) + } + } + } + } + } + } +} diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantUiState.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantUiState.kt new file mode 100644 index 0000000..ba48ded --- /dev/null +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantUiState.kt @@ -0,0 +1,9 @@ +package com.google.firebase.example.friendlymeals.ui.live + +import com.google.firebase.example.friendlymeals.data.model.Recipe + +sealed interface LiveAssistantUiState { + data object Loading : LiveAssistantUiState + data class Success(val recipe: Recipe) : LiveAssistantUiState + data class Error(val message: String) : LiveAssistantUiState +} diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt new file mode 100644 index 0000000..4324916 --- /dev/null +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt @@ -0,0 +1,157 @@ +package com.google.firebase.example.friendlymeals.ui.live + +import android.annotation.SuppressLint +import android.graphics.Bitmap +import android.util.Log +import androidx.lifecycle.SavedStateHandle +import androidx.lifecycle.viewModelScope +import androidx.navigation.toRoute +import com.google.firebase.ai.FirebaseAI +import com.google.firebase.ai.type.FunctionCallPart +import com.google.firebase.ai.type.FunctionResponsePart +import com.google.firebase.ai.type.InlineData +import com.google.firebase.ai.type.LiveSession +import com.google.firebase.ai.type.PublicPreviewAPI +import com.google.firebase.ai.type.ResponseModality +import com.google.firebase.ai.type.SpeechConfig +import com.google.firebase.ai.type.Voice +import com.google.firebase.ai.type.content +import com.google.firebase.ai.type.liveGenerationConfig +import com.google.firebase.example.friendlymeals.MainViewModel +import com.google.firebase.example.friendlymeals.data.model.Recipe +import com.google.firebase.example.friendlymeals.data.repository.DatabaseRepository +import dagger.hilt.android.lifecycle.HiltViewModel +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.JsonObject +import java.io.ByteArrayOutputStream +import javax.inject.Inject + +@HiltViewModel +@OptIn(PublicPreviewAPI::class) +class LiveAssistantViewModel @Inject constructor( + savedStateHandle: SavedStateHandle, + private val databaseRepository: DatabaseRepository, + private val firebaseAI: FirebaseAI +) : MainViewModel() { + private val route = savedStateHandle.toRoute() + val recipeId: String = route.recipeId + + private val _uiState = MutableStateFlow(LiveAssistantUiState.Loading) + val uiState: StateFlow = _uiState.asStateFlow() + + private var liveSession: LiveSession? = null + private var isConnected = false + private var lastFrameTime = 0L + + init { + loadRecipeAndConnect() + } + + private fun loadRecipeAndConnect() { + viewModelScope.launch { + try { + val recipe = databaseRepository.getRecipe(recipeId) + _uiState.value = LiveAssistantUiState.Success(recipe) + setupAndConnectLiveSession(recipe) + } catch (e: Exception) { + Log.e(TAG, "Error loading recipe for Live Assistant", e) + _uiState.value = LiveAssistantUiState.Error(e.message ?: "Failed to load recipe") + } + } + } + + private suspend fun setupAndConnectLiveSession(recipe: Recipe) { + try { + val liveGenerationConfig = liveGenerationConfig { + speechConfig = SpeechConfig(voice = Voice("CHARON")) + responseModality = ResponseModality.AUDIO + } + + val instructionText = """ + You are a helpful live cooking assistant. The user is currently preparing the following recipe: + Title: ${recipe.title} + Prep time: ${recipe.prepTime}, Cook time: ${recipe.cookTime}, Servings: ${recipe.servings} + + Ingredients: + ${recipe.ingredients.joinToString("\n")} + + Instructions: + ${recipe.instructions} + + The user will stream real-time video of their cooking and ask questions like "Is this the expected texture of the recipe?". + Confirm or deny accurately based on the recipe context and the video content. Be concise and helpful. + """.trimIndent() + + val liveModel = firebaseAI.liveModel( + modelName = "gemini-2.5-flash-native-audio-preview-09-2025", + generationConfig = liveGenerationConfig, + systemInstruction = content { text(instructionText) } + ) + + withContext(Dispatchers.IO) { + liveSession = liveModel.connect() + } + isConnected = true + startConversation() + } catch (e: Exception) { + Log.e(TAG, "Failed to connect LiveSession", e) + _uiState.value = LiveAssistantUiState.Error("Failed to connect to live assistant: ${e.message}") + } + } + + private fun handler(functionCall: FunctionCallPart): FunctionResponsePart { + return FunctionResponsePart(functionCall.name, JsonObject(emptyMap()), functionCall.id) + } + + @SuppressLint("MissingPermission") + fun startConversation() { + viewModelScope.launch { + try { + liveSession?.startAudioConversation(::handler) + } catch (e: Exception) { + Log.e(TAG, "Error starting audio conversation", e) + } + } + } + + fun endConversation() { + try { + liveSession?.stopAudioConversation() + } catch (e: Exception) { + Log.e(TAG, "Error stopping audio conversation", e) + } + } + + fun sendVideoFrame(bitmap: Bitmap) { + if (!isConnected || liveSession == null) return + val currentTime = System.currentTimeMillis() + // Limit sending frames to once per second to conserve bandwidth and processing + if (currentTime - lastFrameTime < 1000) return + lastFrameTime = currentTime + + viewModelScope.launch(Dispatchers.IO) { + try { + val outputStream = ByteArrayOutputStream() + bitmap.compress(Bitmap.CompressFormat.JPEG, 80, outputStream) + val jpegBytes = outputStream.toByteArray() + liveSession?.sendVideoRealtime(InlineData(jpegBytes, "image/jpeg")) + } catch (e: Exception) { + Log.e(TAG, "Error sending video frame", e) + } + } + } + + override fun onCleared() { + super.onCleared() + endConversation() + } + + companion object { + private const val TAG = "LiveAssistantViewModel" + } +} diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeScreen.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeScreen.kt index 29ac8f9..5914e5e 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeScreen.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeScreen.kt @@ -1,6 +1,14 @@ package com.google.firebase.example.friendlymeals.ui.recipe +import android.Manifest +import android.content.pm.PackageManager +import android.widget.Toast +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.foundation.background +import androidx.compose.material3.Button +import androidx.compose.material3.ButtonDefaults +import androidx.core.content.ContextCompat import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column @@ -54,7 +62,8 @@ data class RecipeRoute(val recipeId: String) @Composable fun RecipeScreen( viewModel: RecipeViewModel = hiltViewModel(), - navigateBack: () -> Unit + navigateBack: () -> Unit, + navigateToLiveAssistant: (String) -> Unit ) { val recipeViewState = viewModel.recipeViewState.collectAsStateWithLifecycle() @@ -62,7 +71,10 @@ fun RecipeScreen( navigateBack = navigateBack, toggleFavorite = viewModel::toggleFavorite, leaveReview = viewModel::leaveReview, - recipeViewState = recipeViewState.value + recipeViewState = recipeViewState.value, + onLiveAssistantClick = { + navigateToLiveAssistant(recipeViewState.value.recipeId) + } ) } @@ -71,8 +83,24 @@ fun RecipeScreenContent( navigateBack: () -> Unit = {}, toggleFavorite: () -> Unit = {}, leaveReview: (Int) -> Unit = {}, - recipeViewState: RecipeViewState + recipeViewState: RecipeViewState, + onLiveAssistantClick: () -> Unit = {} ) { + val context = LocalContext.current + val multiplePermissionsLauncher = rememberLauncherForActivityResult( + contract = ActivityResultContracts.RequestMultiplePermissions() + ) { permissions -> + val cameraGranted = permissions[Manifest.permission.CAMERA] == true + val audioGranted = permissions[Manifest.permission.RECORD_AUDIO] == true + if (cameraGranted && audioGranted) { + onLiveAssistantClick() + } else { + Toast.makeText(context, "Camera and Microphone permissions are required to use the Live Assistant.", Toast.LENGTH_LONG).show() + } + } + val cameraPermissionGranted = ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED + val audioPermissionGranted = ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED + val favoriteIcon = if (recipeViewState.favorite) { painterResource(R.drawable.ic_favorite_filled) } else { @@ -165,6 +193,42 @@ fun RecipeScreenContent( lineHeight = 34.sp ) + Spacer(modifier = Modifier.height(16.dp)) + + Button( + onClick = { + if (cameraPermissionGranted && audioPermissionGranted) { + onLiveAssistantClick() + } else { + multiplePermissionsLauncher.launch( + arrayOf(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO) + ) + } + }, + colors = ButtonDefaults.buttonColors(containerColor = Teal), + shape = RoundedCornerShape(16.dp), + modifier = Modifier.fillMaxWidth().height(52.dp) + ) { + Row( + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.Center + ) { + Icon( + painter = painterResource(R.drawable.ic_cook), + contentDescription = null, + tint = Color.White, + modifier = Modifier.size(20.dp) + ) + Spacer(modifier = Modifier.width(8.dp)) + Text( + text = "Live Cooking Assistant", + color = Color.White, + fontSize = 16.sp, + fontWeight = FontWeight.Bold + ) + } + } + Spacer(modifier = Modifier.height(24.dp)) Row( diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeViewModel.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeViewModel.kt index ed0ae52..1233a58 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeViewModel.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeViewModel.kt @@ -35,6 +35,7 @@ class RecipeViewModel @Inject constructor( fun loadRecipe() { launchCatching { _recipeViewState.value = RecipeViewState( + recipeId = recipeId, recipe = databaseRepository.getRecipe(recipeId), favorite = loadFavorite(), rating = loadRating() diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeViewState.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeViewState.kt index 2238413..1e8aee9 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeViewState.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeViewState.kt @@ -3,6 +3,7 @@ package com.google.firebase.example.friendlymeals.ui.recipe import com.google.firebase.example.friendlymeals.data.model.Recipe data class RecipeViewState( + val recipeId: String = "", val recipe: Recipe = Recipe(), val favorite: Boolean = false, val rating: Int = 0 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 0d5f938..0132bc6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,5 +1,6 @@ [versions] agp = "9.2.1" +camerax = "1.4.1" coilCompose = "2.7.0" exifinterface = "1.4.2" firebaseAi = "17.12.0" @@ -15,6 +16,7 @@ activityCompose = "1.13.0" composeBom = "2026.05.00" googleServices = "4.4.4" googleHilt = "2.59.2" +guava = "33.4.0-android" googleKotlinKsp = "2.3.2" hiltAndroidCompiler = "2.59.2" coreSplashscreen = "1.2.0" @@ -58,6 +60,11 @@ androidx-navigation-compose = { module = "androidx.navigation:navigation-compose androidx-constraintlayout-compose = { module = "androidx.constraintlayout:constraintlayout-compose", version.ref = "constraintlayoutCompose" } kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" } richtext-commonmark = { module = "com.halilibo.compose-richtext:richtext-commonmark", version.ref = "richtextCommonmark" } +androidx-camera-core = { group = "androidx.camera", name = "camera-core", version.ref = "camerax" } +androidx-camera-camera2 = { group = "androidx.camera", name = "camera-camera2", version.ref = "camerax" } +androidx-camera-lifecycle = { group = "androidx.camera", name = "camera-lifecycle", version.ref = "camerax" } +androidx-camera-view = { group = "androidx.camera", name = "camera-view", version.ref = "camerax" } +guava = { module = "com.google.guava:guava", version.ref = "guava" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" } From cf6e994e08ba6229eb6f825f958ce5b5e401c489 Mon Sep 17 00:00:00 2001 From: Marina Coelho Date: Thu, 14 May 2026 15:55:45 +0100 Subject: [PATCH 4/8] Clean up code and fix architecture --- app/build.gradle.kts | 1 + .../example/friendlymeals/MainActivity.kt | 6 +- .../data/datasource/LiveAIRemoteDataSource.kt | 64 +++++++++++ .../data/repository/LiveAIRepository.kt | 16 +++ .../ui/live/LiveAssistantScreen.kt | 91 ++++++++++----- .../ui/live/LiveAssistantViewModel.kt | 107 ++++++------------ .../friendlymeals/ui/recipe/RecipeScreen.kt | 15 +-- .../ui/recipeList/filter/FilterScreen.kt | 2 +- app/src/main/res/values/strings.xml | 5 +- .../main/res/xml/remote_config_defaults.xml | 8 ++ gradle/libs.versions.toml | 2 + 11 files changed, 205 insertions(+), 112 deletions(-) create mode 100644 app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt create mode 100644 app/src/main/java/com/google/firebase/example/friendlymeals/data/repository/LiveAIRepository.kt diff --git a/app/build.gradle.kts b/app/build.gradle.kts index b4130f5..0a04664 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -58,6 +58,7 @@ dependencies { implementation(libs.androidx.camera.camera2) implementation(libs.androidx.camera.lifecycle) implementation(libs.androidx.camera.view) + implementation(libs.camera.view) implementation(libs.guava) implementation(libs.kotlinx.serialization.json) implementation(libs.hilt.android) diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/MainActivity.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/MainActivity.kt index ce96d9d..30c476c 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/MainActivity.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/MainActivity.kt @@ -133,7 +133,11 @@ class MainActivity : ComponentActivity() { } composable { LiveAssistantScreen( - navigateBack = { navController.popBackStack() } + navigateBack = { navController.popBackStack() }, + showError = { + val message = this@MainActivity.getString(R.string.camera_error_message) + scope.launch { snackbarHostState.showSnackbar(message) } + } ) } } diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt new file mode 100644 index 0000000..ce81864 --- /dev/null +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt @@ -0,0 +1,64 @@ +package com.google.firebase.example.friendlymeals.data.datasource + +import com.google.firebase.ai.FirebaseAI +import com.google.firebase.ai.type.LiveSession +import com.google.firebase.ai.type.PublicPreviewAPI +import com.google.firebase.ai.type.ResponseModality +import com.google.firebase.ai.type.SpeechConfig +import com.google.firebase.ai.type.Voice +import com.google.firebase.ai.type.content +import com.google.firebase.ai.type.liveGenerationConfig +import com.google.firebase.example.friendlymeals.data.model.Recipe +import com.google.firebase.remoteconfig.FirebaseRemoteConfig +import javax.inject.Inject + +@OptIn(PublicPreviewAPI::class) +class LiveAIRemoteDataSource @Inject constructor( + private val aiModel: FirebaseAI, + private val remoteConfig: FirebaseRemoteConfig +) { + @OptIn(PublicPreviewAPI::class) + suspend fun setupLiveSession(recipe: Recipe): LiveSession? { + val liveGenerationConfig = liveGenerationConfig { + speechConfig = SpeechConfig(voice = Voice(LIVE_MODEL_VOICE)) + responseModality = ResponseModality.AUDIO + } + + // This is temporary, prompt will be moved to Remote Config soon + val instructionText = """ + You are a helpful live cooking assistant. The user is currently preparing the following recipe: + Title: ${recipe.title} + Prep time: ${recipe.prepTime}, Cook time: ${recipe.cookTime}, Servings: ${recipe.servings} + + Ingredients: + ${recipe.ingredients.joinToString("\n")} + + Instructions: + ${recipe.instructions} + + The user will stream real-time video of their cooking and ask questions like "Is this the expected texture of the recipe?". + Confirm or deny accurately based on the recipe context and the video content. Be concise and helpful. + """.trimIndent() + + val liveModel = aiModel.liveModel( + modelName = remoteConfig.getString(LIVE_MODEL_NAME_KEY), + generationConfig = liveGenerationConfig, + systemInstruction = content { text(instructionText) } + ) + + return try { + liveModel.connect() + } catch (_: Exception) { + null + } + } + + companion object { + //Live Model Config + private const val LIVE_MODEL_VOICE = "CHARON" + + //Remote Config Keys + private const val LIVE_MODEL_NAME_KEY = "live_model_name" + private const val LIVE_MODEL_PROMPT_KEY = "live_model_prompt" + } +} \ No newline at end of file diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/data/repository/LiveAIRepository.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/data/repository/LiveAIRepository.kt new file mode 100644 index 0000000..066cd82 --- /dev/null +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/data/repository/LiveAIRepository.kt @@ -0,0 +1,16 @@ +package com.google.firebase.example.friendlymeals.data.repository + +import com.google.firebase.ai.type.LiveSession +import com.google.firebase.ai.type.PublicPreviewAPI +import com.google.firebase.example.friendlymeals.data.datasource.LiveAIRemoteDataSource +import com.google.firebase.example.friendlymeals.data.model.Recipe +import javax.inject.Inject + +class LiveAIRepository @Inject constructor( + private val liveAiRemoteDataSource: LiveAIRemoteDataSource +) { + @OptIn(PublicPreviewAPI::class) + suspend fun setupLiveSession(recipe: Recipe): LiveSession? { + return liveAiRemoteDataSource.setupLiveSession(recipe) + } +} \ No newline at end of file diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantScreen.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantScreen.kt index ad9321e..5f16da9 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantScreen.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantScreen.kt @@ -1,8 +1,9 @@ package com.google.firebase.example.friendlymeals.ui.live -import android.util.Log -import androidx.camera.core.CameraSelector +import android.graphics.Bitmap +import androidx.camera.core.CameraSelector.DEFAULT_BACK_CAMERA import androidx.camera.core.ImageAnalysis +import androidx.camera.core.ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST import androidx.camera.core.Preview import androidx.camera.lifecycle.ProcessCameraProvider import androidx.camera.view.PreviewView @@ -24,23 +25,24 @@ import androidx.compose.material3.IconButton import androidx.compose.material3.Scaffold import androidx.compose.material3.Text import androidx.compose.runtime.Composable -import androidx.compose.runtime.getValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip import androidx.compose.ui.graphics.Color -import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.res.painterResource +import androidx.compose.ui.res.stringResource import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp import androidx.compose.ui.viewinterop.AndroidView -import androidx.core.content.ContextCompat +import androidx.core.content.ContextCompat.getMainExecutor import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel import androidx.lifecycle.compose.LocalLifecycleOwner import androidx.lifecycle.compose.collectAsStateWithLifecycle import com.google.firebase.example.friendlymeals.R +import com.google.firebase.example.friendlymeals.data.model.Recipe +import com.google.firebase.example.friendlymeals.ui.theme.FriendlyMealsTheme import com.google.firebase.example.friendlymeals.ui.theme.Teal import kotlinx.serialization.Serializable @@ -50,9 +52,26 @@ data class LiveAssistantRoute(val recipeId: String) @Composable fun LiveAssistantScreen( viewModel: LiveAssistantViewModel = hiltViewModel(), - navigateBack: () -> Unit + navigateBack: () -> Unit, + showError: () -> Unit +) { + val uiState = viewModel.uiState.collectAsStateWithLifecycle() + + LiveAssistantScreenContent( + uiState = uiState.value, + navigateBack = navigateBack, + sendVideoFrame = viewModel::sendVideoFrame, + showError = showError + ) +} + +@Composable +fun LiveAssistantScreenContent( + uiState: LiveAssistantUiState, + navigateBack: () -> Unit = {}, + sendVideoFrame: (Bitmap) -> Unit = {}, + showError: () -> Unit = {} ) { - val uiState by viewModel.uiState.collectAsStateWithLifecycle() val lifecycleOwner = LocalLifecycleOwner.current Scaffold { innerPadding -> @@ -63,7 +82,6 @@ fun LiveAssistantScreen( .background(Color.Black), horizontalAlignment = Alignment.CenterHorizontally ) { - // Custom Top Bar Row( modifier = Modifier .fillMaxWidth() @@ -78,29 +96,37 @@ fun LiveAssistantScreen( ) { Icon( painter = painterResource(R.drawable.ic_arrow_back), - contentDescription = "Back", + contentDescription = stringResource(id = R.string.back_button_content_description), tint = Color.White ) } + Spacer(modifier = Modifier.width(16.dp)) + Text( - text = "Live Cooking Assistant", + text = stringResource(id = R.string.live_assistant_title), fontSize = 22.sp, fontWeight = FontWeight.Bold, color = Color.White ) } - when (val state = uiState) { + when (uiState) { is LiveAssistantUiState.Loading -> { - Box(modifier = Modifier.weight(1f), contentAlignment = Alignment.Center) { + Box( + modifier = Modifier.weight(1f), + contentAlignment = Alignment.Center + ) { CircularProgressIndicator(color = Teal) } } is LiveAssistantUiState.Error -> { - Box(modifier = Modifier.weight(1f), contentAlignment = Alignment.Center) { + Box( + modifier = Modifier.weight(1f), + contentAlignment = Alignment.Center + ) { Text( - text = state.message, + text = uiState.message, color = Color.Red, modifier = Modifier.padding(16.dp), textAlign = TextAlign.Center @@ -115,45 +141,44 @@ fun LiveAssistantScreen( .padding(horizontal = 16.dp, vertical = 8.dp) .clip(RoundedCornerShape(24.dp)) ) { - // CameraX Live Preview streaming to ViewModel AndroidView( - factory = { ctx -> - val previewView = PreviewView(ctx) - val cameraProviderFuture = ProcessCameraProvider.getInstance(ctx) + factory = { context -> + val previewView = PreviewView(context) + val cameraProviderFuture = ProcessCameraProvider.getInstance(context) + cameraProviderFuture.addListener({ val cameraProvider = cameraProviderFuture.get() val preview = Preview.Builder().build().also { it.surfaceProvider = previewView.surfaceProvider } val imageAnalyzer = ImageAnalysis.Builder() - .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) + .setBackpressureStrategy(STRATEGY_KEEP_ONLY_LATEST) .build() .also { analysis -> - analysis.setAnalyzer(ContextCompat.getMainExecutor(ctx)) { imageProxy -> + analysis.setAnalyzer(getMainExecutor(context)) { imageProxy -> val bitmap = imageProxy.toBitmap() - viewModel.sendVideoFrame(bitmap) + sendVideoFrame(bitmap) imageProxy.close() } } - val cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA + try { cameraProvider.unbindAll() cameraProvider.bindToLifecycle( lifecycleOwner, - cameraSelector, + DEFAULT_BACK_CAMERA, preview, imageAnalyzer ) - } catch (exc: Exception) { - Log.e("LiveAssistantScreen", "Use case binding failed", exc) + } catch (_: Exception) { + showError() } - }, ContextCompat.getMainExecutor(ctx)) + }, getMainExecutor(context)) previewView }, modifier = Modifier.fillMaxSize() ) - // Overlay instructions Box( modifier = Modifier .align(Alignment.BottomCenter) @@ -163,7 +188,7 @@ fun LiveAssistantScreen( contentAlignment = Alignment.Center ) { Text( - text = "Point your camera at your cooking and ask questions like:\n\"Is this the expected texture?\"", + text = stringResource(id = R.string.live_assistant_hint), color = Color.White, fontSize = 14.sp, textAlign = TextAlign.Center @@ -175,3 +200,13 @@ fun LiveAssistantScreen( } } } + +@androidx.compose.ui.tooling.preview.Preview +@Composable +fun LiveAssistantScreenPreview() { + FriendlyMealsTheme { + LiveAssistantScreenContent( + uiState = LiveAssistantUiState.Success(Recipe()) + ) + } +} diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt index 4324916..98d5b54 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt @@ -2,31 +2,22 @@ package com.google.firebase.example.friendlymeals.ui.live import android.annotation.SuppressLint import android.graphics.Bitmap -import android.util.Log import androidx.lifecycle.SavedStateHandle -import androidx.lifecycle.viewModelScope import androidx.navigation.toRoute -import com.google.firebase.ai.FirebaseAI import com.google.firebase.ai.type.FunctionCallPart import com.google.firebase.ai.type.FunctionResponsePart import com.google.firebase.ai.type.InlineData import com.google.firebase.ai.type.LiveSession import com.google.firebase.ai.type.PublicPreviewAPI -import com.google.firebase.ai.type.ResponseModality -import com.google.firebase.ai.type.SpeechConfig -import com.google.firebase.ai.type.Voice -import com.google.firebase.ai.type.content -import com.google.firebase.ai.type.liveGenerationConfig import com.google.firebase.example.friendlymeals.MainViewModel import com.google.firebase.example.friendlymeals.data.model.Recipe import com.google.firebase.example.friendlymeals.data.repository.DatabaseRepository +import com.google.firebase.example.friendlymeals.data.repository.LiveAIRepository +import com.google.firebase.example.friendlymeals.ui.live.LiveAssistantUiState.Loading import dagger.hilt.android.lifecycle.HiltViewModel -import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow -import kotlinx.coroutines.launch -import kotlinx.coroutines.withContext import kotlinx.serialization.json.JsonObject import java.io.ByteArrayOutputStream import javax.inject.Inject @@ -36,12 +27,12 @@ import javax.inject.Inject class LiveAssistantViewModel @Inject constructor( savedStateHandle: SavedStateHandle, private val databaseRepository: DatabaseRepository, - private val firebaseAI: FirebaseAI + private val liveAIRepository: LiveAIRepository ) : MainViewModel() { private val route = savedStateHandle.toRoute() val recipeId: String = route.recipeId - private val _uiState = MutableStateFlow(LiveAssistantUiState.Loading) + private val _uiState = MutableStateFlow(Loading) val uiState: StateFlow = _uiState.asStateFlow() private var liveSession: LiveSession? = null @@ -53,54 +44,27 @@ class LiveAssistantViewModel @Inject constructor( } private fun loadRecipeAndConnect() { - viewModelScope.launch { - try { - val recipe = databaseRepository.getRecipe(recipeId) + launchCatching { + val recipe = databaseRepository.getRecipe(recipeId) + + if (recipe.title.isBlank()) { + _uiState.value = LiveAssistantUiState.Error(RECIPE_ERROR) + } else { _uiState.value = LiveAssistantUiState.Success(recipe) - setupAndConnectLiveSession(recipe) - } catch (e: Exception) { - Log.e(TAG, "Error loading recipe for Live Assistant", e) - _uiState.value = LiveAssistantUiState.Error(e.message ?: "Failed to load recipe") + setupLiveSession(recipe) } } } - private suspend fun setupAndConnectLiveSession(recipe: Recipe) { - try { - val liveGenerationConfig = liveGenerationConfig { - speechConfig = SpeechConfig(voice = Voice("CHARON")) - responseModality = ResponseModality.AUDIO - } + private suspend fun setupLiveSession(recipe: Recipe) { + val session = liveAIRepository.setupLiveSession(recipe) - val instructionText = """ - You are a helpful live cooking assistant. The user is currently preparing the following recipe: - Title: ${recipe.title} - Prep time: ${recipe.prepTime}, Cook time: ${recipe.cookTime}, Servings: ${recipe.servings} - - Ingredients: - ${recipe.ingredients.joinToString("\n")} - - Instructions: - ${recipe.instructions} - - The user will stream real-time video of their cooking and ask questions like "Is this the expected texture of the recipe?". - Confirm or deny accurately based on the recipe context and the video content. Be concise and helpful. - """.trimIndent() - - val liveModel = firebaseAI.liveModel( - modelName = "gemini-2.5-flash-native-audio-preview-09-2025", - generationConfig = liveGenerationConfig, - systemInstruction = content { text(instructionText) } - ) - - withContext(Dispatchers.IO) { - liveSession = liveModel.connect() - } + if (session == null) { + _uiState.value = LiveAssistantUiState.Error(CONNECTION_ERROR) + } else { + liveSession = session isConnected = true startConversation() - } catch (e: Exception) { - Log.e(TAG, "Failed to connect LiveSession", e) - _uiState.value = LiveAssistantUiState.Error("Failed to connect to live assistant: ${e.message}") } } @@ -108,41 +72,34 @@ class LiveAssistantViewModel @Inject constructor( return FunctionResponsePart(functionCall.name, JsonObject(emptyMap()), functionCall.id) } + // Suppressing MissingPermission warning as we're + // checking permissions before opening the screen @SuppressLint("MissingPermission") - fun startConversation() { - viewModelScope.launch { - try { - liveSession?.startAudioConversation(::handler) - } catch (e: Exception) { - Log.e(TAG, "Error starting audio conversation", e) - } + private fun startConversation() { + launchCatching { + liveSession?.startAudioConversation(::handler) } } - fun endConversation() { - try { + private fun endConversation() { + launchCatching { liveSession?.stopAudioConversation() - } catch (e: Exception) { - Log.e(TAG, "Error stopping audio conversation", e) } } fun sendVideoFrame(bitmap: Bitmap) { if (!isConnected || liveSession == null) return val currentTime = System.currentTimeMillis() + // Limit sending frames to once per second to conserve bandwidth and processing if (currentTime - lastFrameTime < 1000) return lastFrameTime = currentTime - viewModelScope.launch(Dispatchers.IO) { - try { - val outputStream = ByteArrayOutputStream() - bitmap.compress(Bitmap.CompressFormat.JPEG, 80, outputStream) - val jpegBytes = outputStream.toByteArray() - liveSession?.sendVideoRealtime(InlineData(jpegBytes, "image/jpeg")) - } catch (e: Exception) { - Log.e(TAG, "Error sending video frame", e) - } + launchCatching { + val outputStream = ByteArrayOutputStream() + bitmap.compress(Bitmap.CompressFormat.JPEG, 80, outputStream) + val jpegBytes = outputStream.toByteArray() + liveSession?.sendVideoRealtime(InlineData(jpegBytes, MIME_TYPE)) } } @@ -152,6 +109,8 @@ class LiveAssistantViewModel @Inject constructor( } companion object { - private const val TAG = "LiveAssistantViewModel" + private const val MIME_TYPE = "image/jpeg" + private const val RECIPE_ERROR = "Failed to load recipe" + private const val CONNECTION_ERROR = "Failed to connect to live assistant" } } diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeScreen.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeScreen.kt index 5914e5e..1e91c5b 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeScreen.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeScreen.kt @@ -1,6 +1,7 @@ package com.google.firebase.example.friendlymeals.ui.recipe -import android.Manifest +import android.Manifest.permission.CAMERA +import android.Manifest.permission.RECORD_AUDIO import android.content.pm.PackageManager import android.widget.Toast import androidx.activity.compose.rememberLauncherForActivityResult @@ -90,16 +91,16 @@ fun RecipeScreenContent( val multiplePermissionsLauncher = rememberLauncherForActivityResult( contract = ActivityResultContracts.RequestMultiplePermissions() ) { permissions -> - val cameraGranted = permissions[Manifest.permission.CAMERA] == true - val audioGranted = permissions[Manifest.permission.RECORD_AUDIO] == true + val cameraGranted = permissions[CAMERA] == true + val audioGranted = permissions[RECORD_AUDIO] == true if (cameraGranted && audioGranted) { onLiveAssistantClick() } else { Toast.makeText(context, "Camera and Microphone permissions are required to use the Live Assistant.", Toast.LENGTH_LONG).show() } } - val cameraPermissionGranted = ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED - val audioPermissionGranted = ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED + val cameraPermissionGranted = ContextCompat.checkSelfPermission(context, CAMERA) == PackageManager.PERMISSION_GRANTED + val audioPermissionGranted = ContextCompat.checkSelfPermission(context, RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED val favoriteIcon = if (recipeViewState.favorite) { painterResource(R.drawable.ic_favorite_filled) @@ -158,7 +159,7 @@ fun RecipeScreenContent( ) { Icon( painter = painterResource(R.drawable.ic_arrow_back), - contentDescription = stringResource(id = R.string.recipe_back_button_content_description), + contentDescription = stringResource(id = R.string.back_button_content_description), tint = TextColor ) } @@ -201,7 +202,7 @@ fun RecipeScreenContent( onLiveAssistantClick() } else { multiplePermissionsLauncher.launch( - arrayOf(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO) + arrayOf(CAMERA, RECORD_AUDIO) ) } }, diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipeList/filter/FilterScreen.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipeList/filter/FilterScreen.kt index 4a1ff0e..76aabba 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipeList/filter/FilterScreen.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipeList/filter/FilterScreen.kt @@ -121,7 +121,7 @@ fun FilterScreenContent( IconButton(onClick = { navigateBack() }) { Icon( painter = painterResource(R.drawable.ic_arrow_back), - contentDescription = stringResource(id = R.string.recipe_back_button_content_description) + contentDescription = stringResource(id = R.string.back_button_content_description) ) } } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 1cfb76c..b0f6e8f 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -3,6 +3,7 @@ Enter your list of ingredients Any notes or preferred cuisines? Generate recipe + Back Friendly Meals Logo New recipe List your ingredients @@ -13,7 +14,6 @@ Generate Recipe Recipe image Could not load image - Back Favorite Prep Time Cook Time @@ -53,4 +53,7 @@ Recipes Take a picture of ingredients Sorry, something went wrong :( + Error loading camera + Live Cooking Assistant + Point your camera at your cooking and ask questions \ No newline at end of file diff --git a/app/src/main/res/xml/remote_config_defaults.xml b/app/src/main/res/xml/remote_config_defaults.xml index c8fc36d..6a45bcb 100644 --- a/app/src/main/res/xml/remote_config_defaults.xml +++ b/app/src/main/res/xml/remote_config_defaults.xml @@ -19,4 +19,12 @@ generate_recipe generate-recipe-template-v1-0-0 + + live_model_name + gemini-2.5-flash-native-audio-preview-12-2025 + + + live_model_prompt + tbd + \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 0132bc6..e93ab8e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -26,6 +26,7 @@ constraintlayoutCompose = "1.1.1" kotlinxSerializationJson = "1.11.0" richtextCommonmark = "1.0.0-alpha04" firebasePerf = "2.0.2" +cameraView = "1.6.1" [libraries] androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" } @@ -65,6 +66,7 @@ androidx-camera-camera2 = { group = "androidx.camera", name = "camera-camera2", androidx-camera-lifecycle = { group = "androidx.camera", name = "camera-lifecycle", version.ref = "camerax" } androidx-camera-view = { group = "androidx.camera", name = "camera-view", version.ref = "camerax" } guava = { module = "com.google.guava:guava", version.ref = "guava" } +camera-view = { group = "androidx.camera", name = "camera-view", version.ref = "cameraView" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" } From 2904d71f8d9b1a4328754291803d2b493d03c1dd Mon Sep 17 00:00:00 2001 From: Marina Coelho Date: Fri, 15 May 2026 18:03:48 +0100 Subject: [PATCH 5/8] Add grocery list tab and functionality, and tool calling to Gemini Live --- app/build.gradle.kts | 4 +- .../example/friendlymeals/MainActivity.kt | 5 + .../datasource/DatabaseRemoteDataSource.kt | 67 +++++ .../data/datasource/LiveAIRemoteDataSource.kt | 24 +- .../friendlymeals/data/model/GroceryItem.kt | 8 + .../data/repository/DatabaseRepository.kt | 22 ++ .../ui/groceryList/GroceryListScreen.kt | 276 ++++++++++++++++++ .../ui/groceryList/GroceryListViewModel.kt | 64 ++++ .../ui/live/LiveAssistantViewModel.kt | 39 +++ .../friendlymeals/ui/recipe/RecipeScreen.kt | 42 ++- .../ui/recipe/RecipeViewModel.kt | 9 + .../friendlymeals/ui/shared/BottomNavBar.kt | 5 +- app/src/main/res/drawable/ic_add.xml | 9 + app/src/main/res/drawable/ic_delete.xml | 9 + app/src/main/res/values/strings.xml | 8 + gradle.properties | 3 +- gradle/gradle-daemon-jvm.properties | 13 - 17 files changed, 588 insertions(+), 19 deletions(-) create mode 100644 app/src/main/java/com/google/firebase/example/friendlymeals/data/model/GroceryItem.kt create mode 100644 app/src/main/java/com/google/firebase/example/friendlymeals/ui/groceryList/GroceryListScreen.kt create mode 100644 app/src/main/java/com/google/firebase/example/friendlymeals/ui/groceryList/GroceryListViewModel.kt create mode 100644 app/src/main/res/drawable/ic_add.xml create mode 100644 app/src/main/res/drawable/ic_delete.xml delete mode 100644 gradle/gradle-daemon-jvm.properties diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 0a04664..7657536 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -10,12 +10,12 @@ plugins { android { namespace = "com.google.firebase.example.friendlymeals" - compileSdk = 37 + compileSdk = 36 defaultConfig { applicationId = "com.google.firebase.example.friendlymeals" minSdk = 26 - targetSdk = 36 + targetSdk = 35 versionCode = 1 versionName = "1.0" diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/MainActivity.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/MainActivity.kt index 30c476c..91ccc36 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/MainActivity.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/MainActivity.kt @@ -23,6 +23,8 @@ import androidx.navigation.navigation import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel import com.google.firebase.example.friendlymeals.ui.generate.GenerateRoute import com.google.firebase.example.friendlymeals.ui.generate.GenerateScreen +import com.google.firebase.example.friendlymeals.ui.groceryList.GroceryListRoute +import com.google.firebase.example.friendlymeals.ui.groceryList.GroceryListScreen import com.google.firebase.example.friendlymeals.ui.live.LiveAssistantRoute import com.google.firebase.example.friendlymeals.ui.live.LiveAssistantScreen import com.google.firebase.example.friendlymeals.ui.recipe.RecipeRoute @@ -140,6 +142,9 @@ class MainActivity : ComponentActivity() { } ) } + composable { + GroceryListScreen() + } } } } diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/DatabaseRemoteDataSource.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/DatabaseRemoteDataSource.kt index 59d0f90..69b6230 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/DatabaseRemoteDataSource.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/DatabaseRemoteDataSource.kt @@ -1,6 +1,7 @@ package com.google.firebase.example.friendlymeals.data.datasource import android.util.Log +import com.google.firebase.example.friendlymeals.data.model.GroceryItem import com.google.firebase.example.friendlymeals.data.model.Recipe import com.google.firebase.example.friendlymeals.data.model.Review import com.google.firebase.example.friendlymeals.data.model.Like @@ -19,6 +20,9 @@ import com.google.firebase.firestore.pipeline.Expression.Companion.documentId import com.google.firebase.firestore.pipeline.Expression.Companion.field import com.google.firebase.firestore.pipeline.Expression.Companion.variable import com.google.firebase.firestore.pipeline.SearchStage +import kotlinx.coroutines.channels.awaitClose +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.callbackFlow import kotlinx.coroutines.tasks.await import javax.inject.Inject import kotlin.collections.first @@ -251,12 +255,73 @@ class DatabaseRemoteDataSource @Inject constructor( } } + fun getGroceriesFlow(userId: String): Flow> = callbackFlow { + if (userId.isEmpty()) { + trySend(emptyList()) + close() + return@callbackFlow + } + + val listener = firestore.collection(GROCERIES_COLLECTION) + .whereEqualTo(USER_ID_FIELD, userId) + .addSnapshotListener { snapshot, error -> + if (error != null) { + close(error) + return@addSnapshotListener + } + + if (snapshot != null) { + val items = snapshot.documents.mapNotNull { doc -> + doc.toObject(GroceryItem::class.java)?.copy(id = doc.id) + } + trySend(items) + } + } + awaitClose { listener.remove() } + } + + suspend fun addGroceryItem(item: GroceryItem) { + val docRef = firestore.collection(GROCERIES_COLLECTION).document() + val itemWithId = item.copy(id = docRef.id) + docRef.set(itemWithId).await() + } + + suspend fun updateGroceryItemChecked(itemId: String, checked: Boolean) { + firestore.collection(GROCERIES_COLLECTION).document(itemId) + .update(CHECKED_FIELD, checked).await() + } + + suspend fun deleteGroceryItem(itemId: String) { + firestore.collection(GROCERIES_COLLECTION).document(itemId) + .delete().await() + } + + suspend fun addIngredientsToGroceries(userId: String, ingredients: List) { + if (userId.isEmpty() || ingredients.isEmpty()) return + + val batch = firestore.batch() + val collection = firestore.collection(GROCERIES_COLLECTION) + + for (ingredient in ingredients) { + val docRef = collection.document() + val item = GroceryItem( + id = docRef.id, + userId = userId, + name = ingredient, + checked = false + ) + batch.set(docRef, item) + } + batch.commit().await() + } + companion object { //Collections private const val USERS_COLLECTION = "users" private const val RECIPES_COLLECTION = "recipes" private const val LIKES_COLLECTION = "likes" private const val REVIEWS_SUBCOLLECTION = "reviews" + private const val GROCERIES_COLLECTION = "groceries" //Fields private const val RATING_FIELD = "rating" @@ -272,6 +337,8 @@ class DatabaseRemoteDataSource @Inject constructor( private const val INSTRUCTIONS_FIELD = "instructions" private const val INGREDIENTS_FIELD = "ingredients" private const val RECIPE_ID_FIELD = "recipeId" + private const val USER_ID_FIELD = "userId" + private const val CHECKED_FIELD = "checked" //Field aliases private const val AVG_RATING_ALIAS = "avg_rating" diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt index ce81864..d52f159 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt @@ -8,6 +8,9 @@ import com.google.firebase.ai.type.SpeechConfig import com.google.firebase.ai.type.Voice import com.google.firebase.ai.type.content import com.google.firebase.ai.type.liveGenerationConfig +import com.google.firebase.ai.type.Tool +import com.google.firebase.ai.type.FunctionDeclaration +import com.google.firebase.ai.type.Schema import com.google.firebase.example.friendlymeals.data.model.Recipe import com.google.firebase.remoteconfig.FirebaseRemoteConfig import javax.inject.Inject @@ -17,6 +20,16 @@ class LiveAIRemoteDataSource @Inject constructor( private val aiModel: FirebaseAI, private val remoteConfig: FirebaseRemoteConfig ) { + private val groceryListTool = Tool.functionDeclarations(listOf( + FunctionDeclaration( + name = ADD_INGREDIENTS_TOOL_NAME, + description = ADD_INGREDIENTS_TOOL_DESCRIPTION, + parameters = mapOf( + INGREDIENT_FIELD_NAME to Schema.string(INGREDIENT_FIELD_DESCRIPTION) + ) + ) + )) + @OptIn(PublicPreviewAPI::class) suspend fun setupLiveSession(recipe: Recipe): LiveSession? { val liveGenerationConfig = liveGenerationConfig { @@ -38,12 +51,14 @@ class LiveAIRemoteDataSource @Inject constructor( The user will stream real-time video of their cooking and ask questions like "Is this the expected texture of the recipe?". Confirm or deny accurately based on the recipe context and the video content. Be concise and helpful. + If the user asks you to add an ingredient or item to their grocery list or shopping list, call the addIngredientToGroceryList function. """.trimIndent() val liveModel = aiModel.liveModel( modelName = remoteConfig.getString(LIVE_MODEL_NAME_KEY), generationConfig = liveGenerationConfig, - systemInstruction = content { text(instructionText) } + systemInstruction = content { text(instructionText) }, + tools = listOf(groceryListTool) ) return try { @@ -57,6 +72,13 @@ class LiveAIRemoteDataSource @Inject constructor( //Live Model Config private const val LIVE_MODEL_VOICE = "CHARON" + //Tools config + private const val ADD_INGREDIENTS_TOOL_NAME = "addIngredientToGroceryList" + private const val ADD_INGREDIENTS_TOOL_DESCRIPTION = "Adds a specified ingredient to the " + + "user's grocery list in the database." + private const val INGREDIENT_FIELD_NAME = "ingredient" + private const val INGREDIENT_FIELD_DESCRIPTION = "The name of the ingredient to add." + //Remote Config Keys private const val LIVE_MODEL_NAME_KEY = "live_model_name" private const val LIVE_MODEL_PROMPT_KEY = "live_model_prompt" diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/data/model/GroceryItem.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/data/model/GroceryItem.kt new file mode 100644 index 0000000..b33e935 --- /dev/null +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/data/model/GroceryItem.kt @@ -0,0 +1,8 @@ +package com.google.firebase.example.friendlymeals.data.model + +data class GroceryItem( + val id: String = "", + val userId: String = "", + val name: String = "", + val checked: Boolean = false +) diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/data/repository/DatabaseRepository.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/data/repository/DatabaseRepository.kt index d064328..951ef93 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/data/repository/DatabaseRepository.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/data/repository/DatabaseRepository.kt @@ -7,6 +7,8 @@ import com.google.firebase.example.friendlymeals.data.model.Like import com.google.firebase.example.friendlymeals.data.model.User import com.google.firebase.example.friendlymeals.ui.recipeList.RecipeListItem import com.google.firebase.example.friendlymeals.ui.recipeList.filter.FilterOptions +import com.google.firebase.example.friendlymeals.data.model.GroceryItem +import kotlinx.coroutines.flow.Flow import javax.inject.Inject class DatabaseRepository @Inject constructor( @@ -58,4 +60,24 @@ class DatabaseRepository @Inject constructor( ): List { return databaseRemoteDataSource.getFilteredRecipes(filterOptions, userId) } + + fun getGroceriesFlow(userId: String): Flow> { + return databaseRemoteDataSource.getGroceriesFlow(userId) + } + + suspend fun addGroceryItem(item: GroceryItem) { + databaseRemoteDataSource.addGroceryItem(item) + } + + suspend fun updateGroceryItemChecked(itemId: String, checked: Boolean) { + databaseRemoteDataSource.updateGroceryItemChecked(itemId, checked) + } + + suspend fun deleteGroceryItem(itemId: String) { + databaseRemoteDataSource.deleteGroceryItem(itemId) + } + + suspend fun addIngredientsToGroceries(userId: String, ingredients: List) { + databaseRemoteDataSource.addIngredientsToGroceries(userId, ingredients) + } } \ No newline at end of file diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/groceryList/GroceryListScreen.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/groceryList/GroceryListScreen.kt new file mode 100644 index 0000000..fd56772 --- /dev/null +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/groceryList/GroceryListScreen.kt @@ -0,0 +1,276 @@ +package com.google.firebase.example.friendlymeals.ui.groceryList + +import androidx.compose.foundation.background +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.PaddingValues +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.foundation.text.KeyboardActions +import androidx.compose.foundation.text.KeyboardOptions +import androidx.compose.material3.Button +import androidx.compose.material3.ButtonDefaults +import androidx.compose.material3.Card +import androidx.compose.material3.CardDefaults +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.OutlinedTextFieldDefaults +import androidx.compose.material3.Scaffold +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.res.painterResource +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.input.ImeAction +import androidx.compose.ui.text.style.TextDecoration +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.google.firebase.example.friendlymeals.R +import com.google.firebase.example.friendlymeals.data.model.GroceryItem +import com.google.firebase.example.friendlymeals.ui.theme.BorderColor +import com.google.firebase.example.friendlymeals.ui.theme.FriendlyMealsTheme +import com.google.firebase.example.friendlymeals.ui.theme.LightTeal +import com.google.firebase.example.friendlymeals.ui.theme.Teal +import com.google.firebase.example.friendlymeals.ui.theme.TextColor +import kotlinx.serialization.Serializable + +@Serializable +object GroceryListRoute + +@Composable +fun GroceryListScreen( + viewModel: GroceryListViewModel = hiltViewModel() +) { + val groceries = viewModel.groceries.collectAsStateWithLifecycle() + + GroceryListScreenContent( + groceries = groceries.value, + onAddItem = viewModel::addItem, + onToggleItem = viewModel::toggleItem, + onDeleteItem = viewModel::deleteItem + ) +} + +@Composable +fun GroceryListScreenContent( + groceries: List, + onAddItem: (String) -> Unit = {}, + onToggleItem: (GroceryItem) -> Unit = {}, + onDeleteItem: (GroceryItem) -> Unit = {} +) { + var inputText by remember { mutableStateOf("") } + + fun handleAdd() { + if (inputText.isNotBlank()) { + onAddItem(inputText) + inputText = "" + } + } + + Scaffold( + topBar = { + Row( + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 24.dp, vertical = 16.dp), + verticalAlignment = Alignment.CenterVertically + ) { + Text( + text = stringResource(id = R.string.grocery_list_title), + fontSize = 28.sp, + fontWeight = FontWeight.Bold + ) + } + } + ) { innerPadding -> + Column( + modifier = Modifier + .fillMaxSize() + .padding(innerPadding) + .padding(horizontal = 24.dp) + ) { + Row( + modifier = Modifier + .fillMaxWidth() + .padding(bottom = 20.dp), + verticalAlignment = Alignment.CenterVertically + ) { + OutlinedTextField( + value = inputText, + onValueChange = { inputText = it }, + placeholder = { Text( + stringResource(R.string.add_grocery_item_hint), + color = Color.Gray + ) }, + shape = RoundedCornerShape(16.dp), + colors = OutlinedTextFieldDefaults.colors( + focusedBorderColor = Teal, + unfocusedBorderColor = BorderColor + ), + keyboardOptions = KeyboardOptions(imeAction = ImeAction.Done), + keyboardActions = KeyboardActions(onDone = { handleAdd() }), + modifier = Modifier + .weight(1f) + .height(54.dp) + ) + + Spacer(modifier = Modifier.width(12.dp)) + + Button( + onClick = { handleAdd() }, + colors = ButtonDefaults.buttonColors(containerColor = Teal), + shape = RoundedCornerShape(16.dp), + modifier = Modifier.size(54.dp), + contentPadding = PaddingValues(0.dp) + ) { + Icon( + painter = painterResource(R.drawable.ic_add), + contentDescription = stringResource(R.string.add_button), + tint = Color.White, + modifier = Modifier.size(24.dp) + ) + } + } + + if (groceries.isEmpty()) { + Box( + modifier = Modifier.fillMaxSize(), + contentAlignment = Alignment.Center + ) { + Text( + text = stringResource(R.string.grocery_list_empty_message), + fontSize = 16.sp, + color = Color.Gray + ) + } + } else { + LazyColumn( + verticalArrangement = Arrangement.spacedBy(12.dp), + contentPadding = PaddingValues(bottom = 24.dp), + modifier = Modifier.fillMaxSize() + ) { + items(items = groceries, key = { it.id }) { item -> + GroceryCard( + item = item, + onToggle = { onToggleItem(item) }, + onDelete = { onDeleteItem(item) } + ) + } + } + } + } + } +} + +@Composable +fun GroceryCard( + item: GroceryItem, + onToggle: () -> Unit, + onDelete: () -> Unit +) { + Card( + shape = RoundedCornerShape(20.dp), + colors = CardDefaults.cardColors(containerColor = Color.White), + elevation = CardDefaults.cardElevation(defaultElevation = if (item.checked) 0.dp else 2.dp), + modifier = Modifier + .fillMaxWidth() + .clip(RoundedCornerShape(20.dp)) + .clickable { onToggle() } + ) { + Row( + modifier = Modifier + .padding(horizontal = 16.dp, vertical = 14.dp) + .fillMaxWidth(), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.SpaceBetween + ) { + Row( + verticalAlignment = Alignment.CenterVertically, + modifier = Modifier.weight(1f) + ) { + Box( + modifier = Modifier + .size(28.dp) + .background(if (item.checked) Teal else LightTeal, CircleShape), + contentAlignment = Alignment.Center + ) { + if (item.checked) { + Icon( + painter = painterResource(R.drawable.ic_check), + contentDescription = null, + tint = Color.White, + modifier = Modifier.size(16.dp) + ) + } else { + Box( + modifier = Modifier + .size(14.dp) + .background(Color.White, CircleShape) + ) + } + } + + Spacer(modifier = Modifier.width(16.dp)) + + Text( + text = item.name, + fontSize = 16.sp, + fontWeight = FontWeight.Medium, + color = if (item.checked) Color.Gray else TextColor, + textDecoration = if (item.checked) TextDecoration.LineThrough else TextDecoration.None, + lineHeight = 22.sp + ) + } + + IconButton( + onClick = { onDelete() }, + modifier = Modifier.size(36.dp) + ) { + Icon( + painter = painterResource(R.drawable.ic_delete), + contentDescription = stringResource(R.string.delete_button_content_description), + tint = Color.Gray, + modifier = Modifier.size(20.dp) + ) + } + } + } +} + +@Preview +@Composable +fun GroceryListScreenPreview() { + FriendlyMealsTheme { + GroceryListScreenContent( + groceries = listOf( + GroceryItem(id = "1", name = "2 cloves garlic", checked = true), + GroceryItem(id = "2", name = "400g canned tomatoes", checked = false), + GroceryItem(id = "3", name = "Fresh basil", checked = false) + ) + ) + } +} diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/groceryList/GroceryListViewModel.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/groceryList/GroceryListViewModel.kt new file mode 100644 index 0000000..48f7d9f --- /dev/null +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/groceryList/GroceryListViewModel.kt @@ -0,0 +1,64 @@ +package com.google.firebase.example.friendlymeals.ui.groceryList + +import com.google.firebase.example.friendlymeals.MainViewModel +import com.google.firebase.example.friendlymeals.data.model.GroceryItem +import com.google.firebase.example.friendlymeals.data.repository.AuthRepository +import com.google.firebase.example.friendlymeals.data.repository.DatabaseRepository +import dagger.hilt.android.lifecycle.HiltViewModel +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.catch +import javax.inject.Inject + +@HiltViewModel +class GroceryListViewModel @Inject constructor( + private val authRepository: AuthRepository, + private val databaseRepository: DatabaseRepository +) : MainViewModel() { + private val _groceries = MutableStateFlow>(emptyList()) + val groceries: StateFlow> = _groceries.asStateFlow() + + val userId: String get() = authRepository.currentUser?.uid.orEmpty() + + init { + loadGroceries() + } + + private fun loadGroceries() { + if (userId.isEmpty()) return + + launchCatching { + databaseRepository.getGroceriesFlow(userId) + .catch { _ -> } + .collect { items -> + _groceries.value = items + } + } + } + + fun toggleItem(item: GroceryItem) { + launchCatching { + databaseRepository.updateGroceryItemChecked(item.id, !item.checked) + } + } + + fun addItem(name: String) { + if (name.isBlank() || userId.isEmpty()) return + + launchCatching { + val item = GroceryItem( + userId = userId, + name = name.trim(), + checked = false + ) + databaseRepository.addGroceryItem(item) + } + } + + fun deleteItem(item: GroceryItem) { + launchCatching { + databaseRepository.deleteGroceryItem(item.id) + } + } +} diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt index 98d5b54..0098fcc 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt @@ -10,7 +10,9 @@ import com.google.firebase.ai.type.InlineData import com.google.firebase.ai.type.LiveSession import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.example.friendlymeals.MainViewModel +import com.google.firebase.example.friendlymeals.data.model.GroceryItem import com.google.firebase.example.friendlymeals.data.model.Recipe +import com.google.firebase.example.friendlymeals.data.repository.AuthRepository import com.google.firebase.example.friendlymeals.data.repository.DatabaseRepository import com.google.firebase.example.friendlymeals.data.repository.LiveAIRepository import com.google.firebase.example.friendlymeals.ui.live.LiveAssistantUiState.Loading @@ -19,6 +21,7 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive import java.io.ByteArrayOutputStream import javax.inject.Inject @@ -26,6 +29,7 @@ import javax.inject.Inject @OptIn(PublicPreviewAPI::class) class LiveAssistantViewModel @Inject constructor( savedStateHandle: SavedStateHandle, + private val authRepository: AuthRepository, private val databaseRepository: DatabaseRepository, private val liveAIRepository: LiveAIRepository ) : MainViewModel() { @@ -69,6 +73,36 @@ class LiveAssistantViewModel @Inject constructor( } private fun handler(functionCall: FunctionCallPart): FunctionResponsePart { + if (functionCall.name == ADD_INGREDIENTS_TOOL_NAME) { + val ingredient = functionCall.args[INGREDIENT_FIELD_NAME] + val ingredientName = when (ingredient) { + is JsonPrimitive -> ingredient.content + else -> ingredient?.toString() + }?.trim()?.removeSurrounding("\"") + + if (!ingredientName.isNullOrBlank()) { + val userId = authRepository.currentUser?.uid.orEmpty() + if (userId.isNotEmpty()) { + launchCatching { + val item = GroceryItem( + userId = userId, + name = ingredientName, + checked = false + ) + databaseRepository.addGroceryItem(item) + } + } + } + + return FunctionResponsePart( + functionCall.name, + JsonObject(mapOf( + "result" to JsonPrimitive("Successfully added $ingredientName to grocery list") + )), + functionCall.id + ) + } + return FunctionResponsePart(functionCall.name, JsonObject(emptyMap()), functionCall.id) } @@ -109,8 +143,13 @@ class LiveAssistantViewModel @Inject constructor( } companion object { + //Connection config private const val MIME_TYPE = "image/jpeg" private const val RECIPE_ERROR = "Failed to load recipe" private const val CONNECTION_ERROR = "Failed to connect to live assistant" + + //Tool config + private const val INGREDIENT_FIELD_NAME = "ingredient" + private const val ADD_INGREDIENTS_TOOL_NAME = "addIngredientToGroceryList" } } diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeScreen.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeScreen.kt index 1e91c5b..b1cec0b 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeScreen.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeScreen.kt @@ -67,6 +67,8 @@ fun RecipeScreen( navigateToLiveAssistant: (String) -> Unit ) { val recipeViewState = viewModel.recipeViewState.collectAsStateWithLifecycle() + val groceryListToast = stringResource(R.string.added_to_grocery_list_toast) + val context = LocalContext.current RecipeScreenContent( navigateBack = navigateBack, @@ -75,6 +77,11 @@ fun RecipeScreen( recipeViewState = recipeViewState.value, onLiveAssistantClick = { navigateToLiveAssistant(recipeViewState.value.recipeId) + }, + onAddIngredientsToGrocery = { + viewModel.addIngredientsToGroceryList(recipeViewState.value.recipe.ingredients) { + Toast.makeText(context, groceryListToast, Toast.LENGTH_SHORT).show() + } } ) } @@ -85,7 +92,8 @@ fun RecipeScreenContent( toggleFavorite: () -> Unit = {}, leaveReview: (Int) -> Unit = {}, recipeViewState: RecipeViewState, - onLiveAssistantClick: () -> Unit = {} + onLiveAssistantClick: () -> Unit = {}, + onAddIngredientsToGrocery: () -> Unit = {} ) { val context = LocalContext.current val multiplePermissionsLauncher = rememberLauncherForActivityResult( @@ -282,6 +290,38 @@ fun RecipeScreenContent( recipeViewState.recipe.ingredients.forEach { IngredientRow(it) } + + Spacer(modifier = Modifier.height(20.dp)) + + Button( + onClick = { onAddIngredientsToGrocery() }, + colors = ButtonDefaults.buttonColors( + containerColor = LightTeal, + contentColor = Teal + ), + shape = RoundedCornerShape(12.dp), + modifier = Modifier.fillMaxWidth().height(48.dp) + ) { + Row( + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.Center + ) { + Icon( + painter = painterResource(R.drawable.ic_check), + contentDescription = null, + tint = Teal, + modifier = Modifier.size(18.dp) + ) + + Spacer(modifier = Modifier.width(8.dp)) + + Text( + text = stringResource(R.string.add_to_grocery_list_button), + fontSize = 15.sp, + fontWeight = FontWeight.Bold + ) + } + } } } diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeViewModel.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeViewModel.kt index 1233a58..17d5d4d 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeViewModel.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeViewModel.kt @@ -85,4 +85,13 @@ class RecipeViewModel @Inject constructor( ) } } + + fun addIngredientsToGroceryList(ingredients: List, onSuccess: () -> Unit) { + if (userId.isEmpty() || ingredients.isEmpty()) return + + launchCatching { + databaseRepository.addIngredientsToGroceries(userId, ingredients) + onSuccess() + } + } } \ No newline at end of file diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/shared/BottomNavBar.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/shared/BottomNavBar.kt index 82b2d15..c03d62d 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/shared/BottomNavBar.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/shared/BottomNavBar.kt @@ -15,6 +15,7 @@ import androidx.compose.ui.res.painterResource import androidx.compose.ui.res.stringResource import com.google.firebase.example.friendlymeals.R import com.google.firebase.example.friendlymeals.ui.generate.GenerateRoute +import com.google.firebase.example.friendlymeals.ui.groceryList.GroceryListRoute import com.google.firebase.example.friendlymeals.ui.recipeList.RecipeListRoute import com.google.firebase.example.friendlymeals.ui.scanMeal.ScanMealRoute import com.google.firebase.example.friendlymeals.ui.theme.Teal @@ -23,6 +24,7 @@ sealed class BottomNavItem(val route: Any, val icon: Int, val label: Int) { object ScanMeal : BottomNavItem(ScanMealRoute, R.drawable.ic_camera, R.string.nav_bar_scan_meal) object Generate : BottomNavItem(GenerateRoute, R.drawable.ic_generate, R.string.nav_bar_generate) object RecipeList : BottomNavItem(RecipeListRoute, R.drawable.ic_dine, R.string.nav_bar_recipe_list) + object GroceryList : BottomNavItem(GroceryListRoute, R.drawable.ic_check, R.string.nav_bar_grocery_list) } @Composable @@ -32,7 +34,8 @@ fun BottomNavBar(navigateTo: (Any) -> Unit) { val items = listOf( BottomNavItem.ScanMeal, BottomNavItem.Generate, - BottomNavItem.RecipeList + BottomNavItem.RecipeList, + BottomNavItem.GroceryList ) NavigationBar { diff --git a/app/src/main/res/drawable/ic_add.xml b/app/src/main/res/drawable/ic_add.xml new file mode 100644 index 0000000..b9aa461 --- /dev/null +++ b/app/src/main/res/drawable/ic_add.xml @@ -0,0 +1,9 @@ + + + diff --git a/app/src/main/res/drawable/ic_delete.xml b/app/src/main/res/drawable/ic_delete.xml new file mode 100644 index 0000000..494ece6 --- /dev/null +++ b/app/src/main/res/drawable/ic_delete.xml @@ -0,0 +1,9 @@ + + + diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index b0f6e8f..c9ca124 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -56,4 +56,12 @@ Error loading camera Live Cooking Assistant Point your camera at your cooking and ask questions + Grocery List + Grocery List + Add an ingredient… + Add + Delete + Add ingredients to Grocery List + Added to Grocery List! + Your grocery list is empty \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 61e7198..670e9be 100644 --- a/gradle.properties +++ b/gradle.properties @@ -29,4 +29,5 @@ android.uniquePackageNames=false android.dependency.useConstraints=true android.r8.strictFullModeForKeepRules=false android.r8.optimizedResourceShrinking=false -android.disallowKotlinSourceSets=false \ No newline at end of file +android.disallowKotlinSourceSets=false +org.gradle.java.home=/Applications/Android Studio.app/Contents/jbr/Contents/Home \ No newline at end of file diff --git a/gradle/gradle-daemon-jvm.properties b/gradle/gradle-daemon-jvm.properties deleted file mode 100644 index 5c34300..0000000 --- a/gradle/gradle-daemon-jvm.properties +++ /dev/null @@ -1,13 +0,0 @@ -#This file is generated by updateDaemonJvm -toolchainUrl.FREE_BSD.AARCH64=https\://api.foojay.io/disco/v3.0/ids/56a19bc915b9ba2eb62ba7554c61b919/redirect -toolchainUrl.FREE_BSD.X86_64=https\://api.foojay.io/disco/v3.0/ids/398ffe3949748bfb1d5636f023d228fd/redirect -toolchainUrl.LINUX.AARCH64=https\://api.foojay.io/disco/v3.0/ids/56a19bc915b9ba2eb62ba7554c61b919/redirect -toolchainUrl.LINUX.X86_64=https\://api.foojay.io/disco/v3.0/ids/398ffe3949748bfb1d5636f023d228fd/redirect -toolchainUrl.MAC_OS.AARCH64=https\://api.foojay.io/disco/v3.0/ids/e99bae143b75f9a10ead10248f02055e/redirect -toolchainUrl.MAC_OS.X86_64=https\://api.foojay.io/disco/v3.0/ids/04e088f8677de3b384108493cc9481d0/redirect -toolchainUrl.UNIX.AARCH64=https\://api.foojay.io/disco/v3.0/ids/56a19bc915b9ba2eb62ba7554c61b919/redirect -toolchainUrl.UNIX.X86_64=https\://api.foojay.io/disco/v3.0/ids/398ffe3949748bfb1d5636f023d228fd/redirect -toolchainUrl.WINDOWS.AARCH64=https\://api.foojay.io/disco/v3.0/ids/e55dccbfe27cb97945148c61a39c89c5/redirect -toolchainUrl.WINDOWS.X86_64=https\://api.foojay.io/disco/v3.0/ids/dbd05c4936d573642f94cd149e1356c8/redirect -toolchainVendor=JETBRAINS -toolchainVersion=21 From a52b52e5962e314efb40876672bb9e448f322824 Mon Sep 17 00:00:00 2001 From: Marina Coelho Date: Mon, 18 May 2026 07:55:45 +0100 Subject: [PATCH 6/8] Migrating prompt to Firebase Remote Config --- .../data/datasource/LiveAIRemoteDataSource.kt | 28 ++++++++----------- .../main/res/xml/remote_config_defaults.xml | 14 +++++++++- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt index d52f159..417d0d1 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt @@ -37,22 +37,8 @@ class LiveAIRemoteDataSource @Inject constructor( responseModality = ResponseModality.AUDIO } - // This is temporary, prompt will be moved to Remote Config soon - val instructionText = """ - You are a helpful live cooking assistant. The user is currently preparing the following recipe: - Title: ${recipe.title} - Prep time: ${recipe.prepTime}, Cook time: ${recipe.cookTime}, Servings: ${recipe.servings} - - Ingredients: - ${recipe.ingredients.joinToString("\n")} - - Instructions: - ${recipe.instructions} - - The user will stream real-time video of their cooking and ask questions like "Is this the expected texture of the recipe?". - Confirm or deny accurately based on the recipe context and the video content. Be concise and helpful. - If the user asks you to add an ingredient or item to their grocery list or shopping list, call the addIngredientToGroceryList function. - """.trimIndent() + val promptTemplate = remoteConfig.getString(LIVE_MODEL_PROMPT_KEY) + val instructionText = formatInstructionPrompt(promptTemplate, recipe) val liveModel = aiModel.liveModel( modelName = remoteConfig.getString(LIVE_MODEL_NAME_KEY), @@ -68,6 +54,16 @@ class LiveAIRemoteDataSource @Inject constructor( } } + private fun formatInstructionPrompt(template: String, recipe: Recipe): String { + return template + .replace("{{title}}", recipe.title) + .replace("{{prepTime}}", recipe.prepTime) + .replace("{{cookTime}}", recipe.cookTime) + .replace("{{servings}}", recipe.servings) + .replace("{{ingredients}}", recipe.ingredients.joinToString("\n")) + .replace("{{instructions}}", recipe.instructions) + } + companion object { //Live Model Config private const val LIVE_MODEL_VOICE = "CHARON" diff --git a/app/src/main/res/xml/remote_config_defaults.xml b/app/src/main/res/xml/remote_config_defaults.xml index 6a45bcb..91c3b0f 100644 --- a/app/src/main/res/xml/remote_config_defaults.xml +++ b/app/src/main/res/xml/remote_config_defaults.xml @@ -25,6 +25,18 @@ live_model_prompt - tbd + You are a helpful live cooking assistant. The user is currently preparing the following recipe: +Title: {{title}} +Prep time: {{prepTime}}, Cook time: {{cookTime}}, Servings: {{servings}} + +Ingredients: +{{ingredients}} + +Instructions: +{{instructions}} + +The user will stream real-time video of their cooking and ask questions like "Is this the expected texture of the recipe?". +Confirm or deny accurately based on the recipe context and the video content. Be concise and helpful. +If the user asks you to add an ingredient or item to their grocery list or shopping list, call the addIngredientToGroceryList function. \ No newline at end of file From 7ba5e20d74b249690cd84b1e59944c43721c3816 Mon Sep 17 00:00:00 2001 From: Marina Coelho Date: Mon, 18 May 2026 14:07:22 +0100 Subject: [PATCH 7/8] Remove unused libraries, use Dispatchers IO when compressin bitmap --- app/build.gradle.kts | 4 +--- .../firebase/example/friendlymeals/MainViewModel.kt | 9 +++++++++ .../friendlymeals/ui/live/LiveAssistantScreen.kt | 4 +++- .../friendlymeals/ui/live/LiveAssistantViewModel.kt | 2 +- .../example/friendlymeals/ui/recipe/RecipeScreen.kt | 2 +- gradle/libs.versions.toml | 13 ++++--------- 6 files changed, 19 insertions(+), 15 deletions(-) diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 7657536..2a94eae 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -10,7 +10,7 @@ plugins { android { namespace = "com.google.firebase.example.friendlymeals" - compileSdk = 36 + compileSdk = 37 defaultConfig { applicationId = "com.google.firebase.example.friendlymeals" @@ -58,8 +58,6 @@ dependencies { implementation(libs.androidx.camera.camera2) implementation(libs.androidx.camera.lifecycle) implementation(libs.androidx.camera.view) - implementation(libs.camera.view) - implementation(libs.guava) implementation(libs.kotlinx.serialization.json) implementation(libs.hilt.android) ksp(libs.hilt.android.compiler) diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/MainViewModel.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/MainViewModel.kt index c528afd..98cf927 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/MainViewModel.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/MainViewModel.kt @@ -5,6 +5,7 @@ import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import kotlinx.coroutines.CoroutineExceptionHandler import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch open class MainViewModel : ViewModel() { @@ -15,4 +16,12 @@ open class MainViewModel : ViewModel() { }, block = block ) + + fun launchCatchingIO(block: suspend CoroutineScope.() -> Unit) = + viewModelScope.launch( + Dispatchers.IO + CoroutineExceptionHandler { _, throwable -> + Log.e("MainViewModel", throwable.message ?: "Unknown error") + }, + block = block + ) } \ No newline at end of file diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantScreen.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantScreen.kt index 5f16da9..32f4dab 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantScreen.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantScreen.kt @@ -36,6 +36,8 @@ import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp import androidx.compose.ui.viewinterop.AndroidView +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.asExecutor import androidx.core.content.ContextCompat.getMainExecutor import androidx.hilt.lifecycle.viewmodel.compose.hiltViewModel import androidx.lifecycle.compose.LocalLifecycleOwner @@ -155,7 +157,7 @@ fun LiveAssistantScreenContent( .setBackpressureStrategy(STRATEGY_KEEP_ONLY_LATEST) .build() .also { analysis -> - analysis.setAnalyzer(getMainExecutor(context)) { imageProxy -> + analysis.setAnalyzer(Dispatchers.IO.asExecutor()) { imageProxy -> val bitmap = imageProxy.toBitmap() sendVideoFrame(bitmap) imageProxy.close() diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt index 0098fcc..c15b501 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt @@ -129,7 +129,7 @@ class LiveAssistantViewModel @Inject constructor( if (currentTime - lastFrameTime < 1000) return lastFrameTime = currentTime - launchCatching { + launchCatchingIO { val outputStream = ByteArrayOutputStream() bitmap.compress(Bitmap.CompressFormat.JPEG, 80, outputStream) val jpegBytes = outputStream.toByteArray() diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeScreen.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeScreen.kt index b1cec0b..f547bca 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeScreen.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/recipe/RecipeScreen.kt @@ -230,7 +230,7 @@ fun RecipeScreenContent( ) Spacer(modifier = Modifier.width(8.dp)) Text( - text = "Live Cooking Assistant", + text = stringResource(R.string.live_assistant_title), color = Color.White, fontSize = 16.sp, fontWeight = FontWeight.Bold diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e93ab8e..7a3479a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,6 +1,5 @@ [versions] agp = "9.2.1" -camerax = "1.4.1" coilCompose = "2.7.0" exifinterface = "1.4.2" firebaseAi = "17.12.0" @@ -16,7 +15,6 @@ activityCompose = "1.13.0" composeBom = "2026.05.00" googleServices = "4.4.4" googleHilt = "2.59.2" -guava = "33.4.0-android" googleKotlinKsp = "2.3.2" hiltAndroidCompiler = "2.59.2" coreSplashscreen = "1.2.0" @@ -61,16 +59,13 @@ androidx-navigation-compose = { module = "androidx.navigation:navigation-compose androidx-constraintlayout-compose = { module = "androidx.constraintlayout:constraintlayout-compose", version.ref = "constraintlayoutCompose" } kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" } richtext-commonmark = { module = "com.halilibo.compose-richtext:richtext-commonmark", version.ref = "richtextCommonmark" } -androidx-camera-core = { group = "androidx.camera", name = "camera-core", version.ref = "camerax" } -androidx-camera-camera2 = { group = "androidx.camera", name = "camera-camera2", version.ref = "camerax" } -androidx-camera-lifecycle = { group = "androidx.camera", name = "camera-lifecycle", version.ref = "camerax" } -androidx-camera-view = { group = "androidx.camera", name = "camera-view", version.ref = "camerax" } -guava = { module = "com.google.guava:guava", version.ref = "guava" } -camera-view = { group = "androidx.camera", name = "camera-view", version.ref = "cameraView" } +androidx-camera-core = { group = "androidx.camera", name = "camera-core", version.ref = "cameraView" } +androidx-camera-camera2 = { group = "androidx.camera", name = "camera-camera2", version.ref = "cameraView" } +androidx-camera-lifecycle = { group = "androidx.camera", name = "camera-lifecycle", version.ref = "cameraView" } +androidx-camera-view = { group = "androidx.camera", name = "camera-view", version.ref = "cameraView" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" } -kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" } kotlin-compose = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" } google-services = { id = "com.google.gms.google-services", version.ref = "googleServices" } google-hilt = { id = "com.google.dagger.hilt.android", version.ref = "googleHilt" } From a25bc02e82c38b7023e4103959d39a7200215096 Mon Sep 17 00:00:00 2001 From: Marina Coelho Date: Mon, 18 May 2026 14:55:32 +0100 Subject: [PATCH 8/8] Improve database function, upgrade target sdk --- app/build.gradle.kts | 2 +- .../datasource/DatabaseRemoteDataSource.kt | 29 +++++++------------ 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 2a94eae..04a0a2a 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -15,7 +15,7 @@ android { defaultConfig { applicationId = "com.google.firebase.example.friendlymeals" minSdk = 26 - targetSdk = 35 + targetSdk = 37 versionCode = 1 versionName = "1.0" diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/DatabaseRemoteDataSource.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/DatabaseRemoteDataSource.kt index 69b6230..dde709b 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/DatabaseRemoteDataSource.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/DatabaseRemoteDataSource.kt @@ -20,9 +20,10 @@ import com.google.firebase.firestore.pipeline.Expression.Companion.documentId import com.google.firebase.firestore.pipeline.Expression.Companion.field import com.google.firebase.firestore.pipeline.Expression.Companion.variable import com.google.firebase.firestore.pipeline.SearchStage -import kotlinx.coroutines.channels.awaitClose +import com.google.firebase.firestore.snapshots import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.callbackFlow +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.flow.mapNotNull import kotlinx.coroutines.tasks.await import javax.inject.Inject import kotlin.collections.first @@ -255,29 +256,19 @@ class DatabaseRemoteDataSource @Inject constructor( } } - fun getGroceriesFlow(userId: String): Flow> = callbackFlow { + fun getGroceriesFlow(userId: String): Flow> { if (userId.isEmpty()) { - trySend(emptyList()) - close() - return@callbackFlow + return flowOf(emptyList()) } - val listener = firestore.collection(GROCERIES_COLLECTION) + return firestore.collection(GROCERIES_COLLECTION) .whereEqualTo(USER_ID_FIELD, userId) - .addSnapshotListener { snapshot, error -> - if (error != null) { - close(error) - return@addSnapshotListener - } - - if (snapshot != null) { - val items = snapshot.documents.mapNotNull { doc -> - doc.toObject(GroceryItem::class.java)?.copy(id = doc.id) - } - trySend(items) + .snapshots() + .mapNotNull { snapshot -> + snapshot.documents.mapNotNull { doc -> + doc.toObject(GroceryItem::class.java)?.copy(id = doc.id) } } - awaitClose { listener.remove() } } suspend fun addGroceryItem(item: GroceryItem) {