From ada2ad8d962fa95f46c220e4e3f70d43906fbe98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Wed, 19 Nov 2025 19:43:44 -0500 Subject: [PATCH 1/9] Index strings https://github.com/mtransitapps/commons-java/pull/10 --- .../provider/GTFSProviderDbHelper.java | 30 +++++++++++++++++-- src/main/res-current/raw/current_gtfs_strings | 0 src/main/res-next/raw/next_gtfs_strings | 0 src/main/res/raw/gtfs_strings | 0 4 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 src/main/res-current/raw/current_gtfs_strings create mode 100644 src/main/res-next/raw/next_gtfs_strings create mode 100644 src/main/res/raw/gtfs_strings diff --git a/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java b/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java index d0fbd930..f04fe7ad 100644 --- a/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java +++ b/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java @@ -38,6 +38,13 @@ public String getLogTag() { */ public static final String DB_NAME = "gtfs_rts.db"; // do not change to avoid breaking compat w/ old modules + static final String T_STRINGS = GTFSCommons.T_STRINGS; + static final String T_STRINGS_K_ID = GTFSCommons.T_STRINGS_K_ID; + static final String T_STRINGS_K_STRING = GTFSCommons.T_STRINGS_K_STRING; + private static final String T_STRINGS_SQL_CREATE = GTFSCommons.getT_STRINGS_SQL_CREATE(); + private static final String T_STRINGS_SQL_INSERT = GTFSCommons.getT_STRINGS_SQL_INSERT(); + private static final String T_STRINGS_SQL_DROP = GTFSCommons.getT_STRINGS_SQL_DROP(); + static final String T_ROUTE = GTFSCommons.T_ROUTE; static final String T_ROUTE_K_ID = GTFSCommons.T_ROUTE_K_ID; static final String T_ROUTE_K_SHORT_NAME = GTFSCommons.T_ROUTE_K_SHORT_NAME; @@ -151,7 +158,7 @@ public boolean isDbExist(@NonNull Context context) { private void initAllDbTables(@NonNull SQLiteDatabase db, boolean upgrade) { MTLog.i(this, "Data: deploying DB..."); int nId = TimeUtils.currentTimeSec(); - int nbTotalOperations = 7; + int nbTotalOperations = 8; final NotificationManagerCompat nm = NotificationManagerCompat.from(this.context); final boolean notifEnabled = nm.areNotificationsEnabled(); final NotificationCompat.Builder nb; @@ -194,10 +201,14 @@ private void initAllDbTables(@NonNull SQLiteDatabase db, boolean upgrade) { if (notifEnabled) { NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 6); } + initDbTableWithRetry(db, T_STRINGS, T_STRINGS_SQL_CREATE, T_STRINGS_SQL_INSERT, T_STRINGS_SQL_DROP, getServiceDatesFiles()); + if (notifEnabled) { + NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 7); + } db.execSQL(T_ROUTE_DIRECTION_STOP_STATUS_SQL_CREATE); if (notifEnabled) { nb.setSmallIcon(android.R.drawable.stat_notify_sync_noanim); // - NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 7); + NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 8); nm.cancel(nId); } MTLog.i(this, "Data: deploying DB... DONE"); @@ -272,6 +283,21 @@ private int[] getServiceIdsFiles() { } } + /** + * Override if multiple {@link GTFSProviderDbHelper} implementations in same app. + */ + private int[] getStringsFiles() { + if (GTFSCurrentNextProvider.hasCurrentData(context)) { + if (GTFSCurrentNextProvider.isNextData(context)) { + return new int[]{R.raw.next_gtfs_strings}; + } else { // CURRENT = default + return new int[]{R.raw.current_gtfs_strings}; + } + } else { + return new int[]{R.raw.gtfs_strings}; + } + } + /** * Override if multiple {@link GTFSProviderDbHelper} implementations in same app. */ diff --git a/src/main/res-current/raw/current_gtfs_strings b/src/main/res-current/raw/current_gtfs_strings new file mode 100644 index 00000000..e69de29b diff --git a/src/main/res-next/raw/next_gtfs_strings b/src/main/res-next/raw/next_gtfs_strings new file mode 100644 index 00000000..e69de29b diff --git a/src/main/res/raw/gtfs_strings b/src/main/res/raw/gtfs_strings new file mode 100644 index 00000000..e69de29b From cde4bfcf48d2f4e84a6eb3275b914a193523c7a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Wed, 19 Nov 2025 21:29:56 -0500 Subject: [PATCH 2/9] WIP --- .../android/commons/data/Schedule.java | 12 ++++ .../commons/provider/GTFSPOIProvider.java | 16 +++--- .../provider/GTFSProviderDbHelper.java | 2 +- .../commons/provider/GTFSRDSProvider.java | 8 +-- .../GTFSScheduleTimestampsProvider.java | 4 +- .../commons/provider/GTFSStatusProvider.java | 21 ++++--- .../commons/provider/GTFSStringsUtils.kt | 55 +++++++++++++++++++ 7 files changed, 95 insertions(+), 23 deletions(-) create mode 100644 src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt diff --git a/src/main/java/org/mtransit/android/commons/data/Schedule.java b/src/main/java/org/mtransit/android/commons/data/Schedule.java index 99881f32..6f1f9b18 100644 --- a/src/main/java/org/mtransit/android/commons/data/Schedule.java +++ b/src/main/java/org/mtransit/android/commons/data/Schedule.java @@ -4,6 +4,7 @@ import android.database.Cursor; import android.text.TextUtils; +import androidx.annotation.Discouraged; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; @@ -464,6 +465,17 @@ public void setResetHeadsign() { this.headsignValue = null; } + @Discouraged(message = "should call getHeadsign()") + @Nullable + public String getHeadsignValue() { + return headsignValue; + } + + @Discouraged(message = "should call setHeadsign()") + public void setHeadsignValue(@Nullable String headsignValue) { + this.headsignValue = headsignValue; + } + public boolean hasHeadsign() { if (this.headsignType == Direction.HEADSIGN_TYPE_NO_PICKUP) { return true; diff --git a/src/main/java/org/mtransit/android/commons/provider/GTFSPOIProvider.java b/src/main/java/org/mtransit/android/commons/provider/GTFSPOIProvider.java index 1487920a..a87b0a93 100644 --- a/src/main/java/org/mtransit/android/commons/provider/GTFSPOIProvider.java +++ b/src/main/java/org/mtransit/android/commons/provider/GTFSPOIProvider.java @@ -23,12 +23,12 @@ @SuppressWarnings("WeakerAccess") public class GTFSPOIProvider implements MTLog.Loggable { - private static final String TAG = GTFSPOIProvider.class.getSimpleName(); + private static final String LOG_TAG = GTFSPOIProvider.class.getSimpleName(); @NonNull @Override public String getLogTag() { - return TAG; + return LOG_TAG; } public static void append(@NonNull UriMatcher uriMatcher, @NonNull String authority) { @@ -125,15 +125,15 @@ public static Cursor getPOIFromDB(@NonNull GTFSProvider provider, @Nullable POIP poiProjection = ArrayUtils.addAllNonNull(poiProjection, new String[]{POIProviderContract.Columns.T_POI_K_SCORE_META_OPT}); } if (poiProjection.length != poiProjectionMap.size()) { - MTLog.w(TAG, "getPOIFromDB() > different projection sizes (%d VS %d)", poiProjection.length, poiProjectionMap.size()); + MTLog.w(LOG_TAG, "getPOIFromDB() > different projection sizes (%d VS %d)", poiProjection.length, poiProjectionMap.size()); if (Constants.DEBUG) { - MTLog.w(TAG, "getPOIFromDB() > poiProjection: %d", poiProjection.length); + MTLog.w(LOG_TAG, "getPOIFromDB() > poiProjection: %d", poiProjection.length); for (String string : poiProjection) { - MTLog.w(TAG, "getPOIFromDB() > poiProjection: - %s.", string); + MTLog.w(LOG_TAG, "getPOIFromDB() > poiProjection: - %s.", string); } - MTLog.w(TAG, "getPOIFromDB() > poiProjectionMap: %d", poiProjectionMap.size()); + MTLog.w(LOG_TAG, "getPOIFromDB() > poiProjectionMap: %d", poiProjectionMap.size()); for (Map.Entry keyValue : poiProjectionMap.entrySet()) { - MTLog.w(TAG, "getPOIFromDB() > poiProjectionMap: - %s: %s.", keyValue.getKey(), keyValue.getValue()); + MTLog.w(LOG_TAG, "getPOIFromDB() > poiProjectionMap: - %s: %s.", keyValue.getKey(), keyValue.getValue()); } } } @@ -147,7 +147,7 @@ public static Cursor getPOIFromDB(@NonNull GTFSProvider provider, @Nullable POIP } return qb.query(provider.getReadDB(), poiProjection, selection, null, groupBy, null, sortOrder, null); } catch (Exception e) { - MTLog.w(TAG, e, "Error while loading POIs '%s'!", poiFilter); + MTLog.w(LOG_TAG, e, "Error while loading POIs '%s'!", poiFilter); return null; } } diff --git a/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java b/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java index f04fe7ad..202492d7 100644 --- a/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java +++ b/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java @@ -201,7 +201,7 @@ private void initAllDbTables(@NonNull SQLiteDatabase db, boolean upgrade) { if (notifEnabled) { NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 6); } - initDbTableWithRetry(db, T_STRINGS, T_STRINGS_SQL_CREATE, T_STRINGS_SQL_INSERT, T_STRINGS_SQL_DROP, getServiceDatesFiles()); + initDbTableWithRetry(db, T_STRINGS, T_STRINGS_SQL_CREATE, T_STRINGS_SQL_INSERT, T_STRINGS_SQL_DROP, getStringsFiles()); if (notifEnabled) { NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 7); } diff --git a/src/main/java/org/mtransit/android/commons/provider/GTFSRDSProvider.java b/src/main/java/org/mtransit/android/commons/provider/GTFSRDSProvider.java index 5367ac7e..9614c705 100644 --- a/src/main/java/org/mtransit/android/commons/provider/GTFSRDSProvider.java +++ b/src/main/java/org/mtransit/android/commons/provider/GTFSRDSProvider.java @@ -20,12 +20,12 @@ public class GTFSRDSProvider implements MTLog.Loggable { - private static final String TAG = GTFSRDSProvider.class.getSimpleName(); + private static final String LOG_TAG = GTFSRDSProvider.class.getSimpleName(); @NonNull @Override public String getLogTag() { - return TAG; + return LOG_TAG; } protected static final int ROUTES = 1; @@ -223,13 +223,13 @@ public static Cursor queryS(@NonNull GTFSProvider provider, @NonNull Uri uri, @N if (TextUtils.isEmpty(sortOrder)) { sortOrder = provider.getSortOrder(uri); } - Cursor cursor = qb.query(provider.getReadDB(), projection, selection, selectionArgs, null, null, sortOrder, null); + final Cursor cursor = qb.query(provider.getReadDB(), projection, selection, selectionArgs, null, null, sortOrder, null); if (cursor != null) { cursor.setNotificationUri(provider.requireContextCompat().getContentResolver(), uri); } return cursor; } catch (Exception e) { - MTLog.w(TAG, e, "Error while resolving query '%s'!", uri); + MTLog.w(LOG_TAG, e, "Error while resolving query '%s'!", uri); return null; } } diff --git a/src/main/java/org/mtransit/android/commons/provider/GTFSScheduleTimestampsProvider.java b/src/main/java/org/mtransit/android/commons/provider/GTFSScheduleTimestampsProvider.java index 9ea8c06c..a0b93e21 100644 --- a/src/main/java/org/mtransit/android/commons/provider/GTFSScheduleTimestampsProvider.java +++ b/src/main/java/org/mtransit/android/commons/provider/GTFSScheduleTimestampsProvider.java @@ -18,7 +18,7 @@ import java.util.ArrayList; import java.util.Calendar; -import java.util.HashSet; +import java.util.Set; import java.util.TimeZone; import java.util.concurrent.TimeUnit; @@ -48,7 +48,7 @@ static ScheduleTimestamps getScheduleTimestamps(@NonNull GTFSProvider provider, final TimeZone timeZone = TimeZone.getTimeZone(AgencyUtils.getRDSAgencyTimeZone(context)); final Calendar startsAt = TimeUtils.getNewCalendar(timeZone, startsAtInMs); startsAt.add(Calendar.DATE, -1); // starting yesterday - HashSet dayTimestamps; + Set dayTimestamps; String lookupDayTime; String lookupDayDate; int dataRequests = 0; diff --git a/src/main/java/org/mtransit/android/commons/provider/GTFSStatusProvider.java b/src/main/java/org/mtransit/android/commons/provider/GTFSStatusProvider.java index 6b7b064e..4326b916 100644 --- a/src/main/java/org/mtransit/android/commons/provider/GTFSStatusProvider.java +++ b/src/main/java/org/mtransit/android/commons/provider/GTFSStatusProvider.java @@ -252,7 +252,7 @@ private static ArrayList findTimestamps(@NonNull GTFSProvide } } now.add(Calendar.DATE, -1); // starting yesterday - HashSet dayTimestamps; + Set dayTimestamps; String lookupDayTime; String lookupDayDate; int nbTimestamps = 0; @@ -349,14 +349,16 @@ private static String getSTOP_SCHEDULE_RAW_FILE_FORMAT(@NonNull Context context) private static final int GTFS_SCHEDULE_STOP_FILE_COL_ACCESSIBLE_IDX = 5; @NonNull - static HashSet findScheduleList(@NonNull GTFSProvider provider, - @SuppressWarnings("unused") long routeId, - long directionId, - int stopId, - String dateS, String timeS, - long diffWithRealityInMs) { + static Set findScheduleList( + @NonNull GTFSProvider provider, + @SuppressWarnings("unused") long routeId, + long directionId, + int stopId, + String dateS, String timeS, + long diffWithRealityInMs + ) { final int timeI = Integer.parseInt(timeS); - HashSet result = new HashSet<>(); + Set result = new HashSet<>(); final Set> serviceIdOrIntAndExceptionTypes = findServicesAndExceptionTypes(provider, dateS); final Set serviceIdOrInts = filterServiceIdOrInts(serviceIdOrIntAndExceptionTypes, diffWithRealityInMs > 0L); BufferedReader br = null; @@ -461,6 +463,9 @@ static HashSet findScheduleList(@NonNull GTFSProvider provid } finally { FileUtils.closeQuietly(br); } + if (FeatureFlags.F_EXPORT_STRINGS) { + result = GTFSStringsUtils.updateStrings(result, provider); + } return result; } diff --git a/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt b/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt new file mode 100644 index 00000000..459c4cd5 --- /dev/null +++ b/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt @@ -0,0 +1,55 @@ +package org.mtransit.android.commons.provider + +import org.mtransit.android.commons.data.Schedule + +object GTFSStringsUtils { + + private const val SPACE = " " + + @Suppress("DiscouragedApi") + @JvmStatic + fun updateStrings(timestamps: Set, gtfsProvider: GTFSProvider): Set { + val stringIds = timestamps + .mapNotNull { it.headsignValue?.split(SPACE) } + .flatten() + .distinct() + .takeIf { it.isNotEmpty() } + ?: return timestamps + val idToStringMap = loadStrings(gtfsProvider, stringIds) + timestamps.forEach { timestamp -> + timestamp.headsignValue?.let { headsignValue -> + timestamp.headsignValue = headsignValue + .split(SPACE) + .mapNotNull { idToStringMap[it.toIntOrNull()] } + .joinToString(SPACE) + } + } + return timestamps + } + + private fun loadStrings(gtfsProvider: GTFSProvider, stringIds: List): Map { + if (stringIds.isEmpty()) return emptyMap() + return gtfsProvider.readDB.query( + GTFSProviderDbHelper.T_STRINGS, + arrayOf(GTFSProviderDbHelper.T_STRINGS_K_ID, GTFSProviderDbHelper.T_STRINGS_K_STRING), + "${GTFSProviderDbHelper.T_STRINGS_K_ID} IN (${stringIds.joinToString(",")})", + null, + null, + null, + null + ).use { cursor -> + buildMap { + while (cursor.moveToNext()) { + try { + put( + cursor.getInt(cursor.getColumnIndexOrThrow(GTFSProviderDbHelper.T_STRINGS_K_ID)), + cursor.getString(cursor.getColumnIndexOrThrow(GTFSProviderDbHelper.T_STRINGS_K_STRING)) ?: continue + ) + } catch (e: Exception) { + e.printStackTrace() + } + } + } + } + } +} From 1e376972be07ca57ff1108dbc341e3b1c500b4b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Mon, 24 Nov 2025 10:16:17 -0500 Subject: [PATCH 3/9] Replacing strings IDs with strings during DB init --- .../provider/GTFSProviderDbHelper.java | 115 ++++++++++++------ .../commons/provider/GTFSStringsUtils.kt | 100 ++++++++++++--- 2 files changed, 165 insertions(+), 50 deletions(-) diff --git a/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java b/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java index 202492d7..cd4cde0a 100644 --- a/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java +++ b/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java @@ -5,6 +5,7 @@ import android.database.sqlite.SQLiteDatabase; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import androidx.core.app.NotificationCompat; import androidx.core.app.NotificationManagerCompat; @@ -22,6 +23,10 @@ import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; +import java.util.HashMap; +import java.util.Map; + +import kotlin.Pair; public class GTFSProviderDbHelper extends MTSQLiteOpenHelper { @@ -52,6 +57,7 @@ public String getLogTag() { static final String T_ROUTE_K_COLOR = GTFSCommons.T_ROUTE_K_COLOR; static final String T_ROUTE_K_ORIGINAL_ID_HASH = GTFSCommons.T_ROUTE_K_ORIGINAL_ID_HASH; static final String T_ROUTE_K_TYPE = GTFSCommons.T_ROUTE_K_TYPE; + private static final int[] T_ROUTE_STRINGS_COLUMN_IDX = GTFSCommons.T_ROUTE_STRINGS_COLUMN_IDX; private static final String T_ROUTE_SQL_CREATE = GTFSCommons.getT_ROUTE_SQL_CREATE(); private static final String T_ROUTE_SQL_INSERT = GTFSCommons.getT_ROUTE_SQL_INSERT(); private static final String T_ROUTE_SQL_DROP = GTFSCommons.getT_ROUTE_SQL_DROP(); @@ -61,6 +67,7 @@ public String getLogTag() { static final String T_DIRECTION_K_HEADSIGN_TYPE = GTFSCommons.T_DIRECTION_K_HEADSIGN_TYPE; static final String T_DIRECTION_K_HEADSIGN_VALUE = GTFSCommons.T_DIRECTION_K_HEADSIGN_VALUE; // really? static final String T_DIRECTION_K_ROUTE_ID = GTFSCommons.T_DIRECTION_K_ROUTE_ID; + private static final int[] T_DIRECTION_STRINGS_COLUMN_IDX = GTFSCommons.T_DIRECTION_STRINGS_COLUMN_IDX; private static final String T_DIRECTION_SQL_CREATE = GTFSCommons.getT_DIRECTION_SQL_CREATE(); private static final String T_DIRECTION_SQL_INSERT = GTFSCommons.getT_DIRECTION_SQL_INSERT(); private static final String T_DIRECTION_SQL_DROP = GTFSCommons.getT_DIRECTION_SQL_DROP(); @@ -73,6 +80,7 @@ public String getLogTag() { static final String T_STOP_K_LNG = GTFSCommons.T_STOP_K_LNG; static final String T_STOP_K_ACCESSIBLE = GTFSCommons.T_STOP_K_ACCESSIBLE; static final String T_STOP_K_ORIGINAL_ID_HASH = GTFSCommons.T_STOP_K_ORIGINAL_ID_HASH; + private static final int[] T_STOP_STRINGS_COLUMN_IDX = GTFSCommons.T_STOP_STRINGS_COLUMN_IDX; private static final String T_STOP_SQL_CREATE = GTFSCommons.getT_STOP_SQL_CREATE(); private static final String T_STOP_SQL_INSERT = GTFSCommons.getT_STOP_SQL_INSERT(); private static final String T_STOP_SQL_DROP = GTFSCommons.getT_STOP_SQL_DROP(); @@ -154,11 +162,12 @@ public boolean isDbExist(@NonNull Context context) { return SqlUtils.isDbExist(context, DB_NAME); } - @SuppressLint("MissingPermission") // no notification if not permitted (not requesting permission) + @SuppressLint("MissingPermission") // no notification if not permitted (not requesting permission for that) private void initAllDbTables(@NonNull SQLiteDatabase db, boolean upgrade) { MTLog.i(this, "Data: deploying DB..."); - int nId = TimeUtils.currentTimeSec(); - int nbTotalOperations = 8; + final int nId = TimeUtils.currentTimeSec(); + final int nbTotalOperations = 8; + int progress = 0; final NotificationManagerCompat nm = NotificationManagerCompat.from(this.context); final boolean notifEnabled = nm.areNotificationsEnabled(); final NotificationCompat.Builder nb; @@ -174,59 +183,56 @@ private void initAllDbTables(@NonNull SQLiteDatabase db, boolean upgrade) { nb = null; } db.execSQL(SQLUtils.PRAGMA_AUTO_VACUUM_NONE); - if (notifEnabled) { - NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 0); - } - initDbTableWithRetry(db, T_ROUTE, T_ROUTE_SQL_CREATE, T_ROUTE_SQL_INSERT, T_ROUTE_SQL_DROP, getRouteFiles()); - if (notifEnabled) { - NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 1); - } - initDbTableWithRetry(db, T_DIRECTION, T_DIRECTION_SQL_CREATE, T_DIRECTION_SQL_INSERT, T_DIRECTION_SQL_DROP, getDirectionFiles()); - if (notifEnabled) { - NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 2); - } - initDbTableWithRetry(db, T_STOP, T_STOP_SQL_CREATE, T_STOP_SQL_INSERT, T_STOP_SQL_DROP, getStopFiles()); - if (notifEnabled) { - NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 3); + if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, progress++); + final Map allStrings = readStrings(getStringsFiles()); + if (FeatureFlags.F_EXPORT_STRINGS) { + initDbTableWithRetry(db, T_STRINGS, T_STRINGS_SQL_CREATE, T_STRINGS_SQL_INSERT, T_STRINGS_SQL_DROP, getStringsFiles()); // 1st } + if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, progress++); + initDbTableWithRetry(db, T_ROUTE, T_ROUTE_SQL_CREATE, T_ROUTE_SQL_INSERT, T_ROUTE_SQL_DROP, getRouteFiles(), allStrings, T_ROUTE_STRINGS_COLUMN_IDX); + if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, progress++); + initDbTableWithRetry(db, T_DIRECTION, T_DIRECTION_SQL_CREATE, T_DIRECTION_SQL_INSERT, T_DIRECTION_SQL_DROP, getDirectionFiles(), allStrings, T_DIRECTION_STRINGS_COLUMN_IDX); + if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, progress++); + initDbTableWithRetry(db, T_STOP, T_STOP_SQL_CREATE, T_STOP_SQL_INSERT, T_STOP_SQL_DROP, getStopFiles(), allStrings, T_STOP_STRINGS_COLUMN_IDX); + if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, progress++); initDbTableWithRetry(db, T_DIRECTION_STOPS, T_DIRECTION_STOPS_SQL_CREATE, T_DIRECTION_STOPS_SQL_INSERT, T_DIRECTION_STOPS_SQL_DROP, getDirectionStopsFiles()); - if (notifEnabled) { - NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 4); - } - initDbTableWithRetry(db, T_SERVICE_IDS, T_SERVICE_IDS_SQL_CREATE, T_SERVICE_IDS_SQL_INSERT, T_SERVICE_IDS_SQL_DROP, getServiceIdsFiles()); - if (notifEnabled) { - NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 5); + if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, progress++); + if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { + initDbTableWithRetry(db, T_SERVICE_IDS, T_SERVICE_IDS_SQL_CREATE, T_SERVICE_IDS_SQL_INSERT, T_SERVICE_IDS_SQL_DROP, getServiceIdsFiles()); } + if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, progress++); initDbTableWithRetry(db, T_SERVICE_DATES, T_SERVICE_DATES_SQL_CREATE, T_SERVICE_DATES_SQL_INSERT, T_SERVICE_DATES_SQL_DROP, getServiceDatesFiles()); - if (notifEnabled) { - NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 6); - } - initDbTableWithRetry(db, T_STRINGS, T_STRINGS_SQL_CREATE, T_STRINGS_SQL_INSERT, T_STRINGS_SQL_DROP, getStringsFiles()); - if (notifEnabled) { - NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 7); - } + if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, progress++); db.execSQL(T_ROUTE_DIRECTION_STOP_STATUS_SQL_CREATE); if (notifEnabled) { nb.setSmallIcon(android.R.drawable.stat_notify_sync_noanim); // - NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 8); + NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, progress); nm.cancel(nId); } MTLog.i(this, "Data: deploying DB... DONE"); } private void initDbTableWithRetry(@NonNull SQLiteDatabase db, String table, String sqlCreate, String sqlInsert, String sqlDrop, int[] files) { + initDbTableWithRetry(db, table, sqlCreate, sqlInsert, sqlDrop, files, null, null); + } + + private void initDbTableWithRetry(@NonNull SQLiteDatabase db, String table, String sqlCreate, String sqlInsert, String sqlDrop, int[] files, + @Nullable Map allStrings, @Nullable int[] stringsColumnIdx) { + int tried = 0; boolean success; do { try { - success = initDbTable(db, table, sqlCreate, sqlInsert, sqlDrop, files); + success = initDbTable(db, table, sqlCreate, sqlInsert, sqlDrop, files, allStrings, stringsColumnIdx); } catch (Exception e) { MTLog.w(this, e, "Error while deploying DB table %s!", table); success = false; } - } while (!success); + tried++; + } while (!success && tried < 3); } - private boolean initDbTable(@NonNull SQLiteDatabase db, String table, String sqlCreate, String sqlInsert, String sqlDrop, int[] files) { + private boolean initDbTable(@NonNull SQLiteDatabase db, String table, String sqlCreate, String sqlInsert, String sqlDrop, int[] files, + @Nullable Map allStrings, @Nullable int[] stringsColumnIdx) { try { db.beginTransaction(); db.execSQL(sqlDrop); // drop if exists @@ -241,6 +247,10 @@ private boolean initDbTable(@NonNull SQLiteDatabase db, String table, String sql isr = new InputStreamReader(is, FileUtils.getUTF8()); br = new BufferedReader(isr, 8192); while ((line = br.readLine()) != null) { + if (FeatureFlags.F_EXPORT_STRINGS + && allStrings != null && stringsColumnIdx != null && stringsColumnIdx.length > 0) { + line = GTFSStringsUtils.replaceLineStrings(line, allStrings, stringsColumnIdx); + } String sql = String.format(sqlInsert, line); try { db.execSQL(sql); @@ -268,6 +278,43 @@ private boolean initDbTable(@NonNull SQLiteDatabase db, String table, String sql } } + @Nullable + private Map readStrings(int[] files) { + if (!FeatureFlags.F_EXPORT_STRINGS) return null; + try { + Map allStrings = new HashMap<>(); + String line; + Pair idAndString; + BufferedReader br = null; + InputStreamReader isr = null; + InputStream is = null; + for (int file : files) { + try { + is = this.context.getResources().openRawResource(file); + isr = new InputStreamReader(is, FileUtils.getUTF8()); + br = new BufferedReader(isr, 8192); + while ((line = br.readLine()) != null) { + idAndString = GTFSStringsUtils.fromFileLine(line); + if (idAndString != null) { + allStrings.put(idAndString.getFirst(), idAndString.getSecond()); + } + } + } catch (Exception e) { + MTLog.w(this, e, "ERROR while reading strings from the database '%s' file '%s'!", DB_NAME, file); + return null; + } finally { + FileUtils.closeQuietly(br); + FileUtils.closeQuietly(isr); + FileUtils.closeQuietly(is); + } + } + return allStrings; + } catch (Exception e) { + MTLog.w(this, e, "ERROR while reading strings from the database '%s' file!", DB_NAME); + return null; + } + } + /** * Override if multiple {@link GTFSProviderDbHelper} implementations in same app. */ diff --git a/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt b/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt index 459c4cd5..db40d75e 100644 --- a/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt +++ b/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt @@ -1,16 +1,24 @@ package org.mtransit.android.commons.provider +import android.database.Cursor +import org.mtransit.android.commons.MTLog import org.mtransit.android.commons.data.Schedule +import org.mtransit.commons.GTFSCommons +import org.mtransit.commons.sql.SQLUtils +import org.mtransit.commons.sql.SQLUtils.quotes +import org.mtransit.commons.sql.SQLUtils.unquotes -object GTFSStringsUtils { +object GTFSStringsUtils : MTLog.Loggable { - private const val SPACE = " " + private val LOG_TAG: String = GTFSStringsUtils::class.java.simpleName + + override fun getLogTag() = LOG_TAG @Suppress("DiscouragedApi") @JvmStatic fun updateStrings(timestamps: Set, gtfsProvider: GTFSProvider): Set { val stringIds = timestamps - .mapNotNull { it.headsignValue?.split(SPACE) } + .mapNotNull { it.headsignValue?.split(GTFSCommons.STRINGS_SEPARATOR) } .flatten() .distinct() .takeIf { it.isNotEmpty() } @@ -19,9 +27,9 @@ object GTFSStringsUtils { timestamps.forEach { timestamp -> timestamp.headsignValue?.let { headsignValue -> timestamp.headsignValue = headsignValue - .split(SPACE) + .split(GTFSCommons.STRINGS_SEPARATOR) .mapNotNull { idToStringMap[it.toIntOrNull()] } - .joinToString(SPACE) + .joinToString(GTFSCommons.STRINGS_SEPARATOR) } } return timestamps @@ -38,18 +46,78 @@ object GTFSStringsUtils { null, null ).use { cursor -> - buildMap { - while (cursor.moveToNext()) { - try { - put( - cursor.getInt(cursor.getColumnIndexOrThrow(GTFSProviderDbHelper.T_STRINGS_K_ID)), - cursor.getString(cursor.getColumnIndexOrThrow(GTFSProviderDbHelper.T_STRINGS_K_STRING)) ?: continue - ) - } catch (e: Exception) { - e.printStackTrace() - } - } + cursorToStrings(cursor) + } + } + + private fun cursorToStrings(cursor: Cursor) = buildMap { + while (cursor.moveToNext()) { + try { + put( + cursor.getInt(cursor.getColumnIndexOrThrow(GTFSProviderDbHelper.T_STRINGS_K_ID)), + cursor.getString(cursor.getColumnIndexOrThrow(GTFSProviderDbHelper.T_STRINGS_K_STRING)) ?: continue + ) + } catch (e: Exception) { + MTLog.w(this@GTFSStringsUtils, e, "Cannot parse strings cursor: '$cursor'!") } } } + + @JvmStatic + fun loadAllStrings(gtfsProvider: GTFSProvider): Map { + return gtfsProvider.readDB.query( + GTFSProviderDbHelper.T_STRINGS, + arrayOf(GTFSProviderDbHelper.T_STRINGS_K_ID, GTFSProviderDbHelper.T_STRINGS_K_STRING), + null, + null, + null, + null, + null + ).use { cursor -> + cursorToStrings(cursor) + } + } + + @JvmStatic + fun fromFileLine(line: String) = + line.split(SQLUtils.COLUMN_SEPARATOR) + .takeIf { it.size == 2 } + ?.let { columns -> + columns[0].toInt() to columns[1].unquotes() + } + ?: run { + MTLog.w(this@GTFSStringsUtils, "Invalid string line: '$line'!") + null + } + + @JvmStatic + fun replaceLineStrings(line: String, allStrings: Map?, stringsColumnIdx: IntArray): String { + return line + .takeUnless { allStrings.isNullOrEmpty() || stringsColumnIdx.isEmpty() } + ?.split(SQLUtils.COLUMN_SEPARATOR) + ?.takeIf { it.isNotEmpty() } + ?.toMutableList() + ?.apply { + for (idx in stringsColumnIdx) { + this.getOrNull(idx)?.unquotes()?.let { + replaceStrings(it.unquotes(), allStrings).quotes() + }?.let { this[idx] = it } + } + }?.joinToString(SQLUtils.COLUMN_SEPARATOR) + ?: line + } + + fun replaceStrings(stringIds: String, allStrings: Map?): String { + return stringIds + .takeUnless { allStrings.isNullOrEmpty() } + ?.split(GTFSCommons.STRINGS_SEPARATOR) + ?.takeIf { it.isNotEmpty() } + ?.toMutableList() + ?.apply { + indices.forEach { idx -> + allStrings?.get(this[idx].toIntOrNull())?.let { this[idx] = it } + } + }?.joinToString(GTFSCommons.STRINGS_SEPARATOR) + ?: stringIds + } } From 0a6ea6807d9406bff585dafc7442871b345f627b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Mon, 24 Nov 2025 11:11:32 -0500 Subject: [PATCH 4/9] PR comments --- .../provider/GTFSProviderDbHelper.java | 16 +++++----------- .../commons/provider/GTFSStringsUtils.kt | 19 +------------------ 2 files changed, 6 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java b/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java index cd4cde0a..b9fddec1 100644 --- a/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java +++ b/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java @@ -285,14 +285,12 @@ private Map readStrings(int[] files) { Map allStrings = new HashMap<>(); String line; Pair idAndString; - BufferedReader br = null; - InputStreamReader isr = null; - InputStream is = null; for (int file : files) { - try { - is = this.context.getResources().openRawResource(file); - isr = new InputStreamReader(is, FileUtils.getUTF8()); - br = new BufferedReader(isr, 8192); + try ( + final InputStream is = this.context.getResources().openRawResource(file); + final InputStreamReader isr = new InputStreamReader(is, FileUtils.getUTF8()); + final BufferedReader br = new BufferedReader(isr, 8192) + ) { while ((line = br.readLine()) != null) { idAndString = GTFSStringsUtils.fromFileLine(line); if (idAndString != null) { @@ -302,10 +300,6 @@ private Map readStrings(int[] files) { } catch (Exception e) { MTLog.w(this, e, "ERROR while reading strings from the database '%s' file '%s'!", DB_NAME, file); return null; - } finally { - FileUtils.closeQuietly(br); - FileUtils.closeQuietly(isr); - FileUtils.closeQuietly(is); } } return allStrings; diff --git a/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt b/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt index db40d75e..6bbe5838 100644 --- a/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt +++ b/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt @@ -63,21 +63,6 @@ object GTFSStringsUtils : MTLog.Loggable { } } - @JvmStatic - fun loadAllStrings(gtfsProvider: GTFSProvider): Map { - return gtfsProvider.readDB.query( - GTFSProviderDbHelper.T_STRINGS, - arrayOf(GTFSProviderDbHelper.T_STRINGS_K_ID, GTFSProviderDbHelper.T_STRINGS_K_STRING), - null, - null, - null, - null, - null - ).use { cursor -> - cursorToStrings(cursor) - } - } - @JvmStatic fun fromFileLine(line: String) = line.split(SQLUtils.COLUMN_SEPARATOR) @@ -99,9 +84,7 @@ object GTFSStringsUtils : MTLog.Loggable { ?.toMutableList() ?.apply { for (idx in stringsColumnIdx) { - this.getOrNull(idx)?.unquotes()?.let { - replaceStrings(it.unquotes(), allStrings).quotes() - }?.let { this[idx] = it } + this.getOrNull(idx)?.unquotes()?.let { replaceStrings(it, allStrings) }?.quotes()?.let { this[idx] = it } } }?.joinToString(SQLUtils.COLUMN_SEPARATOR) ?: line From b4d8b337550d7b65d812518434fcf61d02230a94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Mon, 24 Nov 2025 14:09:18 -0500 Subject: [PATCH 5/9] PR comments --- .../android/commons/provider/GTFSProviderDbHelper.java | 4 +++- .../mtransit/android/commons/provider/GTFSStringsUtils.kt | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java b/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java index b9fddec1..fa7b80b1 100644 --- a/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java +++ b/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java @@ -216,6 +216,8 @@ private void initDbTableWithRetry(@NonNull SQLiteDatabase db, String table, Stri initDbTableWithRetry(db, table, sqlCreate, sqlInsert, sqlDrop, files, null, null); } + private static final int MAX_DB_INIT_RETRIES = 3; + private void initDbTableWithRetry(@NonNull SQLiteDatabase db, String table, String sqlCreate, String sqlInsert, String sqlDrop, int[] files, @Nullable Map allStrings, @Nullable int[] stringsColumnIdx) { int tried = 0; @@ -228,7 +230,7 @@ private void initDbTableWithRetry(@NonNull SQLiteDatabase db, String table, Stri success = false; } tried++; - } while (!success && tried < 3); + } while (!success && tried < MAX_DB_INIT_RETRIES); } private boolean initDbTable(@NonNull SQLiteDatabase db, String table, String sqlCreate, String sqlInsert, String sqlDrop, int[] files, diff --git a/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt b/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt index 6bbe5838..315c7104 100644 --- a/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt +++ b/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt @@ -37,11 +37,12 @@ object GTFSStringsUtils : MTLog.Loggable { private fun loadStrings(gtfsProvider: GTFSProvider, stringIds: List): Map { if (stringIds.isEmpty()) return emptyMap() + val placeholders = stringIds.joinToString(",") { "?" } return gtfsProvider.readDB.query( GTFSProviderDbHelper.T_STRINGS, arrayOf(GTFSProviderDbHelper.T_STRINGS_K_ID, GTFSProviderDbHelper.T_STRINGS_K_STRING), - "${GTFSProviderDbHelper.T_STRINGS_K_ID} IN (${stringIds.joinToString(",")})", - null, + "${GTFSProviderDbHelper.T_STRINGS_K_ID} IN ($placeholders)", + stringIds.toTypedArray(), null, null, null From fb18cafa9ed39b38aed8e7e1d54f6c41dcbb2af9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Mon, 24 Nov 2025 14:17:17 -0500 Subject: [PATCH 6/9] PR comments --- .../org/mtransit/android/commons/provider/GTFSStringsUtils.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt b/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt index 315c7104..cb907ffd 100644 --- a/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt +++ b/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt @@ -69,7 +69,7 @@ object GTFSStringsUtils : MTLog.Loggable { line.split(SQLUtils.COLUMN_SEPARATOR) .takeIf { it.size == 2 } ?.let { columns -> - columns[0].toInt() to columns[1].unquotes() + columns[0].toIntOrNull()?.let { it to columns[1].unquotes() } } ?: run { MTLog.w(this@GTFSStringsUtils, "Invalid string line: '$line'!") From 0b7a152386e47a8eefcdadeadb06c66790dd2865 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Mon, 24 Nov 2025 14:53:51 -0500 Subject: [PATCH 7/9] read strings file once --- .../provider/GTFSProviderDbHelper.java | 129 +++--------------- .../gtfs/GTFSProviderDBHelperUtils.kt | 107 +++++++++++++++ 2 files changed, 123 insertions(+), 113 deletions(-) create mode 100644 src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDBHelperUtils.kt diff --git a/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java b/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java index fa7b80b1..bd25ba6b 100644 --- a/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java +++ b/src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java @@ -1,15 +1,15 @@ package org.mtransit.android.commons.provider; +import static org.mtransit.android.commons.provider.gtfs.GTFSProviderDBHelperUtils.initDbTableWithRetry; + import android.annotation.SuppressLint; import android.content.Context; import android.database.sqlite.SQLiteDatabase; import androidx.annotation.NonNull; -import androidx.annotation.Nullable; import androidx.core.app.NotificationCompat; import androidx.core.app.NotificationManagerCompat; -import org.mtransit.android.commons.FileUtils; import org.mtransit.android.commons.MTLog; import org.mtransit.android.commons.NotificationUtils; import org.mtransit.android.commons.PackageManagerUtils; @@ -20,13 +20,10 @@ import org.mtransit.commons.GTFSCommons; import org.mtransit.commons.sql.SQLUtils; -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; import java.util.HashMap; import java.util.Map; -import kotlin.Pair; +import kotlin.Unit; public class GTFSProviderDbHelper extends MTSQLiteOpenHelper { @@ -184,24 +181,29 @@ private void initAllDbTables(@NonNull SQLiteDatabase db, boolean upgrade) { } db.execSQL(SQLUtils.PRAGMA_AUTO_VACUUM_NONE); if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, progress++); - final Map allStrings = readStrings(getStringsFiles()); + final Map allStrings = new HashMap<>(); if (FeatureFlags.F_EXPORT_STRINGS) { - initDbTableWithRetry(db, T_STRINGS, T_STRINGS_SQL_CREATE, T_STRINGS_SQL_INSERT, T_STRINGS_SQL_DROP, getStringsFiles()); // 1st + initDbTableWithRetry(context, db, T_STRINGS, T_STRINGS_SQL_CREATE, T_STRINGS_SQL_INSERT, T_STRINGS_SQL_DROP, getStringsFiles(), null, null, + (id, string) -> { + allStrings.put(id, string); + return Unit.INSTANCE; + } + ); // 1st } if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, progress++); - initDbTableWithRetry(db, T_ROUTE, T_ROUTE_SQL_CREATE, T_ROUTE_SQL_INSERT, T_ROUTE_SQL_DROP, getRouteFiles(), allStrings, T_ROUTE_STRINGS_COLUMN_IDX); + initDbTableWithRetry(context, db, T_ROUTE, T_ROUTE_SQL_CREATE, T_ROUTE_SQL_INSERT, T_ROUTE_SQL_DROP, getRouteFiles(), allStrings, T_ROUTE_STRINGS_COLUMN_IDX); if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, progress++); - initDbTableWithRetry(db, T_DIRECTION, T_DIRECTION_SQL_CREATE, T_DIRECTION_SQL_INSERT, T_DIRECTION_SQL_DROP, getDirectionFiles(), allStrings, T_DIRECTION_STRINGS_COLUMN_IDX); + initDbTableWithRetry(context, db, T_DIRECTION, T_DIRECTION_SQL_CREATE, T_DIRECTION_SQL_INSERT, T_DIRECTION_SQL_DROP, getDirectionFiles(), allStrings, T_DIRECTION_STRINGS_COLUMN_IDX); if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, progress++); - initDbTableWithRetry(db, T_STOP, T_STOP_SQL_CREATE, T_STOP_SQL_INSERT, T_STOP_SQL_DROP, getStopFiles(), allStrings, T_STOP_STRINGS_COLUMN_IDX); + initDbTableWithRetry(context, db, T_STOP, T_STOP_SQL_CREATE, T_STOP_SQL_INSERT, T_STOP_SQL_DROP, getStopFiles(), allStrings, T_STOP_STRINGS_COLUMN_IDX); if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, progress++); - initDbTableWithRetry(db, T_DIRECTION_STOPS, T_DIRECTION_STOPS_SQL_CREATE, T_DIRECTION_STOPS_SQL_INSERT, T_DIRECTION_STOPS_SQL_DROP, getDirectionStopsFiles()); + initDbTableWithRetry(context, db, T_DIRECTION_STOPS, T_DIRECTION_STOPS_SQL_CREATE, T_DIRECTION_STOPS_SQL_INSERT, T_DIRECTION_STOPS_SQL_DROP, getDirectionStopsFiles()); if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, progress++); if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { - initDbTableWithRetry(db, T_SERVICE_IDS, T_SERVICE_IDS_SQL_CREATE, T_SERVICE_IDS_SQL_INSERT, T_SERVICE_IDS_SQL_DROP, getServiceIdsFiles()); + initDbTableWithRetry(context, db, T_SERVICE_IDS, T_SERVICE_IDS_SQL_CREATE, T_SERVICE_IDS_SQL_INSERT, T_SERVICE_IDS_SQL_DROP, getServiceIdsFiles()); } if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, progress++); - initDbTableWithRetry(db, T_SERVICE_DATES, T_SERVICE_DATES_SQL_CREATE, T_SERVICE_DATES_SQL_INSERT, T_SERVICE_DATES_SQL_DROP, getServiceDatesFiles()); + initDbTableWithRetry(context, db, T_SERVICE_DATES, T_SERVICE_DATES_SQL_CREATE, T_SERVICE_DATES_SQL_INSERT, T_SERVICE_DATES_SQL_DROP, getServiceDatesFiles()); if (notifEnabled) NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, progress++); db.execSQL(T_ROUTE_DIRECTION_STOP_STATUS_SQL_CREATE); if (notifEnabled) { @@ -212,105 +214,6 @@ private void initAllDbTables(@NonNull SQLiteDatabase db, boolean upgrade) { MTLog.i(this, "Data: deploying DB... DONE"); } - private void initDbTableWithRetry(@NonNull SQLiteDatabase db, String table, String sqlCreate, String sqlInsert, String sqlDrop, int[] files) { - initDbTableWithRetry(db, table, sqlCreate, sqlInsert, sqlDrop, files, null, null); - } - - private static final int MAX_DB_INIT_RETRIES = 3; - - private void initDbTableWithRetry(@NonNull SQLiteDatabase db, String table, String sqlCreate, String sqlInsert, String sqlDrop, int[] files, - @Nullable Map allStrings, @Nullable int[] stringsColumnIdx) { - int tried = 0; - boolean success; - do { - try { - success = initDbTable(db, table, sqlCreate, sqlInsert, sqlDrop, files, allStrings, stringsColumnIdx); - } catch (Exception e) { - MTLog.w(this, e, "Error while deploying DB table %s!", table); - success = false; - } - tried++; - } while (!success && tried < MAX_DB_INIT_RETRIES); - } - - private boolean initDbTable(@NonNull SQLiteDatabase db, String table, String sqlCreate, String sqlInsert, String sqlDrop, int[] files, - @Nullable Map allStrings, @Nullable int[] stringsColumnIdx) { - try { - db.beginTransaction(); - db.execSQL(sqlDrop); // drop if exists - db.execSQL(sqlCreate); // create if not exists - String line; - BufferedReader br = null; - InputStreamReader isr = null; - InputStream is = null; - for (int file : files) { - try { - is = this.context.getResources().openRawResource(file); - isr = new InputStreamReader(is, FileUtils.getUTF8()); - br = new BufferedReader(isr, 8192); - while ((line = br.readLine()) != null) { - if (FeatureFlags.F_EXPORT_STRINGS - && allStrings != null && stringsColumnIdx != null && stringsColumnIdx.length > 0) { - line = GTFSStringsUtils.replaceLineStrings(line, allStrings, stringsColumnIdx); - } - String sql = String.format(sqlInsert, line); - try { - db.execSQL(sql); - } catch (Exception e) { - MTLog.w(this, e, "ERROR while executing '%s' on database '%s' table '%s' file '%s'!", sql, DB_NAME, table, file); - throw e; - } - } - } catch (Exception e) { - MTLog.w(this, e, "ERROR while copying the database '%s' table '%s' file '%s'!", DB_NAME, table, file); - return false; - } finally { - FileUtils.closeQuietly(br); - FileUtils.closeQuietly(isr); - FileUtils.closeQuietly(is); - } - } - db.setTransactionSuccessful(); - return true; - } catch (Exception e) { - MTLog.w(this, e, "ERROR while copying the database '%s' table '%s' file!", DB_NAME, table); - return false; - } finally { - SqlUtils.endTransactionQuietly(db); - } - } - - @Nullable - private Map readStrings(int[] files) { - if (!FeatureFlags.F_EXPORT_STRINGS) return null; - try { - Map allStrings = new HashMap<>(); - String line; - Pair idAndString; - for (int file : files) { - try ( - final InputStream is = this.context.getResources().openRawResource(file); - final InputStreamReader isr = new InputStreamReader(is, FileUtils.getUTF8()); - final BufferedReader br = new BufferedReader(isr, 8192) - ) { - while ((line = br.readLine()) != null) { - idAndString = GTFSStringsUtils.fromFileLine(line); - if (idAndString != null) { - allStrings.put(idAndString.getFirst(), idAndString.getSecond()); - } - } - } catch (Exception e) { - MTLog.w(this, e, "ERROR while reading strings from the database '%s' file '%s'!", DB_NAME, file); - return null; - } - } - return allStrings; - } catch (Exception e) { - MTLog.w(this, e, "ERROR while reading strings from the database '%s' file!", DB_NAME); - return null; - } - } - /** * Override if multiple {@link GTFSProviderDbHelper} implementations in same app. */ diff --git a/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDBHelperUtils.kt b/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDBHelperUtils.kt new file mode 100644 index 00000000..0013925d --- /dev/null +++ b/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDBHelperUtils.kt @@ -0,0 +1,107 @@ +package org.mtransit.android.commons.provider.gtfs + +import android.content.Context +import android.database.sqlite.SQLiteDatabase +import org.mtransit.android.commons.FileUtils +import org.mtransit.android.commons.MTLog +import org.mtransit.android.commons.SqlUtils +import org.mtransit.android.commons.provider.GTFSProviderDbHelper +import org.mtransit.android.commons.provider.GTFSStringsUtils +import org.mtransit.commons.FeatureFlags +import org.mtransit.commons.GTFSCommons +import java.io.BufferedReader +import java.io.InputStreamReader + +object GTFSProviderDBHelperUtils: MTLog.Loggable { + + private val LOG_TAG: String = GTFSProviderDBHelperUtils::class.java.simpleName + + override fun getLogTag() = LOG_TAG + + private const val MAX_DB_INIT_RETRIES = 3 + + @JvmStatic + @JvmOverloads + fun initDbTableWithRetry( + context: Context, + db: SQLiteDatabase, + table: String, + sqlCreate: String?, + sqlInsert: String, + sqlDrop: String?, + files: IntArray, + allStrings: Map? = null, + stringsColumnIdx: IntArray? = null, + addStrings: (Int, String) -> Unit = { _, _ -> }, + ) { + var tried = 0 + var success: Boolean + do { + try { + success = initDbTable(context, db, table, sqlCreate, sqlInsert, sqlDrop, files, allStrings, stringsColumnIdx, addStrings) + } catch (e: Exception) { + MTLog.w(this, e, "Error while deploying DB table '$table'!") + success = false + } + tried++ + } while (!success && tried < MAX_DB_INIT_RETRIES) + } + + private fun initDbTable( + context: Context, + db: SQLiteDatabase, + table: String, + sqlCreate: String?, + sqlInsert: String, + sqlDrop: String?, + files: IntArray, + allStrings: Map?, + stringsColumnIdx: IntArray?, + addStrings: (Int, String) -> Unit, + ): Boolean { + try { + db.beginTransaction() + db.execSQL(sqlDrop) // drop if exists + db.execSQL(sqlCreate) // create if not exists + for (file in files) { + try { + context.resources.openRawResource(file).use { inputStream -> + InputStreamReader(inputStream, FileUtils.getUTF8()).use { inputStreamReader -> + BufferedReader(inputStreamReader).forEachLine { + var line = it + if (FeatureFlags.F_EXPORT_STRINGS) { + if (allStrings != null && stringsColumnIdx != null && stringsColumnIdx.isNotEmpty()) { + MTLog.d(this, "initDbTable(%s) > B-line: %s.", table, line) + line = GTFSStringsUtils.replaceLineStrings(line, allStrings, stringsColumnIdx) + MTLog.d(this, "initDbTable(%s) > A-line: %s.", table, line) + } else if (table == GTFSCommons.T_STRINGS) { + GTFSStringsUtils.fromFileLine(line)?.let { (id, string) -> + addStrings(id, string) + } + } + } + val sql = String.format(sqlInsert, line) + try { + db.execSQL(sql) + } catch (e: Exception) { + MTLog.w(this, e, "ERROR while executing '$sql' on database '${GTFSProviderDbHelper.DB_NAME}' table '$table' file '$file'!") + throw e + } + } + } + } + } catch (e: Exception) { + MTLog.w(this, e, "ERROR while copying the database '${GTFSProviderDbHelper.DB_NAME}' table '$table' file '$file'!") + return false + } + } + db.setTransactionSuccessful() + return true + } catch (e: Exception) { + MTLog.w(this, e, "ERROR while copying the database ${GTFSProviderDbHelper.DB_NAME}' table '$table' file!") + return false + } finally { + SqlUtils.endTransactionQuietly(db) + } + } +} From 99361b1a6b763e16ebc3198561bd11bc181f98e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Mon, 24 Nov 2025 15:05:54 -0500 Subject: [PATCH 8/9] PR comments --- .../commons/provider/GTFSScheduleTimestampsProvider.java | 4 ++++ .../android/commons/provider/GTFSStatusProvider.java | 6 +++--- .../mtransit/android/commons/provider/GTFSStringsUtils.kt | 2 +- .../commons/provider/gtfs/GTFSProviderDBHelperUtils.kt | 2 -- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/mtransit/android/commons/provider/GTFSScheduleTimestampsProvider.java b/src/main/java/org/mtransit/android/commons/provider/GTFSScheduleTimestampsProvider.java index a0b93e21..a94b2368 100644 --- a/src/main/java/org/mtransit/android/commons/provider/GTFSScheduleTimestampsProvider.java +++ b/src/main/java/org/mtransit/android/commons/provider/GTFSScheduleTimestampsProvider.java @@ -15,6 +15,7 @@ import org.mtransit.android.commons.data.Schedule; import org.mtransit.android.commons.data.ScheduleTimestamps; import org.mtransit.android.commons.provider.agency.AgencyUtils; +import org.mtransit.commons.FeatureFlags; import java.util.ArrayList; import java.util.Calendar; @@ -106,6 +107,9 @@ static ScheduleTimestamps getScheduleTimestamps(@NonNull GTFSProvider provider, } startsAt.add(Calendar.DATE, +1); // NEXT DAY } + if (FeatureFlags.F_EXPORT_STRINGS) { + allTimestamps = GTFSStringsUtils.updateStrings(allTimestamps, provider); + } ScheduleTimestamps scheduleTimestamps = new ScheduleTimestamps(rds.getUUID(), startsAtInMs, endsAtInMs); scheduleTimestamps.setSourceLabel(GTFSProvider.getSOURCE_LABEL(provider.requireContextCompat())); scheduleTimestamps.setTimestampsAndSort(allTimestamps); diff --git a/src/main/java/org/mtransit/android/commons/provider/GTFSStatusProvider.java b/src/main/java/org/mtransit/android/commons/provider/GTFSStatusProvider.java index 4326b916..2913e306 100644 --- a/src/main/java/org/mtransit/android/commons/provider/GTFSStatusProvider.java +++ b/src/main/java/org/mtransit/android/commons/provider/GTFSStatusProvider.java @@ -310,6 +310,9 @@ private static ArrayList findTimestamps(@NonNull GTFSProvide } now.add(Calendar.DATE, +1); // NEXT DAY } + if (FeatureFlags.F_EXPORT_STRINGS) { + allTimestamps = GTFSStringsUtils.updateStrings(allTimestamps, provider); + } return allTimestamps; } @@ -463,9 +466,6 @@ static Set findScheduleList( } finally { FileUtils.closeQuietly(br); } - if (FeatureFlags.F_EXPORT_STRINGS) { - result = GTFSStringsUtils.updateStrings(result, provider); - } return result; } diff --git a/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt b/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt index cb907ffd..b3fa7b2a 100644 --- a/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt +++ b/src/main/java/org/mtransit/android/commons/provider/GTFSStringsUtils.kt @@ -16,7 +16,7 @@ object GTFSStringsUtils : MTLog.Loggable { @Suppress("DiscouragedApi") @JvmStatic - fun updateStrings(timestamps: Set, gtfsProvider: GTFSProvider): Set { + fun > updateStrings(timestamps: T, gtfsProvider: GTFSProvider): T { val stringIds = timestamps .mapNotNull { it.headsignValue?.split(GTFSCommons.STRINGS_SEPARATOR) } .flatten() diff --git a/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDBHelperUtils.kt b/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDBHelperUtils.kt index 0013925d..658000e5 100644 --- a/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDBHelperUtils.kt +++ b/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDBHelperUtils.kt @@ -71,9 +71,7 @@ object GTFSProviderDBHelperUtils: MTLog.Loggable { var line = it if (FeatureFlags.F_EXPORT_STRINGS) { if (allStrings != null && stringsColumnIdx != null && stringsColumnIdx.isNotEmpty()) { - MTLog.d(this, "initDbTable(%s) > B-line: %s.", table, line) line = GTFSStringsUtils.replaceLineStrings(line, allStrings, stringsColumnIdx) - MTLog.d(this, "initDbTable(%s) > A-line: %s.", table, line) } else if (table == GTFSCommons.T_STRINGS) { GTFSStringsUtils.fromFileLine(line)?.let { (id, string) -> addStrings(id, string) From 2ae276fe564f378bb9af517dc9cc63b28ede03a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Mon, 24 Nov 2025 15:07:00 -0500 Subject: [PATCH 9/9] cleanup --- .../android/commons/provider/gtfs/GTFSProviderDBHelperUtils.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDBHelperUtils.kt b/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDBHelperUtils.kt index 658000e5..691963f3 100644 --- a/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDBHelperUtils.kt +++ b/src/main/java/org/mtransit/android/commons/provider/gtfs/GTFSProviderDBHelperUtils.kt @@ -96,7 +96,7 @@ object GTFSProviderDBHelperUtils: MTLog.Loggable { db.setTransactionSuccessful() return true } catch (e: Exception) { - MTLog.w(this, e, "ERROR while copying the database ${GTFSProviderDbHelper.DB_NAME}' table '$table' file!") + MTLog.w(this, e, "ERROR while copying the database '${GTFSProviderDbHelper.DB_NAME}' table '$table' file!") return false } finally { SqlUtils.endTransactionQuietly(db)