Index Service IDs

src/main/java/org/mtransit/android/commons/provider/GTFSProviderDbHelper.java

@@ -15,6 +15,7 @@
 import org.mtransit.android.commons.R;
 import org.mtransit.android.commons.SqlUtils;
 import org.mtransit.android.commons.TimeUtils;
+import org.mtransit.commons.FeatureFlags;
 import org.mtransit.commons.GTFSCommons;
 import org.mtransit.commons.sql.SQLUtils;
 
@@ -80,8 +81,19 @@ public String getLogTag() {
 	private static final String T_DIRECTION_STOPS_SQL_INSERT = GTFSCommons.getT_DIRECTION_STOPS_SQL_INSERT();
 	private static final String T_DIRECTION_STOPS_SQL_DROP = GTFSCommons.getT_DIRECTION_STOPS_SQL_DROP();
 
+	@SuppressWarnings("WeakerAccess")
+	static final String T_SERVICE_IDS = GTFSCommons.T_SERVICE_IDS;
+	@SuppressWarnings("unused") // not used by main app currently
+	static final String T_SERVICE_IDS_K_ID = GTFSCommons.T_SERVICE_IDS_K_ID;
+	@SuppressWarnings("unused")
+	static final String T_SERVICE_IDS_K_ID_INT = GTFSCommons.T_SERVICE_IDS_K_ID_INT;
+	private static final String T_SERVICE_IDS_SQL_CREATE = GTFSCommons.getT_SERVICE_IDS_SQL_CREATE();
+	private static final String T_SERVICE_IDS_SQL_INSERT = GTFSCommons.getT_SERVICE_IDS_SQL_INSERT();
+	private static final String T_SERVICE_IDS_SQL_DROP = GTFSCommons.getT_SERVICE_IDS_SQL_DROP();
+
 	static final String T_SERVICE_DATES = GTFSCommons.T_SERVICE_DATES;
 	static final String T_SERVICE_DATES_K_SERVICE_ID = GTFSCommons.T_SERVICE_DATES_K_SERVICE_ID;
+	static final String T_SERVICE_DATES_K_SERVICE_ID_INT = GTFSCommons.T_SERVICE_DATES_K_SERVICE_ID_INT;
 	static final String T_SERVICE_DATES_K_DATE = GTFSCommons.T_SERVICE_DATES_K_DATE;
 	static final String T_SERVICE_DATES_K_EXCEPTION_TYPE = GTFSCommons.T_SERVICE_DATES_K_EXCEPTION_TYPE;
 	private static final String T_SERVICE_DATES_SQL_CREATE = GTFSCommons.getT_SERVICE_DATES_SQL_CREATE();
@@ -123,6 +135,9 @@ public void onUpgradeMT(@NonNull SQLiteDatabase db, int oldVersion, int newVersi
 		db.execSQL(T_STOP_SQL_DROP);
 		db.execSQL(T_DIRECTION_SQL_DROP);
 		db.execSQL(T_ROUTE_SQL_DROP);
+		if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) {
+			db.execSQL(T_SERVICE_IDS_SQL_DROP);
+		}
 		db.execSQL(T_SERVICE_DATES_SQL_DROP);
 		db.execSQL(T_ROUTE_DIRECTION_STOP_STATUS_SQL_DROP);
 		initAllDbTables(db, true);
@@ -136,7 +151,7 @@ public boolean isDbExist(@NonNull Context context) {
 	private void initAllDbTables(@NonNull SQLiteDatabase db, boolean upgrade) {
 		MTLog.i(this, "Data: deploying DB...");
 		int nId = TimeUtils.currentTimeSec();
-		int nbTotalOperations = 6;
+		int nbTotalOperations = 7;
 		final NotificationManagerCompat nm = NotificationManagerCompat.from(this.context);
 		final boolean notifEnabled = nm.areNotificationsEnabled();
 		final NotificationCompat.Builder nb;
@@ -171,14 +186,18 @@ private void initAllDbTables(@NonNull SQLiteDatabase db, boolean upgrade) {
 		if (notifEnabled) {
 			NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 4);
 		}
-		initDbTableWithRetry(db, T_SERVICE_DATES, T_SERVICE_DATES_SQL_CREATE, T_SERVICE_DATES_SQL_INSERT, T_SERVICE_DATES_SQL_DROP, getServiceDatesFiles());
+		initDbTableWithRetry(db, T_SERVICE_IDS, T_SERVICE_IDS_SQL_CREATE, T_SERVICE_IDS_SQL_INSERT, T_SERVICE_IDS_SQL_DROP, getServiceIdsFiles());
 		if (notifEnabled) {
 			NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 5);
 		}
+		initDbTableWithRetry(db, T_SERVICE_DATES, T_SERVICE_DATES_SQL_CREATE, T_SERVICE_DATES_SQL_INSERT, T_SERVICE_DATES_SQL_DROP, getServiceDatesFiles());
+		if (notifEnabled) {
+			NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 6);
+		}
 		db.execSQL(T_ROUTE_DIRECTION_STOP_STATUS_SQL_CREATE);
 		if (notifEnabled) {
 			nb.setSmallIcon(android.R.drawable.stat_notify_sync_noanim); //
-			NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 6);
+			NotificationUtils.setProgressAndNotify(nm, nb, nId, nbTotalOperations, 7);
 			nm.cancel(nId);
 		}
 		MTLog.i(this, "Data: deploying DB... DONE");
@@ -238,6 +257,21 @@ private boolean initDbTable(@NonNull SQLiteDatabase db, String table, String sql
 		}
 	}
 
+	/**
+	 * Override if multiple {@link GTFSProviderDbHelper} implementations in same app.
+	 */
+	private int[] getServiceIdsFiles() {
+		if (GTFSCurrentNextProvider.hasCurrentData(context)) {
+			if (GTFSCurrentNextProvider.isNextData(context)) {
+				return new int[]{R.raw.next_gtfs_schedule_service_ids};
+			} else { // CURRENT = default
+				return new int[]{R.raw.current_gtfs_schedule_service_ids};
+			}
+		} else {
+			return new int[]{R.raw.gtfs_schedule_service_ids};
+		}
+	}
+
 	/**
 	 * Override if multiple {@link GTFSProviderDbHelper} implementations in same app.
 	 */

src/main/java/org/mtransit/android/commons/provider/GTFSRealTimeProvider.java

@@ -273,6 +273,20 @@ private static String getAGENCY_SERVICE_ALERTS_URL_CACHED(@NonNull Context conte
 		return agencyServiceAlertsUrlCached;
 	}
 
+	@Nullable
+	private static String serviceIdCleanupRegex = null;
+
+	/**
+	 * Override if multiple {@link GTFSRealTimeProvider} implementations in same app.
+	 */
+	@NonNull
+	private static String getSERVICE_ID_CLEANUP_REGEX(@NonNull Context context) {
+		if (serviceIdCleanupRegex == null) {
+			serviceIdCleanupRegex = context.getResources().getString(R.string.gtfs_rts_service_id_cleanup_regex); // do not change to avoid breaking compat w/ old modules
+		}
+		return serviceIdCleanupRegex;
+	}
+
 	@Nullable
 	private static String routeIdCleanupRegex = null;
 
@@ -1164,6 +1178,21 @@ private ArrayMap<String, String> parseTranslations(@NonNull GtfsRealtime.Transla
 		return translations;
 	}
 
+	@Nullable
+	private Pattern serviceIdCleanupPattern = null;
+
+	private boolean serviceIdCleanupPatternSet = false;
+
+	@SuppressWarnings("unused") // TODO use later for trip_updates, vehicle_location...
+	@Nullable
+	private Pattern getServiceIdCleanupPattern(@NonNull Context context) {
+		if (this.serviceIdCleanupPattern == null && !serviceIdCleanupPatternSet) {
+			this.serviceIdCleanupPattern = GTFSCommons.makeIdCleanupPattern(getSERVICE_ID_CLEANUP_REGEX(context));
+			this.serviceIdCleanupPatternSet = true;
+		}
+		return this.serviceIdCleanupPattern;
+	}
+
 	@Nullable
 	private Pattern routeIdCleanupPattern = null;
 

src/main/java/org/mtransit/android/commons/provider/GTFSStatusProvider.java

@@ -24,6 +24,7 @@
 import org.mtransit.android.commons.data.RouteDirectionStop;
 import org.mtransit.android.commons.data.Schedule;
 import org.mtransit.android.commons.provider.agency.AgencyUtils;
+import org.mtransit.commons.FeatureFlags;
 import org.mtransit.commons.GTFSCommons;
 
 import java.io.BufferedReader;
@@ -356,8 +357,8 @@ static HashSet<Schedule.Timestamp> findScheduleList(@NonNull GTFSProvider provid
 														long diffWithRealityInMs) {
 		final int timeI = Integer.parseInt(timeS);
 		HashSet<Schedule.Timestamp> result = new HashSet<>();
-		final Set<Pair<String, Integer>> serviceIdAndExceptionTypes = findServicesAndExceptionTypes(provider, dateS);
-		final Set<String> serviceIds = filterServiceIds(serviceIdAndExceptionTypes, diffWithRealityInMs > 0L);
+		final Set<Pair<String, Integer>> serviceIdOrIntAndExceptionTypes = findServicesAndExceptionTypes(provider, dateS);
+		final Set<String> serviceIdOrInts = filterServiceIdOrInts(serviceIdOrIntAndExceptionTypes, diffWithRealityInMs > 0L);
 		BufferedReader br = null;
 		String line = null;
 		final Context context = provider.requireContextCompat();
@@ -373,7 +374,7 @@ static HashSet<Schedule.Timestamp> findScheduleList(@NonNull GTFSProvider provid
 			br = new BufferedReader(new InputStreamReader(is, FileUtils.getUTF8()), 8192);
 			String[] lineItems;
 			String lineServiceIdWithQuotes;
-			String lineServiceId;
+			String lineServiceIdOrInt;
 			long lineDirectionId;
 			int lineDeparture;
 			int lineDepartureDelta;
@@ -391,9 +392,13 @@ static HashSet<Schedule.Timestamp> findScheduleList(@NonNull GTFSProvider provid
 						MTLog.w(LOG_TAG, "Cannot parse schedule '%s'!", line);
 						continue;
 					}
-					lineServiceIdWithQuotes = lineItems[GTFS_SCHEDULE_STOP_FILE_COL_SERVICE_IDX];
-					lineServiceId = lineServiceIdWithQuotes.substring(1, lineServiceIdWithQuotes.length() - 1);
-					if (!serviceIds.contains(lineServiceId)) {
+					if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) {
+						lineServiceIdOrInt = lineItems[GTFS_SCHEDULE_STOP_FILE_COL_SERVICE_IDX];
+					} else {
+						lineServiceIdWithQuotes = lineItems[GTFS_SCHEDULE_STOP_FILE_COL_SERVICE_IDX];
+						lineServiceIdOrInt = lineServiceIdWithQuotes.substring(1, lineServiceIdWithQuotes.length() - 1);
+					}
+					if (!serviceIdOrInts.contains(lineServiceIdOrInt)) {
 						continue;
 					}
 					lineDirectionId = Long.parseLong(lineItems[GTFS_SCHEDULE_STOP_FILE_COL_DIRECTION_IDX]);
@@ -460,44 +465,44 @@ static HashSet<Schedule.Timestamp> findScheduleList(@NonNull GTFSProvider provid
 	}
 
 	@NonNull
-	protected static Set<String> filterServiceIds(@NonNull Set<Pair<String, Integer>> serviceIdAndExceptionTypes, boolean usingAnotherDate) {
-		final HashSet<String> serviceIds = new HashSet<>();
-		final HashSet<String> serviceIdsToRemove = new HashSet<>();
-		final HashSet<String> serviceIdsToAdd = new HashSet<>();
-		for (Pair<String, Integer> serviceIdAndExceptionType : serviceIdAndExceptionTypes) {
-			final String serviceId = serviceIdAndExceptionType.first;
-			final Integer exceptionType = serviceIdAndExceptionType.second;
+	protected static Set<String> filterServiceIdOrInts(@NonNull Set<Pair<String, Integer>> serviceIdOrIntAndExceptionTypes, boolean usingAnotherDate) {
+		final HashSet<String> serviceIdOrInts = new HashSet<>();
+		final HashSet<String> serviceIdOrIntsToRemove = new HashSet<>();
+		final HashSet<String> serviceIdOrIntsToAdd = new HashSet<>();
+		for (Pair<String, Integer> serviceIdOrIntAndExceptionType : serviceIdOrIntAndExceptionTypes) {
+			final String serviceIdOrInt = serviceIdOrIntAndExceptionType.first;
+			final Integer exceptionType = serviceIdOrIntAndExceptionType.second;
 			if (exceptionType == null) {
-				MTLog.w(LOG_TAG, "Skip invalid exception type fr service ID '%s'!", serviceId);
+				MTLog.w(LOG_TAG, "SKIP invalid exception type for service ID '%s'!", serviceIdOrInt);
 				continue;
 			}
 			switch (exceptionType) {
 			case GTFSCommons.EXCEPTION_TYPE_DEFAULT:
-				serviceIds.add(serviceId);
+				serviceIdOrInts.add(serviceIdOrInt);
 				break;
 			case GTFSCommons.EXCEPTION_TYPE_ADDED:
 				if (usingAnotherDate) {
-					serviceIdsToAdd.add(serviceId); // maybe all services add ADDED (no calendar.txt provided)
+					serviceIdOrIntsToAdd.add(serviceIdOrInt); // maybe all services add ADDED (no calendar.txt provided)
 				} else {
-					serviceIds.add(serviceId);
+					serviceIdOrInts.add(serviceIdOrInt);
 				}
 				break;
 			case GTFSCommons.EXCEPTION_TYPE_REMOVED:
-				serviceIdsToRemove.add(serviceId);
+				serviceIdOrIntsToRemove.add(serviceIdOrInt);
 				break;
 			default:
-				MTLog.w(LOG_TAG, "Unexpected service ID exception type '%s' for '%s'!", exceptionType, serviceId);
+				MTLog.w(LOG_TAG, "Unexpected service ID exception type '%s' for '%s'!", exceptionType, serviceIdOrInt);
 				break;
 			}
 		}
 		if (usingAnotherDate) {
-			if (serviceIds.isEmpty()) {
-				serviceIds.addAll(serviceIdsToAdd); // maybe all services add ADDED (no calendar.txt provided)
+			if (serviceIdOrInts.isEmpty()) {
+				serviceIdOrInts.addAll(serviceIdOrIntsToAdd); // maybe all services add ADDED (no calendar.txt provided)
 			}
 		} else {
-			serviceIds.removeAll(serviceIdsToRemove);
+			serviceIdOrInts.removeAll(serviceIdOrIntsToRemove);
 		}
-		return serviceIds;
+		return serviceIdOrInts;
 	}
 
 	@NonNull
@@ -577,16 +582,16 @@ private static HashSet<Schedule.Frequency> findFrequencyList(@NonNull GTFSProvid
 																 long diffWithRealityInMs) {
 		long timeI = Integer.parseInt(timeS);
 		final HashSet<Schedule.Frequency> result = new HashSet<>();
-		final Set<Pair<String, Integer>> serviceIdAndExceptionTypes = findServicesAndExceptionTypes(provider, dateS);
-		final Set<String> serviceIds = filterServiceIds(serviceIdAndExceptionTypes, diffWithRealityInMs > 0L);
+		final Set<Pair<String, Integer>> serviceIdOrIntAndExceptionTypes = findServicesAndExceptionTypes(provider, dateS);
+		final Set<String> serviceIdOrInts = filterServiceIdOrInts(serviceIdOrIntAndExceptionTypes, diffWithRealityInMs > 0L);
 		BufferedReader br = null;
 		String line = null;
 		final Context context = provider.requireContextCompat();
 		String fileName = String.format(getROUTE_FREQUENCY_RAW_FILE_FORMAT(context), routeId);
 		InputStream is;
 		String[] lineItems;
 		String lineServiceIdWithQuotes;
-		String lineServiceId;
+		String lineServiceIdOrInt;
 		long lineDirectionId;
 		int endTime;
 		int startTime;
@@ -608,9 +613,13 @@ private static HashSet<Schedule.Frequency> findFrequencyList(@NonNull GTFSProvid
 						MTLog.w(LOG_TAG, "Cannot parse frequency '%s'!", line);
 						continue;
 					}
-					lineServiceIdWithQuotes = lineItems[GTFS_ROUTE_FREQUENCY_FILE_COL_SERVICE_IDX];
-					lineServiceId = lineServiceIdWithQuotes.substring(1, lineServiceIdWithQuotes.length() - 1);
-					if (!serviceIds.contains(lineServiceId)) {
+					if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) {
+						lineServiceIdOrInt = lineItems[GTFS_ROUTE_FREQUENCY_FILE_COL_SERVICE_IDX];
+					} else {
+						lineServiceIdWithQuotes = lineItems[GTFS_ROUTE_FREQUENCY_FILE_COL_SERVICE_IDX];
+						lineServiceIdOrInt = lineServiceIdWithQuotes.substring(1, lineServiceIdWithQuotes.length() - 1);
+					}
+					if (!serviceIdOrInts.contains(lineServiceIdOrInt)) {
 						continue;
 					}
 					lineDirectionId = Long.parseLong(lineItems[GTFS_ROUTE_FREQUENCY_FILE_COL_DIRECTION_IDX]);
@@ -700,14 +709,18 @@ public static Integer findLastServiceDate(@NonNull GTFSProvider provider) {
 	}
 
 	@NonNull
-	private static final String[] PROJECTION_SERVICE_DATES = new String[]{
-			GTFSCommons.T_SERVICE_DATES_K_SERVICE_ID,
-			GTFSCommons.T_SERVICE_DATES_K_EXCEPTION_TYPE
-	};
+	private static final String[] PROJECTION_SERVICE_DATES =
+			FeatureFlags.F_EXPORT_SERVICE_ID_INTS ? new String[]{
+					GTFSCommons.T_SERVICE_DATES_K_SERVICE_ID_INT,
+					GTFSCommons.T_SERVICE_DATES_K_EXCEPTION_TYPE
+			} : new String[]{
+					GTFSCommons.T_SERVICE_DATES_K_SERVICE_ID,
+					GTFSCommons.T_SERVICE_DATES_K_EXCEPTION_TYPE
+			};
 
 	@NonNull
 	private static HashSet<Pair<String, Integer>> findServicesAndExceptionTypes(@NonNull GTFSProvider provider, @NonNull String dateS) {
-		final HashSet<Pair<String, Integer>> serviceIdAndExceptionTypes = new HashSet<>();
+		final HashSet<Pair<String, Integer>> serviceIdOrIntAndExceptionTypes = new HashSet<>();
 		Cursor cursor = null;
 		try {
 			final String selection = SqlUtils.getWhereEquals(GTFSProviderDbHelper.T_SERVICE_DATES_K_DATE, dateS);
@@ -717,10 +730,15 @@ private static HashSet<Pair<String, Integer>> findServicesAndExceptionTypes(@Non
 			if (cursor != null && cursor.getCount() > 0) {
 				if (cursor.moveToFirst()) {
 					do {
-						final String serviceId = CursorExtKt.getString(cursor, GTFSProviderDbHelper.T_SERVICE_DATES_K_SERVICE_ID);
+						final String serviceIdOrInt;
+						if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) {
+							serviceIdOrInt = CursorExtKt.getString(cursor, GTFSProviderDbHelper.T_SERVICE_DATES_K_SERVICE_ID_INT);
+						} else {
+							serviceIdOrInt = CursorExtKt.getString(cursor, GTFSProviderDbHelper.T_SERVICE_DATES_K_SERVICE_ID);
+						}
 						final int exceptionType = CursorExtKt.optIntNN(cursor, GTFSProviderDbHelper.T_SERVICE_DATES_K_EXCEPTION_TYPE, GTFSCommons.EXCEPTION_TYPE_DEFAULT);
-						if (!TextUtils.isEmpty(serviceId)) {
-							serviceIdAndExceptionTypes.add(new Pair<>(serviceId, exceptionType));
+						if (!TextUtils.isEmpty(serviceIdOrInt)) {
+							serviceIdOrIntAndExceptionTypes.add(new Pair<>(serviceIdOrInt, exceptionType));
 						}
 					} while (cursor.moveToNext());
 				}
@@ -730,7 +748,7 @@ private static HashSet<Pair<String, Integer>> findServicesAndExceptionTypes(@Non
 		} finally {
 			SqlUtils.closeQuietly(cursor);
 		}
-		return serviceIdAndExceptionTypes;
+		return serviceIdOrIntAndExceptionTypes;
 	}
 
 	public static void cacheStatusS(@NonNull GTFSProvider provider, @NonNull POIStatus newStatusToCache) {

src/main/res/values/gtfs_rts_values.xml

@@ -17,6 +17,7 @@
     <string name="gtfs_rts_area_min_lng" /> <!-- empty: set in module app -->
     <string name="gtfs_rts_area_max_lng" /> <!-- empty: set in module app -->
     <string name="gtfs_rts_color" /> <!-- empty: set in module app -->
+    <string name="gtfs_rts_service_id_cleanup_regex" /> <!-- empty: set in module app -->
     <string name="gtfs_rts_route_id_cleanup_regex" /> <!-- empty: set in module app -->
     <string name="gtfs_rts_trip_id_cleanup_regex" /> <!-- empty: set in module app -->
     <string name="gtfs_rts_stop_id_cleanup_regex" /> <!-- empty: set in module app -->