From 1fd125a899674716b74f0fe6fc4246c338d6331f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Wed, 12 Nov 2025 21:47:26 -0500 Subject: [PATCH 01/13] Index Service IDs --- .../mtransit/parser/DefaultAgencyTools.java | 4 ++ .../mtransit/parser/gtfs/data/GCalendar.kt | 4 +- .../parser/gtfs/data/GCalendarDate.kt | 4 +- .../mtransit/parser/gtfs/data/GServiceIds.kt | 48 ++++++++++++++ .../org/mtransit/parser/gtfs/data/GTrip.kt | 6 +- .../org/mtransit/parser/mt/MGenerator.java | 66 +++++++++++++++++++ .../java/org/mtransit/parser/mt/MReader.kt | 27 ++++++++ .../org/mtransit/parser/mt/data/MFrequency.kt | 5 +- .../org/mtransit/parser/mt/data/MSchedule.kt | 5 +- .../mtransit/parser/mt/data/MServiceDate.kt | 8 +-- .../org/mtransit/parser/mt/data/MServiceId.kt | 34 ++++++++++ 11 files changed, 190 insertions(+), 21 deletions(-) create mode 100644 src/main/java/org/mtransit/parser/gtfs/data/GServiceIds.kt create mode 100644 src/main/java/org/mtransit/parser/mt/data/MServiceId.kt diff --git a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java index 78ca3da2..2f705384 100644 --- a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java +++ b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java @@ -22,6 +22,7 @@ import org.mtransit.parser.gtfs.data.GPickupType; import org.mtransit.parser.gtfs.data.GRoute; import org.mtransit.parser.gtfs.data.GRouteType; +import org.mtransit.parser.gtfs.data.GServiceIds; import org.mtransit.parser.gtfs.data.GSpec; import org.mtransit.parser.gtfs.data.GStop; import org.mtransit.parser.gtfs.data.GStopTime; @@ -35,6 +36,7 @@ import org.mtransit.parser.mt.data.MRoute; import org.mtransit.parser.mt.data.MRouteSNToIDConverter; import org.mtransit.parser.mt.data.MServiceDate; +import org.mtransit.parser.mt.data.MServiceId; import org.mtransit.parser.mt.data.MSpec; import org.mtransit.parser.mt.data.MDirection; @@ -147,6 +149,8 @@ public void start(@NotNull String[] args) { MTLog.log("Generating data..."); MTLog.logDebug("Args [%d]: %s.", args.length, Arrays.asList(args)); final List lastServiceDates = MReader.loadServiceDates(args[2]); + final List lastServiceIds = MReader.loadServiceIds(args[2]); + GServiceIds.addAll(lastServiceIds); this.serviceIdInts = extractUsefulServiceIdInts(args, this, true, lastServiceDates); final String inputUrl = args.length >= 5 ? args[4] : null; if (excludingAll()) { diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GCalendar.kt b/src/main/java/org/mtransit/parser/gtfs/data/GCalendar.kt index 1fd5da60..8fdf934e 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GCalendar.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GCalendar.kt @@ -73,9 +73,7 @@ data class GCalendar( val serviceId = _serviceId private val _serviceId: String - get() { - return GIDs.getString(serviceIdInt) - } + get() = GServiceIds.getId(serviceIdInt) val escapedServiceId: String get() = _serviceId.escape() diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GCalendarDate.kt b/src/main/java/org/mtransit/parser/gtfs/data/GCalendarDate.kt index ebfe013c..861588cb 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GCalendarDate.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GCalendarDate.kt @@ -45,9 +45,7 @@ data class GCalendarDate( val serviceId = _serviceId private val _serviceId: String - get() { - return GIDs.getString(serviceIdInt) - } + get() = GServiceIds.getId(serviceIdInt) val escapedServiceId: String get() = _serviceId.escape() diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GServiceIds.kt b/src/main/java/org/mtransit/parser/gtfs/data/GServiceIds.kt new file mode 100644 index 00000000..f994fad7 --- /dev/null +++ b/src/main/java/org/mtransit/parser/gtfs/data/GServiceIds.kt @@ -0,0 +1,48 @@ +package org.mtransit.parser.gtfs.data + +import androidx.collection.SparseArrayCompat +import androidx.collection.mutableScatterMapOf +import org.mtransit.parser.MTLog +import org.mtransit.parser.mt.data.MServiceId + +object GServiceIds { + + private var increment = 0 + private val intToString = SparseArrayCompat() + private val stringToInt = mutableScatterMapOf() + + @JvmStatic + fun addAll(lastServiceIds: List?) { + lastServiceIds?.forEach { add(it) } + } + + fun add(serviceId: MServiceId) { + intToString.put(serviceId.serviceIdInt, serviceId.serviceId) + stringToInt[serviceId.serviceId] = serviceId.serviceIdInt + increment = maxOf(increment, serviceId.serviceIdInt) + } + + fun add(serviceId: String): Int { + increment++ // move to next + val newServiceId = MServiceId(serviceId, increment) + add(newServiceId) + return newServiceId.serviceIdInt + } + + @JvmStatic + fun getId(serviceIdInt: Int): String { + return intToString[serviceIdInt] ?: throw MTLog.Fatal("Unexpected Service ID integer $serviceIdInt!") + } + + @JvmStatic + fun getInt(serviceId: String): Int { + return stringToInt[serviceId] ?: add(serviceId) + } + + @JvmStatic + fun getAll() = buildList { + stringToInt.forEach { id, idInt -> + add(MServiceId(id, idInt)) + } + } +} diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt b/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt index edcb5f86..c719a7f9 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt @@ -37,7 +37,7 @@ data class GTrip( tripIdInt = GIDs.getInt(tripId), routeIdInt = GIDs.getInt(routeId), originalRouteIdInt = GIDs.getInt(originalRouteId), - serviceIdInt = GIDs.getInt(serviceId), + serviceIdInt = GServiceIds.getInt(serviceId), tripHeadsign = tripHeadsign, tripShortName = tripShortName, directionIdE = GDirectionId.parse(directionId), @@ -94,9 +94,7 @@ data class GTrip( val serviceId = _serviceId private val _serviceId: String - get() { - return GIDs.getString(serviceIdInt) - } + get() = GServiceIds.getId(serviceIdInt) @Suppress("unused") private fun getCleanServiceId(agencyTools: GAgencyTools): String { diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index 417c7b8b..fa58bfe0 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -23,12 +23,14 @@ import org.mtransit.parser.gtfs.GAgencyTools; import org.mtransit.parser.gtfs.GReader; import org.mtransit.parser.gtfs.data.GFieldTypes; +import org.mtransit.parser.gtfs.data.GServiceIds; import org.mtransit.parser.gtfs.data.GSpec; import org.mtransit.parser.mt.data.MAgency; import org.mtransit.parser.mt.data.MFrequency; import org.mtransit.parser.mt.data.MRoute; import org.mtransit.parser.mt.data.MSchedule; import org.mtransit.parser.mt.data.MServiceDate; +import org.mtransit.parser.mt.data.MServiceId; import org.mtransit.parser.mt.data.MSpec; import org.mtransit.parser.mt.data.MStop; import org.mtransit.parser.mt.data.MDirection; @@ -198,6 +200,7 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age private static final String GTFS_SCHEDULE = "gtfs_schedule"; private static final String GTFS_SCHEDULE_SERVICE_DATES = GTFS_SCHEDULE + "_service_dates"; // DB + private static final String GTFS_SCHEDULE_SERVICE_IDS = GTFS_SCHEDULE + "_service_ids"; // DB private static final String GTFS_SCHEDULE_STOP = GTFS_SCHEDULE + "_stop_"; // file private static final String GTFS_FREQUENCY = "gtfs_frequency"; private static final String GTFS_FREQUENCY_ROUTE = GTFS_FREQUENCY + "_route_"; // file @@ -267,6 +270,8 @@ public static void dumpFiles(@NotNull GAgencyTools gAgencyTools, // STOPS Pair, Pair> minMaxLatLng = dumpRDSStops(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); + // SERVICE IDS + dumpScheduleServiceIds(gAgencyTools, mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); // SCHEDULE SERVICE DATES Pair minMaxDates = dumpScheduleServiceDates(gAgencyTools, mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); @@ -513,6 +518,67 @@ private static Pair, Pair> dumpRDSStops(@Nu return minMaxLatLng; } + @NotNull + private static Pair dumpScheduleServiceIds(@NotNull GAgencyTools gAgencyTools, + @Nullable MSpec mSpec, + @NotNull String fileBase, + boolean deleteAll, + @NotNull File dataDirF, + @NotNull File rawDirF, + @Nullable Connection dbConnection) { + if (!deleteAll + && (mSpec == null || !mSpec.isValid() || (F_PRE_FILLED_DB && dbConnection == null))) { + throw new MTLog.Fatal("Generated data invalid (agencies: %s)!", mSpec); + } + Pair minMaxDates = new Pair<>(null, null); + if (F_PRE_FILLED_DB) { + FileUtils.deleteIfExist(new File(rawDirF, fileBase + GTFS_SCHEDULE_SERVICE_IDS)); // migration from src/main/res/raw to data + } + File file = new File(F_PRE_FILLED_DB ? dataDirF : rawDirF, fileBase + GTFS_SCHEDULE_SERVICE_IDS); + FileUtils.deleteIfExist(file); // delete previous + BufferedWriter ow = null; + try { + if (!deleteAll) { + ow = new BufferedWriter(new FileWriter(file)); + MTLog.logPOINT(); // LOG + Integer minDate = null, maxDate = null; + Statement dbStatement = null; + String sqlInsert = null; + if (F_PRE_FILLED_DB) { + SQLUtils.setAutoCommit(dbConnection, false); // START TRANSACTION + dbStatement = dbConnection.createStatement(); + sqlInsert = GTFSCommons.getT_SERVICE_DATES_SQL_INSERT(); + } + for (MServiceId mServiceId : GServiceIds.getAll()) { // TODO? mSpec.getServiceIds() + final String serviceIdsInsert = mServiceId.toFile(); + if (F_PRE_FILLED_DB) { + SQLUtils.executeUpdate( + dbStatement, + String.format(sqlInsert, serviceIdsInsert) + ); + } + ow.write(serviceIdsInsert); + ow.write(Constants.NEW_LINE); + if (minDate == null || minDate > mServiceDate.getCalendarDate()) { + minDate = mServiceDate.getCalendarDate(); + } + if (maxDate == null || maxDate.doubleValue() < mServiceDate.getCalendarDate()) { + maxDate = mServiceDate.getCalendarDate(); + } + } + if (F_PRE_FILLED_DB) { + SQLUtils.setAutoCommit(dbConnection, true); // END TRANSACTION == commit() + } + minMaxDates = new Pair<>(minDate, maxDate); + } + } catch (Exception ioe) { + throw new MTLog.Fatal(ioe, "I/O Error while writing service dates file!"); + } finally { + CloseableUtils.closeQuietly(ow); + } + return minMaxDates; + } + @NotNull private static Pair dumpScheduleServiceDates(@NotNull GAgencyTools gAgencyTools, @Nullable MSpec mSpec, diff --git a/src/main/java/org/mtransit/parser/mt/MReader.kt b/src/main/java/org/mtransit/parser/mt/MReader.kt index 7613d9dc..f7b2d49e 100644 --- a/src/main/java/org/mtransit/parser/mt/MReader.kt +++ b/src/main/java/org/mtransit/parser/mt/MReader.kt @@ -13,6 +13,7 @@ import org.mtransit.parser.MTLog import org.mtransit.parser.Pair import org.mtransit.parser.gtfs.data.GFieldTypes import org.mtransit.parser.mt.data.MServiceDate +import org.mtransit.parser.mt.data.MServiceId import java.io.File import java.util.TimeZone @@ -140,4 +141,30 @@ object MReader { } // endregion + + // region service IDs + + private const val GTFS_SCHEDULE_SERVICE_IDS = "gtfs_schedule_service_ids" + + @JvmStatic + fun loadServiceIds(fileBase: String): List? { + try { + val gtfsScheduleServiceIds = getResDirName(fileBase) + "/$RAW/${fileBase}$GTFS_SCHEDULE_SERVICE_IDS" + val gtfsScheduleServiceIdsFile = File(gtfsScheduleServiceIds) + if (!gtfsScheduleServiceIdsFile.exists()) { + MTLog.log("File not found '$gtfsScheduleServiceIds'!") + return null + } + val gtfsScheduleServiceIdsFileLines = gtfsScheduleServiceIdsFile.readLines() + val serviceIds = gtfsScheduleServiceIdsFileLines.mapNotNull { line -> + MServiceId.fromFileLine(line) + } + return serviceIds + } catch (e: Exception) { + MTLog.logNonFatal(e, "Error while reading '$fileBase' service ids!") + return null + } + } + + // endregion } \ No newline at end of file diff --git a/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt b/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt index 2efccc0f..d8f138b2 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt @@ -5,6 +5,7 @@ import org.mtransit.parser.Constants import org.mtransit.parser.db.SQLUtils.quotesEscape import org.mtransit.parser.gtfs.GAgencyTools import org.mtransit.parser.gtfs.data.GIDs +import org.mtransit.parser.gtfs.data.GServiceIds data class MFrequency( val serviceIdInt: Int, @@ -19,9 +20,7 @@ data class MFrequency( val serviceId = _serviceId private val _serviceId: String - get() { - return GIDs.getString(serviceIdInt) - } + get() = GServiceIds.getId(serviceIdInt) private fun getCleanServiceId(agencyTools: GAgencyTools): String { return agencyTools.cleanServiceId(_serviceId) diff --git a/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt b/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt index 1af0fc48..fff99659 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt @@ -9,6 +9,7 @@ import org.mtransit.parser.db.SQLUtils.quotes import org.mtransit.parser.db.SQLUtils.quotesEscape import org.mtransit.parser.gtfs.GAgencyTools import org.mtransit.parser.gtfs.data.GIDs +import org.mtransit.parser.gtfs.data.GServiceIds data class MSchedule( val routeId: Long, @@ -49,9 +50,7 @@ data class MSchedule( val serviceId = _serviceId private val _serviceId: String - get() { - return GIDs.getString(serviceIdInt) - } + get() = GServiceIds.getId(serviceIdInt) @Discouraged(message = "Not memory efficient") @Suppress("unused") diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt index eaae0a8e..8dada979 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt @@ -7,7 +7,7 @@ import org.mtransit.parser.db.SQLUtils.unquotes import org.mtransit.parser.gtfs.GAgencyTools import org.mtransit.parser.gtfs.data.GCalendarDate import org.mtransit.parser.gtfs.data.GCalendarDatesExceptionType -import org.mtransit.parser.gtfs.data.GIDs +import org.mtransit.parser.gtfs.data.GServiceIds data class MServiceDate( val serviceIdInt: Int, @@ -30,9 +30,7 @@ data class MServiceDate( val serviceId = _serviceId private val _serviceId: String - get() { - return GIDs.getString(serviceIdInt) - } + get() = GServiceIds.getId(serviceIdInt) override fun compareTo(other: MServiceDate): Int = compareBy( MServiceDate::calendarDate, @@ -76,7 +74,7 @@ data class MServiceDate( .takeIf { it.size == 3 } ?.let { columns -> MServiceDate( - serviceIdInt = GIDs.getInt(columns[0].unquotes()), // service ID + serviceIdInt = GServiceIds.getInt(columns[0].unquotes()), // service ID calendarDate = columns[1].toInt(), // calendar date exceptionType = columns[2].toInt() ) diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt new file mode 100644 index 00000000..bee54bbf --- /dev/null +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt @@ -0,0 +1,34 @@ +package org.mtransit.parser.mt.data + +import org.mtransit.parser.Constants +import org.mtransit.parser.db.SQLUtils.quotesEscape +import org.mtransit.parser.db.SQLUtils.unquotes + +data class MServiceId( + val serviceId: String, + val serviceIdInt: Int, +) : Comparable { + + fun toFile() = buildList { + add(serviceId.quotesEscape()) // service ID + add(serviceIdInt.toString()) // service ID Int + }.joinToString(Constants.COLUMN_SEPARATOR_) + + override fun compareTo(other: MServiceId): Int = + // sort by service_id => service_id_int + when { + serviceId != other.serviceId -> serviceId.compareTo(other.serviceId) + else -> serviceIdInt.compareTo(other.serviceIdInt) + } + + companion object { + fun fromFileLine(line: String) = line.split(Constants.COLUMN_SEPARATOR) + .takeIf { it.size == 2 } + ?.let { columns -> + MServiceId( + serviceId = columns[0].unquotes(), + serviceIdInt = columns[1].toInt(), + ) + } + } +} From 25164676bc6445e9de5e69654083243dc8564dd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Wed, 12 Nov 2025 21:53:25 -0500 Subject: [PATCH 02/13] wip --- .../org/mtransit/parser/mt/MGenerator.java | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index fa58bfe0..27d7e66d 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -518,19 +518,18 @@ private static Pair, Pair> dumpRDSStops(@Nu return minMaxLatLng; } - @NotNull - private static Pair dumpScheduleServiceIds(@NotNull GAgencyTools gAgencyTools, - @Nullable MSpec mSpec, - @NotNull String fileBase, - boolean deleteAll, - @NotNull File dataDirF, - @NotNull File rawDirF, - @Nullable Connection dbConnection) { + private static void dumpScheduleServiceIds( + @NotNull GAgencyTools gAgencyTools, + @Nullable MSpec mSpec, + @NotNull String fileBase, + boolean deleteAll, + @NotNull File dataDirF, + @NotNull File rawDirF, + @Nullable Connection dbConnection) { if (!deleteAll && (mSpec == null || !mSpec.isValid() || (F_PRE_FILLED_DB && dbConnection == null))) { throw new MTLog.Fatal("Generated data invalid (agencies: %s)!", mSpec); } - Pair minMaxDates = new Pair<>(null, null); if (F_PRE_FILLED_DB) { FileUtils.deleteIfExist(new File(rawDirF, fileBase + GTFS_SCHEDULE_SERVICE_IDS)); // migration from src/main/res/raw to data } @@ -541,7 +540,6 @@ private static Pair dumpScheduleServiceIds(@NotNull GAgencyToo if (!deleteAll) { ow = new BufferedWriter(new FileWriter(file)); MTLog.logPOINT(); // LOG - Integer minDate = null, maxDate = null; Statement dbStatement = null; String sqlInsert = null; if (F_PRE_FILLED_DB) { @@ -551,6 +549,7 @@ private static Pair dumpScheduleServiceIds(@NotNull GAgencyToo } for (MServiceId mServiceId : GServiceIds.getAll()) { // TODO? mSpec.getServiceIds() final String serviceIdsInsert = mServiceId.toFile(); + // gAgencyTools if (F_PRE_FILLED_DB) { SQLUtils.executeUpdate( dbStatement, @@ -559,24 +558,16 @@ private static Pair dumpScheduleServiceIds(@NotNull GAgencyToo } ow.write(serviceIdsInsert); ow.write(Constants.NEW_LINE); - if (minDate == null || minDate > mServiceDate.getCalendarDate()) { - minDate = mServiceDate.getCalendarDate(); - } - if (maxDate == null || maxDate.doubleValue() < mServiceDate.getCalendarDate()) { - maxDate = mServiceDate.getCalendarDate(); - } } if (F_PRE_FILLED_DB) { SQLUtils.setAutoCommit(dbConnection, true); // END TRANSACTION == commit() } - minMaxDates = new Pair<>(minDate, maxDate); } } catch (Exception ioe) { throw new MTLog.Fatal(ioe, "I/O Error while writing service dates file!"); } finally { CloseableUtils.closeQuietly(ow); } - return minMaxDates; } @NotNull From fc61213b37c7038c7eda88e785c96f98188e547c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Fri, 14 Nov 2025 21:11:07 -0500 Subject: [PATCH 03/13] wip --- .../mtransit/parser/DefaultAgencyTools.java | 76 +++++++++++-------- .../org/mtransit/parser/db/DumpDbUtils.kt | 7 ++ .../mtransit/parser/gtfs/GAgencyTools.java | 15 +--- .../mtransit/parser/gtfs/data/GCalendar.kt | 8 +- .../parser/gtfs/data/GCalendarDate.kt | 7 +- .../org/mtransit/parser/gtfs/data/GSpecExt.kt | 4 +- .../org/mtransit/parser/gtfs/data/GTrip.kt | 21 ++--- .../mtransit/parser/gtfs/data/GTripStop.kt | 9 +-- .../mtransit/parser/mt/MDataChangedManager.kt | 1 - .../org/mtransit/parser/mt/MGenerator.java | 55 +++++++------- .../org/mtransit/parser/mt/data/MFrequency.kt | 26 +++---- .../org/mtransit/parser/mt/data/MSchedule.kt | 32 ++++---- .../mtransit/parser/mt/data/MServiceDate.kt | 19 +++-- .../org/mtransit/parser/mt/data/MServiceId.kt | 14 ++-- .../GServiceIds.kt => mt/data/MServiceIds.kt} | 6 +- .../parser/DefaultAgencyToolsTest.java | 20 ++--- 16 files changed, 153 insertions(+), 167 deletions(-) rename src/main/java/org/mtransit/parser/{gtfs/data/GServiceIds.kt => mt/data/MServiceIds.kt} (92%) diff --git a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java index 2f705384..f7e01bff 100644 --- a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java +++ b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java @@ -22,7 +22,6 @@ import org.mtransit.parser.gtfs.data.GPickupType; import org.mtransit.parser.gtfs.data.GRoute; import org.mtransit.parser.gtfs.data.GRouteType; -import org.mtransit.parser.gtfs.data.GServiceIds; import org.mtransit.parser.gtfs.data.GSpec; import org.mtransit.parser.gtfs.data.GStop; import org.mtransit.parser.gtfs.data.GStopTime; @@ -37,6 +36,7 @@ import org.mtransit.parser.mt.data.MRouteSNToIDConverter; import org.mtransit.parser.mt.data.MServiceDate; import org.mtransit.parser.mt.data.MServiceId; +import org.mtransit.parser.mt.data.MServiceIds; import org.mtransit.parser.mt.data.MSpec; import org.mtransit.parser.mt.data.MDirection; @@ -53,7 +53,6 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; -@SuppressWarnings({"RedundantSuppression", "unused", "WeakerAccess"}) public class DefaultAgencyTools implements GAgencyTools { static { @@ -150,15 +149,15 @@ public void start(@NotNull String[] args) { MTLog.logDebug("Args [%d]: %s.", args.length, Arrays.asList(args)); final List lastServiceDates = MReader.loadServiceDates(args[2]); final List lastServiceIds = MReader.loadServiceIds(args[2]); - GServiceIds.addAll(lastServiceIds); + MServiceIds.addAll(lastServiceIds); this.serviceIdInts = extractUsefulServiceIdInts(args, this, true, lastServiceDates); final String inputUrl = args.length >= 5 ? args[4] : null; if (excludingAll()) { MGenerator.dumpFiles(this, null, args[0], args[1], args[2], inputUrl, true); return; } - long start = System.currentTimeMillis(); - GSpec gtfs = GReader.readGtfsZipFile(args[0], this, false, false); + final long start = System.currentTimeMillis(); + final GSpec gtfs = GReader.readGtfsZipFile(args[0], this, false, false); MDataChangedManager.avoidCalendarDatesDataChanged(lastServiceDates, gtfs); gtfs.cleanupStops(); gtfs.cleanupExcludedData(); @@ -169,7 +168,7 @@ public void start(@NotNull String[] args) { } gtfs.splitByRouteId(this); gtfs.clearRawData(); - MSpec mSpec = MGenerator.generateMSpec(gtfs, this); + final MSpec mSpec = MGenerator.generateMSpec(gtfs, this); if (Constants.SKIP_FILE_DUMP) { return; // DEBUG } @@ -201,12 +200,14 @@ public List getSupportedLanguages() { return null; // no-op } + @SuppressWarnings("WeakerAccess") @Nullable public Locale getFirstLanguage() { final List supportedLanguages = getSupportedLanguages(); return supportedLanguages == null || supportedLanguages.isEmpty() ? null : supportedLanguages.get(0); } + @SuppressWarnings("WeakerAccess") @NotNull public Locale getFirstLanguageNN() { final Locale firstLanguage = getFirstLanguage(); @@ -225,6 +226,7 @@ public String getAgencyName() { return "Agency Name"; } + @SuppressWarnings("unused") @NotNull private String getAgencyType() { final int type = getAgencyRouteType(); @@ -344,8 +346,27 @@ public boolean excludeAgency(@NotNull GAgency gAgency) { @NotNull @Override - public String cleanServiceId(@NotNull String serviceId) { - return serviceId; + public String cleanServiceId(@NotNull String gServiceId) { + return GTFSCommons.cleanOriginalId(gServiceId, getServiceIdCleanupPattern()); + } + + @Nullable + @Override + public String getServiceIdCleanupRegex() { + return null; // opt-in + } + + @Nullable + private Pattern serviceIdCleanupPattern = null; + + private boolean serviceIdCleanupPatternSet = false; + + private @Nullable Pattern getServiceIdCleanupPattern() { + if (this.serviceIdCleanupPattern == null && !serviceIdCleanupPatternSet) { + this.serviceIdCleanupPattern = GTFSCommons.makeIdCleanupPattern(getServiceIdCleanupRegex()); + this.serviceIdCleanupPatternSet = true; + } + return this.serviceIdCleanupPattern; } @NotNull @@ -423,8 +444,7 @@ public String getRouteIdCleanupRegex() { private boolean routeIdCleanupPatternSet = false; - @Override - public @Nullable Pattern getRouteIdCleanupPattern() { + private @Nullable Pattern getRouteIdCleanupPattern() { if (this.routeIdCleanupPattern == null && !routeIdCleanupPatternSet) { this.routeIdCleanupPattern = GTFSCommons.makeIdCleanupPattern(getRouteIdCleanupRegex()); this.routeIdCleanupPatternSet = true; @@ -438,7 +458,7 @@ public String getRouteShortName(@NotNull GRoute gRoute) { //noinspection DiscouragedApi final String routeShortName = useRouteIdForRouteShortName() ? gRoute.getRouteId() - : gRoute.getRouteShortName(); + : gRoute.getRouteShortName(); if (org.mtransit.commons.StringUtils.isEmpty(routeShortName)) { return provideMissingRouteShortName(gRoute); } @@ -631,8 +651,7 @@ public String getTripIdCleanupRegex() { private boolean tripIdCleanupPatternSet = false; - @Override - public @Nullable Pattern getTripIdCleanupPattern() { + private @Nullable Pattern getTripIdCleanupPattern() { if (this.tripIdCleanupPattern == null && !tripIdCleanupPatternSet) { this.tripIdCleanupPattern = GTFSCommons.makeIdCleanupPattern(getTripIdCleanupRegex()); this.tripIdCleanupPatternSet = true; @@ -1009,8 +1028,7 @@ public boolean useStopCodeForStopId() { private boolean stopIdCleanupPatternSet = false; - @Override - public @Nullable Pattern getStopIdCleanupPattern() { + private @Nullable Pattern getStopIdCleanupPattern() { if (this.stopIdCleanupPattern == null && !stopIdCleanupPatternSet) { this.stopIdCleanupPattern = GTFSCommons.makeIdCleanupPattern(getStopIdCleanupRegex()); this.stopIdCleanupPatternSet = true; @@ -1230,7 +1248,7 @@ public static HashSet extractUsefulServiceIdInts( if (!gCalendars.isEmpty()) { parseCalendars(gCalendars, gCalendarDates, DATE_FORMAT, c, usefulPeriod, isCurrentOrNext); // CURRENT } else if (!gCalendarDates.isEmpty()) { - parseCalendarDates(gCalendarDates, lastServiceDates, c, usefulPeriod, isCurrentOrNext); // CURRENT + parseCalendarDates(gCalendarDates, c, usefulPeriod, isCurrentOrNext); // CURRENT } else { throw new MTLog.Fatal("NO schedule available for %s! (1)", usefulPeriod.getTodayStringInt()); } @@ -1278,7 +1296,7 @@ public static HashSet extractUsefulServiceIdInts( if (!gCalendars.isEmpty()) { parseCalendars(gCalendars, gCalendarDates, DATE_FORMAT, c, usefulPeriod, false); // NEXT } else if (!gCalendarDates.isEmpty()) { - parseCalendarDates(gCalendarDates, lastServiceDates, c, usefulPeriod, false); // NEXT + parseCalendarDates(gCalendarDates, c, usefulPeriod, false); // NEXT } if (usefulPeriod.getStartDate() == null || usefulPeriod.getEndDate() == null) { MTLog.log("* NO NEXT schedule available for %s. (start:%s|end:%s)", usefulPeriod.getTodayStringInt(), usefulPeriod.getStartDate(), usefulPeriod.getEndDate()); @@ -1292,7 +1310,7 @@ public static HashSet extractUsefulServiceIdInts( MTLog.log("* Generated on %s | NEXT Schedules from %s to %s.", usefulPeriod.getTodayStringInt(), usefulPeriod.getStartDate(), usefulPeriod.getEndDate()); MTLog.log("------------------------------"); } - HashSet serviceIds = getPeriodServiceIds(usefulPeriod.getStartDate(), usefulPeriod.getEndDate(), gCalendars, gCalendarDates); + final HashSet serviceIds = getPeriodServiceIds(usefulPeriod.getStartDate(), usefulPeriod.getEndDate(), gCalendars, gCalendarDates); improveUsefulPeriod(usefulPeriod, c, gCalendars, gCalendarDates); MTLog.log("Extracting useful service IDs... DONE"); //noinspection UnusedAssignment // FIXME @@ -1336,7 +1354,6 @@ private static void improveUsefulPeriod(@NotNull Period usefulPeriod, Calendar c } static void parseCalendarDates(List gCalendarDates, - @Nullable List lastServiceDates, Calendar c, Period p, boolean lookBackward) { @@ -1345,7 +1362,7 @@ static void parseCalendarDates(List gCalendarDates, MTLog.log("NO schedule available for %s in calendar dates.", p.getTodayStringInt()); return; } - refreshStartEndDatesFromCalendarDates(p, todayServiceIds, gCalendarDates, lastServiceDates); + refreshStartEndDatesFromCalendarDates(p, todayServiceIds, gCalendarDates); while (true) { MTLog.log("> Schedules from %s to %s... ", p.getStartDate(), p.getEndDate()); for (GCalendarDate gCalendarDate : gCalendarDates) { @@ -1357,7 +1374,7 @@ static void parseCalendarDates(List gCalendarDates, } } } - boolean newDates = refreshStartEndDatesFromCalendarDates(p, todayServiceIds, gCalendarDates, lastServiceDates); + boolean newDates = refreshStartEndDatesFromCalendarDates(p, todayServiceIds, gCalendarDates); if (newDates) { MTLog.log("> new start date '%s' & end date '%s' from calendar date active during service ID(s).", p.getStartDate(), p.getEndDate()); continue; @@ -1365,7 +1382,7 @@ static void parseCalendarDates(List gCalendarDates, Period pNext = new Period(); pNext.setTodayStringInt(incDateDays(DATE_FORMAT, c, p.getEndDate(), 1)); HashSet nextDayServiceIds = findCalendarDatesTodayServiceIds(gCalendarDates, c, pNext, false, 0, false); - refreshStartEndDatesFromCalendarDates(pNext, nextDayServiceIds, gCalendarDates, lastServiceDates); + refreshStartEndDatesFromCalendarDates(pNext, nextDayServiceIds, gCalendarDates); if (pNext.getStartDate() != null && pNext.getEndDate() != null && diffLowerThan(DATE_FORMAT, c, pNext.getStartDate(), pNext.getEndDate(), MIN_PREVIOUS_NEXT_ADDED_DAYS)) { p.setEndDate(pNext.getEndDate()); @@ -1375,7 +1392,7 @@ && diffLowerThan(DATE_FORMAT, c, pNext.getStartDate(), pNext.getEndDate(), MIN_P Period pPrevious = new Period(); pPrevious.setTodayStringInt(incDateDays(DATE_FORMAT, c, p.getStartDate(), -1)); HashSet previousDayServiceIds = findCalendarDatesTodayServiceIds(gCalendarDates, c, pPrevious, false, 0, false); - refreshStartEndDatesFromCalendarDates(pPrevious, previousDayServiceIds, gCalendarDates, lastServiceDates); + refreshStartEndDatesFromCalendarDates(pPrevious, previousDayServiceIds, gCalendarDates); if (lookBackward // NOT next schedule, only current schedule can look behind && pPrevious.getStartDate() != null && pPrevious.getEndDate() != null && diffLowerThan(DATE_FORMAT, c, pPrevious.getStartDate(), pPrevious.getEndDate(), MIN_PREVIOUS_NEXT_ADDED_DAYS)) { @@ -1424,8 +1441,8 @@ && diffLowerThan(DATE_FORMAT, c, pPrevious.getStartDate(), pPrevious.getEndDate( private static boolean refreshStartEndDatesFromCalendarDates( Period p, HashSet serviceIds, - List gCalendarDates, - @Nullable List lastServiceDates) { + List gCalendarDates + ) { boolean newDates = false; for (GCalendarDate gCalendarDate : gCalendarDates) { if (gCalendarDate.isServiceIdInts(serviceIds)) { @@ -1480,7 +1497,7 @@ && diffLowerThan(DATE_FORMAT, c, p.getTodayStringInt(), initialTodayStringInt, M } static void parseCalendars(@NotNull List gCalendars, @Nullable List gCalendarDates, SimpleDateFormat DATE_FORMAT, Calendar c, Period p, boolean lookBackward) { - findCalendarsTodayPeriod(gCalendars, gCalendarDates, DATE_FORMAT, c, p, lookBackward); + findCalendarsTodayPeriod(gCalendars, gCalendarDates, p, lookBackward); if (p.getStartDate() == null || p.getEndDate() == null) { MTLog.log("NO schedule available for %s in calendars. (start:%s|end:%s)", p.getTodayStringInt(), p.getStartDate(), p.getEndDate()); return; @@ -1541,9 +1558,8 @@ && diffLowerThan(DATE_FORMAT, c, pNext.getStartDate(), pNext.getEndDate(), MIN_P } } - static void findCalendarsTodayPeriod(List gCalendars, List gCalendarDates, SimpleDateFormat DATE_FORMAT, Calendar c, Period p, boolean lookBackward) { + static void findCalendarsTodayPeriod(List gCalendars, List gCalendarDates, Period p, boolean lookBackward) { final Integer initialTodayStringInt = p.getTodayStringInt(); - boolean newDates; final Period entirePeriod = getEntirePeriodMinMaxDate(gCalendars, gCalendarDates); while (true) { for (GCalendar gCalendar : gCalendars) { @@ -1558,14 +1574,10 @@ static void findCalendarsTodayPeriod(List gCalendars, List by lazy { initAllDates( serviceIdInt, diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GCalendarDate.kt b/src/main/java/org/mtransit/parser/gtfs/data/GCalendarDate.kt index 861588cb..0d85c560 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GCalendarDate.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GCalendarDate.kt @@ -45,7 +45,7 @@ data class GCalendarDate( val serviceId = _serviceId private val _serviceId: String - get() = GServiceIds.getId(serviceIdInt) + get() = GIDs.getString(serviceIdInt) val escapedServiceId: String get() = _serviceId.escape() @@ -53,11 +53,6 @@ data class GCalendarDate( val escapedServiceIdInt: Int get() = escapedServiceId.toGIDInt() - @Suppress("unused") - fun getCleanServiceId(agencyTools: GAgencyTools): String { - return agencyTools.cleanServiceId(_serviceId) - } - val uID by lazy { getNewUID(date, serviceIdInt) } @Suppress("unused") diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GSpecExt.kt b/src/main/java/org/mtransit/parser/gtfs/data/GSpecExt.kt index bef2ff18..e4816a85 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GSpecExt.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GSpecExt.kt @@ -1,5 +1,3 @@ package org.mtransit.parser.gtfs.data -fun GSpec.getRoute(gTrip: GTrip): GRoute? { - return this.getRoute(gTrip.routeIdInt) -} \ No newline at end of file +fun GSpec.getRoute(gTrip: GTrip) = this.getRoute(gTrip.routeIdInt) diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt b/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt index c719a7f9..fbb31814 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt @@ -37,7 +37,7 @@ data class GTrip( tripIdInt = GIDs.getInt(tripId), routeIdInt = GIDs.getInt(routeId), originalRouteIdInt = GIDs.getInt(originalRouteId), - serviceIdInt = GServiceIds.getInt(serviceId), + serviceIdInt = GIDs.getInt(serviceId), tripHeadsign = tripHeadsign, tripShortName = tripShortName, directionIdE = GDirectionId.parse(directionId), @@ -76,30 +76,21 @@ data class GTrip( val routeId = _routeId private val _routeId: String - get() { - return GIDs.getString(routeIdInt) - } + get() = GIDs.getString(routeIdInt) @Discouraged(message = "Not memory efficient") @Suppress("unused") val originalRouteId = _originalRouteId private val _originalRouteId: String - get() { - return GIDs.getString(originalRouteIdInt) - } + get() = GIDs.getString(originalRouteIdInt) @Discouraged(message = "Not memory efficient") @Suppress("unused") val serviceId = _serviceId private val _serviceId: String - get() = GServiceIds.getId(serviceIdInt) - - @Suppress("unused") - private fun getCleanServiceId(agencyTools: GAgencyTools): String { - return agencyTools.cleanServiceId(_serviceId) - } + get() = GIDs.getString(serviceIdInt) @Discouraged(message = "Not memory efficient") @Suppress("unused") @@ -107,9 +98,7 @@ data class GTrip( @Suppress("unused") private val _tripId: String - get() { - return GIDs.getString(tripIdInt) - } + get() = GIDs.getString(tripIdInt) fun isServiceIdInts(serviceIdInts: Collection): Boolean { return serviceIdInts.contains(serviceIdInt) diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GTripStop.kt b/src/main/java/org/mtransit/parser/gtfs/data/GTripStop.kt index 4c2227e8..016700a0 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GTripStop.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GTripStop.kt @@ -2,7 +2,6 @@ package org.mtransit.parser.gtfs.data import androidx.annotation.Discouraged -// https://developers.google.com/transit/gtfs/reference#stop_timestxt // https://gtfs.org/reference/static#stop_timestxt // -_trip_id field // - stop_id field @@ -56,18 +55,14 @@ data class GTripStop( @Suppress("unused") private val _tripId: String - get() { - return GIDs.getString(tripIdInt) - } + get() = GIDs.getString(tripIdInt) @Discouraged(message = "Not memory efficient") @Suppress("unused") val stopId = _stopId private val _stopId: String - get() { - return GIDs.getString(stopIdInt) - } + get() = GIDs.getString(stopIdInt) @Suppress("unused") fun toStringPlus(): String { diff --git a/src/main/java/org/mtransit/parser/mt/MDataChangedManager.kt b/src/main/java/org/mtransit/parser/mt/MDataChangedManager.kt index 453cc01a..540d2b59 100644 --- a/src/main/java/org/mtransit/parser/mt/MDataChangedManager.kt +++ b/src/main/java/org/mtransit/parser/mt/MDataChangedManager.kt @@ -108,7 +108,6 @@ object MDataChangedManager { val (lastCalendarsServiceDates, lastCalendarDatesServiceDates) = if (ALL_CALENDARS_IN_CALENDAR_DATES) emptyList() to lastServiceDates // calendar dates only else lastServiceDates.partition { it.exceptionType == MCalendarExceptionType.DEFAULT.id } - @Suppress("DEPRECATION") val allCalendarsWithDays = if (ALL_CALENDARS_IN_CALENDAR_DATES) emptyList() else gtfs.allCalendars.filter { it.hasDays() } MTLog.log("> Service IDS from '${GCalendar.FILENAME}':") //noinspection DiscouragedApi diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index 27d7e66d..274ea054 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -23,7 +23,6 @@ import org.mtransit.parser.gtfs.GAgencyTools; import org.mtransit.parser.gtfs.GReader; import org.mtransit.parser.gtfs.data.GFieldTypes; -import org.mtransit.parser.gtfs.data.GServiceIds; import org.mtransit.parser.gtfs.data.GSpec; import org.mtransit.parser.mt.data.MAgency; import org.mtransit.parser.mt.data.MFrequency; @@ -31,6 +30,7 @@ import org.mtransit.parser.mt.data.MSchedule; import org.mtransit.parser.mt.data.MServiceDate; import org.mtransit.parser.mt.data.MServiceId; +import org.mtransit.parser.mt.data.MServiceIds; import org.mtransit.parser.mt.data.MSpec; import org.mtransit.parser.mt.data.MStop; import org.mtransit.parser.mt.data.MDirection; @@ -94,32 +94,29 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age for (Future future : list) { try { MSpec mRouteSpec = future.get(); - MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (merging...)", mRouteSpec.getFirstRoute().getId()); + final long mRouteId = mRouteSpec.getFirstRoute().getId(); + MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (merging...)", mRouteId); if (mRouteSpec.hasStops() && mRouteSpec.hasServiceDates()) { mAgencies.addAll(mRouteSpec.getAgencies()); mRoutes.addAll(mRouteSpec.getRoutes()); mDirections.addAll(mRouteSpec.getDirections()); mDirectionStops.addAll(mRouteSpec.getDirectionStops()); - MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (merging stops...)", mRouteSpec.getFirstRoute().getId()); + logMerging("stops...", mRouteId); for (MStop mStop : mRouteSpec.getStops()) { if (mStops.containsKey(mStop.getId())) { if (!mStops.get(mStop.getId()).equals(mStop)) { - MTLog.log("Stop ID '%s' already in list! (%s instead of %s)", mStop.getId(), mStops.get(mStop.getId()), mStop); + MTLog.log("%s: Stop ID '%s' already in list! (%s instead of %s)", mRouteId, mStop.getId(), mStops.get(mStop.getId()), mStop); } continue; } mStops.put(mStop.getId(), mStop); } - MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (merging stops... DONE)", mRouteSpec.getFirstRoute() - .getId()); - MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (merging service dates...)", mRouteSpec.getFirstRoute() - .getId()); + logMerging("stops... DONE", mRouteId); + logMerging("service dates...", mRouteId); mServiceDates.addAll(mRouteSpec.getServiceDates()); - MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (merging service dates... DONE)", mRouteSpec - .getFirstRoute().getId()); + logMerging("service dates... DONE", mRouteId); if (mRouteSpec.hasStopSchedules()) { - MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (merging stop schedules...)", mRouteSpec - .getFirstRoute().getId()); + logMerging("stop schedules...", mRouteId); if (mRouteSpec.getSchedules() != null) { DBUtils.setAutoCommit(false); for (MSchedule mSchedule : mRouteSpec.getSchedules()) { @@ -128,12 +125,10 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age DBUtils.setAutoCommit(true); // true => commit() } mRouteSpec.setSchedules(null); // clear - MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (merging stop schedules... DONE)", mRouteSpec - .getFirstRoute().getId()); + logMerging("stop schedules... DONE", mRouteId); } if (mRouteSpec.hasRouteFrequencies()) { - MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (merging route frequencies...)", mRouteSpec - .getFirstRoute().getId()); + logMerging("route frequencies...", mRouteId); for (Entry> routeFrequenciesEntry : mRouteSpec.getRouteFrequencies().entrySet()) { if (routeFrequenciesEntry.getValue() == null || routeFrequenciesEntry.getValue().isEmpty()) { continue; @@ -143,8 +138,7 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age } mRouteFrequencies.get(routeFrequenciesEntry.getKey()).addAll(routeFrequenciesEntry.getValue()); } - MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (merging route frequencies... DONE)", mRouteSpec - .getFirstRoute().getId()); + logMerging("route frequencies... DONE", mRouteId); } if (firstTimestamp < 0L || mRouteSpec.getFirstTimestamp() < firstTimestamp) { firstTimestamp = mRouteSpec.getFirstTimestamp(); @@ -153,9 +147,9 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age lastTimestamp = mRouteSpec.getLastTimestamp(); } } else { - MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (EMPTY)", mRouteSpec.getFirstRoute().getId()); + MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (EMPTY)", mRouteId); } - MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (merging... DONE)", mRouteSpec.getFirstRoute().getId()); + MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (merging... DONE)", mRouteId); } catch (Throwable t) { threadPoolExecutor.shutdownNow(); throw new MTLog.Fatal(t, t.getMessage()); @@ -163,17 +157,17 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age } MTLog.log("Generating routes, trips, trip stops & stops objects... (all routes completed)"); threadPoolExecutor.shutdown(); - ArrayList mAgenciesList = new ArrayList<>(mAgencies); + final ArrayList mAgenciesList = new ArrayList<>(mAgencies); Collections.sort(mAgenciesList); - ArrayList mStopsList = new ArrayList<>(mStops.values()); + final ArrayList mStopsList = new ArrayList<>(mStops.values()); Collections.sort(mStopsList); - ArrayList mRoutesList = new ArrayList<>(mRoutes); + final ArrayList mRoutesList = new ArrayList<>(mRoutes); Collections.sort(mRoutesList); - ArrayList mTripsList = new ArrayList<>(mDirections); + final ArrayList mTripsList = new ArrayList<>(mDirections); Collections.sort(mTripsList); - ArrayList mDirectionStopsList = new ArrayList<>(mDirectionStops); + final ArrayList mDirectionStopsList = new ArrayList<>(mDirectionStops); Collections.sort(mDirectionStopsList); - ArrayList mServiceDatesList = new ArrayList<>(mServiceDates); + final ArrayList mServiceDatesList = new ArrayList<>(mServiceDates); Collections.sort(mServiceDatesList); MTLog.log("Generating routes, trips, trip stops & stops objects... DONE"); MTLog.log("- Agencies: %d", mAgenciesList.size()); @@ -198,6 +192,11 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age ); } + private static void logMerging(@NotNull String msg, long routeId) { + if (true) return; // DEBUG + MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (merging %s)", routeId, msg); + } + private static final String GTFS_SCHEDULE = "gtfs_schedule"; private static final String GTFS_SCHEDULE_SERVICE_DATES = GTFS_SCHEDULE + "_service_dates"; // DB private static final String GTFS_SCHEDULE_SERVICE_IDS = GTFS_SCHEDULE + "_service_ids"; // DB @@ -545,9 +544,9 @@ private static void dumpScheduleServiceIds( if (F_PRE_FILLED_DB) { SQLUtils.setAutoCommit(dbConnection, false); // START TRANSACTION dbStatement = dbConnection.createStatement(); - sqlInsert = GTFSCommons.getT_SERVICE_DATES_SQL_INSERT(); + sqlInsert = GTFSCommons.getT_SERVICE_IDS_SQL_INSERT(); } - for (MServiceId mServiceId : GServiceIds.getAll()) { // TODO? mSpec.getServiceIds() + for (MServiceId mServiceId : MServiceIds.getAll()) { // TODO? mSpec.getServiceIds() final String serviceIdsInsert = mServiceId.toFile(); // gAgencyTools if (F_PRE_FILLED_DB) { diff --git a/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt b/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt index d8f138b2..1c54cec9 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt @@ -1,11 +1,11 @@ package org.mtransit.parser.mt.data import androidx.annotation.Discouraged +import org.mtransit.commons.FeatureFlags import org.mtransit.parser.Constants import org.mtransit.parser.db.SQLUtils.quotesEscape import org.mtransit.parser.gtfs.GAgencyTools import org.mtransit.parser.gtfs.data.GIDs -import org.mtransit.parser.gtfs.data.GServiceIds data class MFrequency( val serviceIdInt: Int, @@ -20,21 +20,21 @@ data class MFrequency( val serviceId = _serviceId private val _serviceId: String - get() = GServiceIds.getId(serviceIdInt) - - private fun getCleanServiceId(agencyTools: GAgencyTools): String { - return agencyTools.cleanServiceId(_serviceId) - } + get() = GIDs.getString(serviceIdInt) val uID by lazy { getNewUID(serviceIdInt, directionId, startTime, endTime) } - fun toFile(agencyTools: GAgencyTools) = listOf( - getCleanServiceId(agencyTools).quotesEscape(), // service ID - directionId.toString(), // direction ID - startTime.toString(), // start time - endTime.toString(), // end time - headwayInSec.toString(), // headway in seconds - ).joinToString(Constants.COLUMN_SEPARATOR_) + fun toFile(agencyTools: GAgencyTools) = buildList { + if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { + add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) // service ID + } else { + add(MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId))) // service ID int + } + add(directionId.toString()) // direction ID + add(startTime.toString()) // start time + add(endTime.toString()) // end time + add(headwayInSec.toString()) // headway in seconds + }.joinToString(Constants.COLUMN_SEPARATOR_) override fun compareTo(other: MFrequency?): Int { return when { diff --git a/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt b/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt index fff99659..48fcc723 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt @@ -1,6 +1,7 @@ package org.mtransit.parser.mt.data import androidx.annotation.Discouraged +import org.mtransit.commons.FeatureFlags import org.mtransit.parser.Constants import org.mtransit.parser.DefaultAgencyTools import org.mtransit.parser.MTLog @@ -9,7 +10,6 @@ import org.mtransit.parser.db.SQLUtils.quotes import org.mtransit.parser.db.SQLUtils.quotesEscape import org.mtransit.parser.gtfs.GAgencyTools import org.mtransit.parser.gtfs.data.GIDs -import org.mtransit.parser.gtfs.data.GServiceIds data class MSchedule( val routeId: Long, @@ -50,16 +50,14 @@ data class MSchedule( val serviceId = _serviceId private val _serviceId: String - get() = GServiceIds.getId(serviceIdInt) + get() = GIDs.getString(serviceIdInt) @Discouraged(message = "Not memory efficient") @Suppress("unused") val tripId = _tripId private val _tripId: String - get() { - return GIDs.getString(tripIdInt) - } + get() = GIDs.getString(tripIdInt) fun setHeadsign(newHeadsignType: Int, newHeadsignValue: String?) { if (newHeadsignValue.isNullOrBlank() @@ -100,10 +98,14 @@ data class MSchedule( } fun toFileNewServiceIdAndDirectionId(agencyTools: GAgencyTools) = buildList { - add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) // service ID + if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { + add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) + } else { + add(MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId))) + } // no route ID, just for file split - add(directionId.toString()) // direction ID - add(departure.toString()) // departure + add(directionId.toString()) + add(departure.toString()) if (DefaultAgencyTools.EXPORT_TRIP_ID) { @Suppress("ControlFlowWithEmptyBody") if (arrivalBeforeDeparture > 0) { @@ -112,13 +114,13 @@ data class MSchedule( add(arrivalBeforeDeparture.takeIf { it > 0 }?.toString() ?: Constants.EMPTY) // arrival before departure add(_tripId.quotesEscape()) } - add(headsignType.takeIf { it >= 0 }?.toString() ?: Constants.EMPTY) // HEADSIGN TYPE - add((headsignValue ?: Constants.EMPTY).quotesEscape()) // HEADSIGN STRING + add(headsignType.takeIf { it >= 0 }?.toString() ?: Constants.EMPTY) + add((headsignValue ?: Constants.EMPTY).quotesEscape()) add(accessible.toString()) }.joinToString(Constants.COLUMN_SEPARATOR_) fun toFileSameServiceIdAndDirectionId(lastSchedule: MSchedule?) = buildList { - add((departure - (lastSchedule?.departure ?: 0)).toString()) // departure + add((departure - (lastSchedule?.departure ?: 0)).toString()) if (DefaultAgencyTools.EXPORT_TRIP_ID) { @Suppress("ControlFlowWithEmptyBody") if (arrivalBeforeDeparture > 0) { @@ -128,11 +130,11 @@ data class MSchedule( add(_tripId.quotesEscape()) } if (headsignType == MDirection.HEADSIGN_TYPE_NO_PICKUP) { - add(MDirection.HEADSIGN_TYPE_NO_PICKUP.toString()) // HEADSIGN TYPE - add(Constants.EMPTY.quotes()) // HEADSIGN STRING + add(MDirection.HEADSIGN_TYPE_NO_PICKUP.toString()) + add(Constants.EMPTY.quotes()) } else { - add(headsignType.takeIf { it >= 0 }?.toString() ?: Constants.EMPTY) // HEADSIGN TYPE - add((headsignValue ?: Constants.EMPTY).quotesEscape()) // HEADSIGN STRING + add(headsignType.takeIf { it >= 0 }?.toString() ?: Constants.EMPTY) + add((headsignValue ?: Constants.EMPTY).quotesEscape()) } add(accessible.toString()) }.joinToString(Constants.COLUMN_SEPARATOR_) diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt index 8dada979..796601c2 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt @@ -1,13 +1,14 @@ package org.mtransit.parser.mt.data import androidx.annotation.Discouraged +import org.mtransit.commons.FeatureFlags import org.mtransit.parser.Constants import org.mtransit.parser.db.SQLUtils.quotesEscape import org.mtransit.parser.db.SQLUtils.unquotes import org.mtransit.parser.gtfs.GAgencyTools import org.mtransit.parser.gtfs.data.GCalendarDate import org.mtransit.parser.gtfs.data.GCalendarDatesExceptionType -import org.mtransit.parser.gtfs.data.GServiceIds +import org.mtransit.parser.gtfs.data.GIDs data class MServiceDate( val serviceIdInt: Int, @@ -30,7 +31,7 @@ data class MServiceDate( val serviceId = _serviceId private val _serviceId: String - get() = GServiceIds.getId(serviceIdInt) + get() = GIDs.getString(serviceIdInt) override fun compareTo(other: MServiceDate): Int = compareBy( MServiceDate::calendarDate, @@ -39,8 +40,12 @@ data class MServiceDate( ).compare(this, other) fun toFile(agencyTools: GAgencyTools) = buildList { - add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) // service ID - add(calendarDate.toString()) // calendar date + if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { + add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) + } else { + add(MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId))) + } + add(calendarDate.toString()) add(exceptionType.toString()) }.joinToString(Constants.COLUMN_SEPARATOR_) @@ -74,9 +79,9 @@ data class MServiceDate( .takeIf { it.size == 3 } ?.let { columns -> MServiceDate( - serviceIdInt = GServiceIds.getInt(columns[0].unquotes()), // service ID - calendarDate = columns[1].toInt(), // calendar date - exceptionType = columns[2].toInt() + serviceIdInt = GIDs.getInt(columns[0].unquotes()), + calendarDate = columns[1].toInt(), + exceptionType = columns[2].toInt(), ) } diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt index bee54bbf..1eea29f4 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt @@ -5,21 +5,19 @@ import org.mtransit.parser.db.SQLUtils.quotesEscape import org.mtransit.parser.db.SQLUtils.unquotes data class MServiceId( - val serviceId: String, + val serviceId: String, // already agencyTools.cleanServiceId(serviceId) before val serviceIdInt: Int, ) : Comparable { fun toFile() = buildList { - add(serviceId.quotesEscape()) // service ID + add(serviceId.quotesEscape()) // service ID // already agencyTools.cleanServiceId(serviceId) before add(serviceIdInt.toString()) // service ID Int }.joinToString(Constants.COLUMN_SEPARATOR_) - override fun compareTo(other: MServiceId): Int = - // sort by service_id => service_id_int - when { - serviceId != other.serviceId -> serviceId.compareTo(other.serviceId) - else -> serviceIdInt.compareTo(other.serviceIdInt) - } + override fun compareTo(other: MServiceId) = compareBy( + MServiceId::serviceId, + MServiceId::serviceIdInt, + ).compare(this, other) companion object { fun fromFileLine(line: String) = line.split(Constants.COLUMN_SEPARATOR) diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GServiceIds.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt similarity index 92% rename from src/main/java/org/mtransit/parser/gtfs/data/GServiceIds.kt rename to src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt index f994fad7..720a1c7e 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GServiceIds.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt @@ -1,11 +1,10 @@ -package org.mtransit.parser.gtfs.data +package org.mtransit.parser.mt.data import androidx.collection.SparseArrayCompat import androidx.collection.mutableScatterMapOf import org.mtransit.parser.MTLog -import org.mtransit.parser.mt.data.MServiceId -object GServiceIds { +object MServiceIds { private var increment = 0 private val intToString = SparseArrayCompat() @@ -29,6 +28,7 @@ object GServiceIds { return newServiceId.serviceIdInt } + @Suppress("unused") @JvmStatic fun getId(serviceIdInt: Int): String { return intToString[serviceIdInt] ?: throw MTLog.Fatal("Unexpected Service ID integer $serviceIdInt!") diff --git a/src/test/java/org/mtransit/parser/DefaultAgencyToolsTest.java b/src/test/java/org/mtransit/parser/DefaultAgencyToolsTest.java index afc332c6..09562784 100644 --- a/src/test/java/org/mtransit/parser/DefaultAgencyToolsTest.java +++ b/src/test/java/org/mtransit/parser/DefaultAgencyToolsTest.java @@ -253,7 +253,7 @@ public void test_parseCalendarDates_Split() { // #KingstonTransit #Should be 2 g p.setTodayStringInt(2024_04_09); boolean lookBackward = true; // Act - DefaultAgencyTools.parseCalendarDates(gCalendarDates, null, c, p, lookBackward); + DefaultAgencyTools.parseCalendarDates(gCalendarDates, c, p, lookBackward); System.out.println("CURRENT: " + p); // Assert assertNotNull(p.getStartDate()); @@ -266,7 +266,7 @@ public void test_parseCalendarDates_Split() { // #KingstonTransit #Should be 2 g p.setEndDate(null); // reset lookBackward = false; // Act - DefaultAgencyTools.parseCalendarDates(gCalendarDates, null, c, p, lookBackward); + DefaultAgencyTools.parseCalendarDates(gCalendarDates, c, p, lookBackward); System.out.println("NEXT: " + p); // Assert assertNotNull(p.getStartDate()); @@ -364,7 +364,7 @@ public void test_parseCalendarDates_Split_ShouldBeOne() { // #GRT p.setTodayStringInt(2024_04_09); boolean lookBackward = true; // Act - DefaultAgencyTools.parseCalendarDates(gCalendarDates, null, c, p, lookBackward); + DefaultAgencyTools.parseCalendarDates(gCalendarDates, c, p, lookBackward); System.out.println("CURRENT: " + p); // Assert assertNotNull(p.getStartDate()); @@ -377,7 +377,7 @@ public void test_parseCalendarDates_Split_ShouldBeOne() { // #GRT p.setEndDate(null); // reset lookBackward = false; // Act - DefaultAgencyTools.parseCalendarDates(gCalendarDates, null, c, p, lookBackward); + DefaultAgencyTools.parseCalendarDates(gCalendarDates, c, p, lookBackward); System.out.println("NEXT: " + p); // Assert assertNotNull(p.getStartDate()); @@ -455,7 +455,7 @@ public void test_findCalendarsTodayPeriod_Current_NoCurrentScheduleNextOnly() { boolean lookBackward = true; List gCalendarDates = Collections.emptyList(); - DefaultAgencyTools.findCalendarsTodayPeriod(gCalendars, gCalendarDates, DATE_FORMAT, c, p, lookBackward); + DefaultAgencyTools.findCalendarsTodayPeriod(gCalendars, gCalendarDates, p, lookBackward); assertNull(p.getStartDate()); assertNull(p.getEndDate()); @@ -473,7 +473,7 @@ public void test_findCalendarsTodayPeriod_Current_InProgress() { boolean lookBackward = true; List gCalendarDates = Collections.emptyList(); - DefaultAgencyTools.findCalendarsTodayPeriod(gCalendars, gCalendarDates, DATE_FORMAT, c, p, lookBackward); + DefaultAgencyTools.findCalendarsTodayPeriod(gCalendars, gCalendarDates, p, lookBackward); assertEquals(2023_01_01, p.getStartDate().intValue()); assertEquals(2023_12_31, p.getEndDate().intValue()); @@ -491,7 +491,7 @@ public void test_findCalendarsTodayPeriod_Current_InTheDistantPast() { boolean lookBackward = true; List gCalendarDates = Collections.emptyList(); - DefaultAgencyTools.findCalendarsTodayPeriod(gCalendars, gCalendarDates, DATE_FORMAT, c, p, lookBackward); + DefaultAgencyTools.findCalendarsTodayPeriod(gCalendars, gCalendarDates, p, lookBackward); assertEquals(2010_01_01, p.getStartDate().intValue()); assertEquals(2010_12_31, p.getEndDate().intValue()); @@ -511,7 +511,7 @@ public void test_findCalendarsTodayPeriod_Next_NoNextScheduleCurrentOnly() { boolean lookBackward = false; List gCalendarDates = Collections.emptyList(); - DefaultAgencyTools.findCalendarsTodayPeriod(gCalendars, gCalendarDates, DATE_FORMAT, c, p, lookBackward); + DefaultAgencyTools.findCalendarsTodayPeriod(gCalendars, gCalendarDates, p, lookBackward); assertNull(p.getStartDate()); assertNull(p.getEndDate()); @@ -529,7 +529,7 @@ public void test_findCalendarsTodayPeriod_Next_NextOnly() { boolean lookBackward = false; List gCalendarDates = Collections.emptyList(); - DefaultAgencyTools.findCalendarsTodayPeriod(gCalendars, gCalendarDates, DATE_FORMAT, c, p, lookBackward); + DefaultAgencyTools.findCalendarsTodayPeriod(gCalendars, gCalendarDates, p, lookBackward); assertEquals(2024_01_01, p.getStartDate().intValue()); assertEquals(2024_12_31, p.getEndDate().intValue()); @@ -549,7 +549,7 @@ public void test_findCalendarsTodayPeriod_Next_NextOnly_StartRemoved() { new GCalendarDate("id1", 2024_01_01, GCalendarDatesExceptionType.SERVICE_REMOVED) ); - DefaultAgencyTools.findCalendarsTodayPeriod(gCalendars, gCalendarDates, DATE_FORMAT, c, p, lookBackward); + DefaultAgencyTools.findCalendarsTodayPeriod(gCalendars, gCalendarDates, p, lookBackward); assertEquals(2024_01_01, p.getStartDate().intValue()); assertEquals(2024_12_31, p.getEndDate().intValue()); From 7b531f86a3ef02cb4ff2fd82407b77f2bef75886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Fri, 14 Nov 2025 21:22:21 -0500 Subject: [PATCH 04/13] WIP --- src/main/java/org/mtransit/parser/mt/MGenerator.java | 7 ++++--- src/main/java/org/mtransit/parser/mt/data/MFrequency.kt | 4 ++-- src/main/java/org/mtransit/parser/mt/data/MSchedule.kt | 6 +++--- src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt | 4 ++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index 274ea054..0d6c7bf9 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -8,6 +8,7 @@ import org.jetbrains.annotations.Nullable; import org.mtransit.commons.Cleaner; import org.mtransit.commons.CloseableUtils; +import org.mtransit.commons.FeatureFlags; import org.mtransit.commons.GTFSCommons; import org.mtransit.commons.SourceUtils; import org.mtransit.commons.StringUtils; @@ -270,7 +271,7 @@ public static void dumpFiles(@NotNull GAgencyTools gAgencyTools, Pair, Pair> minMaxLatLng = dumpRDSStops(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); // SERVICE IDS - dumpScheduleServiceIds(gAgencyTools, mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); + dumpServiceIds(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); // SCHEDULE SERVICE DATES Pair minMaxDates = dumpScheduleServiceDates(gAgencyTools, mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); @@ -517,14 +518,14 @@ private static Pair, Pair> dumpRDSStops(@Nu return minMaxLatLng; } - private static void dumpScheduleServiceIds( - @NotNull GAgencyTools gAgencyTools, + private static void dumpServiceIds( @Nullable MSpec mSpec, @NotNull String fileBase, boolean deleteAll, @NotNull File dataDirF, @NotNull File rawDirF, @Nullable Connection dbConnection) { + if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) return; if (!deleteAll && (mSpec == null || !mSpec.isValid() || (F_PRE_FILLED_DB && dbConnection == null))) { throw new MTLog.Fatal("Generated data invalid (agencies: %s)!", mSpec); diff --git a/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt b/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt index 1c54cec9..22f4984e 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt @@ -26,9 +26,9 @@ data class MFrequency( fun toFile(agencyTools: GAgencyTools) = buildList { if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { - add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) // service ID - } else { add(MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId))) // service ID int + } else { + add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) // service ID } add(directionId.toString()) // direction ID add(startTime.toString()) // start time diff --git a/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt b/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt index 48fcc723..13964094 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt @@ -98,10 +98,10 @@ data class MSchedule( } fun toFileNewServiceIdAndDirectionId(agencyTools: GAgencyTools) = buildList { - if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { - add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) + if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { + add(MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId))) // service ID int } else { - add(MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId))) + add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) // service ID } // no route ID, just for file split add(directionId.toString()) diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt index 796601c2..7dcf2954 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt @@ -41,9 +41,9 @@ data class MServiceDate( fun toFile(agencyTools: GAgencyTools) = buildList { if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { - add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) + add(MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId))) // service ID int } else { - add(MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId))) + add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) // service ID } add(calendarDate.toString()) add(exceptionType.toString()) From 860478e55c50c782bd77eb8e82ed9fa471c1bfb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Fri, 14 Nov 2025 22:12:03 -0500 Subject: [PATCH 05/13] wip --- .../mtransit/parser/DefaultAgencyTools.java | 5 ++- .../parser/config/gtfs/data/AgencyConfig.kt | 2 + .../org/mtransit/parser/gtfs/data/GAgency.kt | 8 ++-- .../mtransit/parser/gtfs/data/GCalendar.kt | 6 +-- .../parser/gtfs/data/GCalendarDate.kt | 4 +- .../mtransit/parser/gtfs/data/GDirection.kt | 8 ++-- .../mtransit/parser/gtfs/data/GFrequency.kt | 28 ++++---------- .../org/mtransit/parser/gtfs/data/GRoute.kt | 24 +++++------- .../org/mtransit/parser/gtfs/data/GStop.kt | 16 +++----- .../mtransit/parser/gtfs/data/GStopTime.kt | 32 +++++----------- .../org/mtransit/parser/gtfs/data/GTrip.kt | 16 ++++---- .../mtransit/parser/gtfs/data/GTripStop.kt | 8 ++-- .../parser/mt/GenerateMObjectsTask.java | 37 ++----------------- .../org/mtransit/parser/mt/MGenerator.java | 7 +++- .../java/org/mtransit/parser/mt/MReader.kt | 23 +++++------- .../org/mtransit/parser/mt/data/MAgency.kt | 4 +- .../org/mtransit/parser/mt/data/MFrequency.kt | 4 +- .../org/mtransit/parser/mt/data/MSchedule.kt | 8 ++-- .../mtransit/parser/mt/data/MServiceDate.kt | 37 +++++++++++++------ .../org/mtransit/parser/mt/data/MServiceId.kt | 6 +-- .../mtransit/parser/mt/data/MServiceIds.kt | 4 +- 21 files changed, 120 insertions(+), 167 deletions(-) diff --git a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java index f7e01bff..3fdf1384 100644 --- a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java +++ b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java @@ -353,7 +353,10 @@ public String cleanServiceId(@NotNull String gServiceId) { @Nullable @Override public String getServiceIdCleanupRegex() { - return null; // opt-in + if (Configs.getAgencyConfig() != null) { + return Configs.getAgencyConfig().getServiceIdCleanupRegex(); + } + return null; // OPT-IN feature } @Nullable diff --git a/src/main/java/org/mtransit/parser/config/gtfs/data/AgencyConfig.kt b/src/main/java/org/mtransit/parser/config/gtfs/data/AgencyConfig.kt index a2940331..8314490c 100644 --- a/src/main/java/org/mtransit/parser/config/gtfs/data/AgencyConfig.kt +++ b/src/main/java/org/mtransit/parser/config/gtfs/data/AgencyConfig.kt @@ -13,4 +13,6 @@ data class AgencyConfig( val defaultColorEnabled: Boolean = false, // OPT-IN feature @SerialName("default_color") val defaultColor: String, // REQUIRED + @SerialName("service_id_cleanup_regex") + val serviceIdCleanupRegex: String? = null, // optional ) diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GAgency.kt b/src/main/java/org/mtransit/parser/gtfs/data/GAgency.kt index e2b9a1b7..d3126c8d 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GAgency.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GAgency.kt @@ -38,14 +38,12 @@ data class GAgency( agencyEmail, ) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val agencyId = _agencyId + @get:Discouraged(message = "Not memory efficient") + val agencyId: AgencyId get() = _agencyId private val _agencyId: AgencyId - get() { - return GIDs.getString(agencyIdInt) - } + get() = GIDs.getString(agencyIdInt) @Suppress("unused") fun isDifferentAgency(otherAgencyIdInt: Int): Boolean = agencyIdInt != otherAgencyIdInt diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GCalendar.kt b/src/main/java/org/mtransit/parser/gtfs/data/GCalendar.kt index f07facfd..a0649292 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GCalendar.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GCalendar.kt @@ -67,9 +67,9 @@ data class GCalendar( endDate ) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val serviceId = _serviceId + @get:Discouraged(message = "Not memory efficient") + val serviceId: String get() = _serviceId private val _serviceId: String get() = GIDs.getString(serviceIdInt) @@ -256,7 +256,7 @@ data class GCalendar( sunday: Boolean, startDate: Int, endDate: Int, - exceptionType: GCalendarDatesExceptionType = GCalendarDatesExceptionType.SERVICE_ADDED, + exceptionType: GCalendarDatesExceptionType = GCalendarDatesExceptionType.SERVICE_DEFAULT, ) = buildList { try { val dateFormat = GFieldTypes.makeDateFormat() diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GCalendarDate.kt b/src/main/java/org/mtransit/parser/gtfs/data/GCalendarDate.kt index 0d85c560..ec6ee9bc 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GCalendarDate.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GCalendarDate.kt @@ -40,9 +40,9 @@ data class GCalendarDate( GCalendarDatesExceptionType.parse(exceptionTypeInt) ) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val serviceId = _serviceId + @get:Discouraged(message = "Not memory efficient") + val serviceId: String get() = _serviceId private val _serviceId: String get() = GIDs.getString(serviceIdInt) diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GDirection.kt b/src/main/java/org/mtransit/parser/gtfs/data/GDirection.kt index 167cdda5..45f7157d 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GDirection.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GDirection.kt @@ -25,14 +25,12 @@ data class GDirection( destination = destination, ) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val routeId = _routeId + @get:Discouraged(message = "Not memory efficient") + val routeId: String get() = _routeId private val _routeId: String - get() { - return GIDs.getString(routeIdInt) - } + get() = GIDs.getString(routeIdInt) @Suppress("unused") fun toStringPlus(): String { diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GFrequency.kt b/src/main/java/org/mtransit/parser/gtfs/data/GFrequency.kt index 2edd0af3..be85746f 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GFrequency.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GFrequency.kt @@ -31,46 +31,34 @@ data class GFrequency( exactTimes, ) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val tripId = _tripId + @get:Discouraged(message = "Not memory efficient") + val tripId: String get() = _tripId @Suppress("unused") private val _tripId: String - get() { - return GIDs.getString(tripIdInt) - } + get() = GIDs.getString(tripIdInt) val startTime: Int = _startTime @Suppress("unused") val startTimeDate: Date - get() { - return GTime.toDate(_startTime) - } + get() = GTime.toDate(_startTime) val startTimeMs: Long - get() { - return GTime.toMs(_startTime) - } + get() = GTime.toMs(_startTime) val endTime: Int = _endTime @Suppress("unused") val endTimeDate: Date - get() { - return GTime.toDate(_endTime) - } + get() = GTime.toDate(_endTime) val endTimeMs: Long - get() { - return GTime.toMs(_endTime) - } + get() = GTime.toMs(_endTime) val headwayMs: Long - get() { - return TimeUnit.SECONDS.toMillis(headwaySecs.toLong()) - } + get() = TimeUnit.SECONDS.toMillis(headwaySecs.toLong()) @Suppress("unused") fun toStringPlus(): String { diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GRoute.kt b/src/main/java/org/mtransit/parser/gtfs/data/GRoute.kt index efaf795b..680cec7f 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GRoute.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GRoute.kt @@ -67,34 +67,30 @@ data class GRoute( @Suppress("unused") fun isDifferentAgency(otherAgencyId: String): Boolean = isDifferentAgency(GIDs.getInt(otherAgencyId)) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val agencyIdOrDefault: AgencyId = _agencyId + @get:Discouraged(message = "Not memory efficient") + val agencyIdOrDefault: AgencyId get() = _agencyId - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val agencyId: AgencyId = _agencyId + @get:Discouraged(message = "Not memory efficient") + val agencyId: AgencyId get() = _agencyId private val _agencyId: AgencyId get() = GIDs.getString(agencyIdInt) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val routeId = _routeId + @get:Discouraged(message = "Not memory efficient") + val routeId: RouteId get() = _routeId private val _routeId: RouteId - get() { - return GIDs.getString(routeIdInt) - } + get() = GIDs.getString(routeIdInt) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val originalRouteId = _originalRouteId + @get:Discouraged(message = "Not memory efficient") + val originalRouteId: String get() = _originalRouteId private val _originalRouteId: String - get() { - return GIDs.getString(originalRouteIdInt) - } + get() = GIDs.getString(originalRouteIdInt) @Suppress("unused") val shortestRouteName = routeShortName.ifEmpty { routeLongName } diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GStop.kt b/src/main/java/org/mtransit/parser/gtfs/data/GStop.kt index 07fb54cd..a26769ba 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GStop.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GStop.kt @@ -42,23 +42,19 @@ data class GStop( GWheelchairBoardingType.parse(wheelchairBoarding), ) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val stopId = _stopId + @get:Discouraged(message = "Not memory efficient") + val stopId: StopId get() = _stopId private val _stopId: StopId - get() { - return GIDs.getString(stopIdInt) - } + get() = GIDs.getString(stopIdInt) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val parentStationId = _parentStationId + @get:Discouraged(message = "Not memory efficient") + val parentStationId: StopId? get() = _parentStationId private val _parentStationId: StopId? - get() { - return parentStationIdInt?.let { GIDs.getString(it) } - } + get() = parentStationIdInt?.let { GIDs.getString(it) } @JvmOverloads @Suppress("unused") diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GStopTime.kt b/src/main/java/org/mtransit/parser/gtfs/data/GStopTime.kt index cd5d2b3e..87f57cb0 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GStopTime.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GStopTime.kt @@ -88,23 +88,19 @@ data class GStopTime( timePoint = timePoint, ) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val tripId = _tripId + @get:Discouraged(message = "Not memory efficient") + val tripId: String get() = _tripId private val _tripId: String - get() { - return GIDs.getString(tripIdInt) - } + get() = GIDs.getString(tripIdInt) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val stopId = _stopId + @get:Discouraged(message = "Not memory efficient") + val stopId: String get() = _stopId private val _stopId: String - get() { - return GIDs.getString(stopIdInt) - } + get() = GIDs.getString(stopIdInt) val arrivalTime: Int = _arrivalTime @@ -112,30 +108,22 @@ data class GStopTime( @Suppress("unused") val arrivalTimeMs: Long - get() { - return GTime.toMs(_arrivalTime) - } + get() = GTime.toMs(_arrivalTime) @Suppress("unused") val arrivalTimeDate: Date - get() { - return GTime.toDate(_arrivalTime) - } + get() = GTime.toDate(_arrivalTime) val departureTime: Int = _departureTime fun hasDepartureTime() = _departureTime >= 0 val departureTimeMs: Long - get() { - return GTime.toMs(_departureTime) - } + get() = GTime.toMs(_departureTime) @Suppress("unused") val departureTimeDate: Date - get() { - return GTime.toDate(_departureTime) - } + get() = GTime.toDate(_departureTime) val uID by lazy { getNewUID(tripIdInt, stopIdInt, stopSequence) } diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt b/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt index fbb31814..a6b25f09 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt @@ -71,30 +71,30 @@ data class GTrip( val uID by lazy { getNewUID(routeIdInt, tripIdInt) } - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val routeId = _routeId + @get:Discouraged(message = "Not memory efficient") + val routeId: String get() = _routeId private val _routeId: String get() = GIDs.getString(routeIdInt) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val originalRouteId = _originalRouteId + @get:Discouraged(message = "Not memory efficient") + val originalRouteId: String get() = _originalRouteId private val _originalRouteId: String get() = GIDs.getString(originalRouteIdInt) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val serviceId = _serviceId + @get:Discouraged(message = "Not memory efficient") + val serviceId: String get() = _serviceId private val _serviceId: String get() = GIDs.getString(serviceIdInt) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val tripId = _tripId + @get:Discouraged(message = "Not memory efficient") + val tripId: String get() = _tripId @Suppress("unused") private val _tripId: String diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GTripStop.kt b/src/main/java/org/mtransit/parser/gtfs/data/GTripStop.kt index 016700a0..4542bde0 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GTripStop.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GTripStop.kt @@ -49,17 +49,17 @@ data class GTripStop( val uID by lazy { getNewUID(routeIdInt, tripIdInt, stopIdInt, stopSequence) } - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val tripId = _tripId + @get:Discouraged(message = "Not memory efficient") + val tripId: String get() = _tripId @Suppress("unused") private val _tripId: String get() = GIDs.getString(tripIdInt) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val stopId = _stopId + @get:Discouraged(message = "Not memory efficient") + val stopId: String get() = _stopId private val _stopId: String get() = GIDs.getString(stopIdInt) diff --git a/src/main/java/org/mtransit/parser/mt/GenerateMObjectsTask.java b/src/main/java/org/mtransit/parser/mt/GenerateMObjectsTask.java index 2974d313..b5123161 100644 --- a/src/main/java/org/mtransit/parser/mt/GenerateMObjectsTask.java +++ b/src/main/java/org/mtransit/parser/mt/GenerateMObjectsTask.java @@ -24,16 +24,15 @@ import org.mtransit.parser.gtfs.data.GTrip; import org.mtransit.parser.gtfs.data.GTripStop; import org.mtransit.parser.mt.data.MAgency; -import org.mtransit.parser.mt.data.MCalendarExceptionType; +import org.mtransit.parser.mt.data.MDirection; import org.mtransit.parser.mt.data.MDirectionCardinalType; +import org.mtransit.parser.mt.data.MDirectionStop; import org.mtransit.parser.mt.data.MFrequency; import org.mtransit.parser.mt.data.MRoute; import org.mtransit.parser.mt.data.MSchedule; import org.mtransit.parser.mt.data.MServiceDate; import org.mtransit.parser.mt.data.MSpec; import org.mtransit.parser.mt.data.MStop; -import org.mtransit.parser.mt.data.MDirection; -import org.mtransit.parser.mt.data.MDirectionStop; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -147,42 +146,14 @@ private MSpec doCall() { if (!serviceIdInts.contains(gCalendarDate.getServiceIdInt())) { continue; } - switch (gCalendarDate.getExceptionType()) { - case SERVICE_REMOVED: // keep list of removed service for calendars processing - mServiceDates.add(new MServiceDate( - gCalendarDate.getServiceIdInt(), - gCalendarDate.getDate(), - MCalendarExceptionType.REMOVED - )); - break; - case SERVICE_ADDED: - mServiceDates.add(new MServiceDate( - gCalendarDate.getServiceIdInt(), - gCalendarDate.getDate(), - MCalendarExceptionType.ADDED - )); - break; - case SERVICE_DEFAULT: - mServiceDates.add(new MServiceDate( - gCalendarDate.getServiceIdInt(), - gCalendarDate.getDate(), - MCalendarExceptionType.DEFAULT - )); - break; - default: - throw new MTLog.Fatal("%s: Unexpected calendar date exception type '%s'!", this.routeId, gCalendarDate.getExceptionType()); - } + mServiceDates.add(MServiceDate.fromCalendarDate(gCalendarDate)); } for (GCalendar gCalendar : routeGTFS.getAllCalendars()) { if (!serviceIdInts.contains(gCalendar.getServiceIdInt())) { continue; } for (GCalendarDate gCalendarDate : gCalendar.getDates()) { - mServiceDates.add(new MServiceDate( - gCalendarDate.getServiceIdInt(), - gCalendarDate.getDate(), - MCalendarExceptionType.DEFAULT - )); + mServiceDates.add(MServiceDate.fromCalendarDate(gCalendarDate)); } } MDirection mDirection; diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index 0d6c7bf9..0daac294 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -525,7 +525,7 @@ private static void dumpServiceIds( @NotNull File dataDirF, @NotNull File rawDirF, @Nullable Connection dbConnection) { - if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) return; + if (!FeatureFlags.F_EXPORT_SERVICE_ID_INTS) return; if (!deleteAll && (mSpec == null || !mSpec.isValid() || (F_PRE_FILLED_DB && dbConnection == null))) { throw new MTLog.Fatal("Generated data invalid (agencies: %s)!", mSpec); @@ -860,6 +860,7 @@ private static String getLastModified(String gtfsDir) { private static final String GTFS_RDS_AGENCY_EXTENDED_TYPE = "gtfs_rts_agency_extended_type"; // do not change to avoid breaking compat w/ old modules private static final String GTFS_RDS_TIMEZONE = "gtfs_rts_timezone"; // do not change to avoid breaking compat w/ old modules private static final String GTFS_RDS_COLOR = "gtfs_rts_color"; // do not change to avoid breaking compat w/ old modules + private static final String GTFS_RDS_SERVICE_ID_CLEANUP_REGEX = "gtfs_rts_service_id_cleanup_regex"; // do not change to avoid breaking compat w/ old modules private static final String GTFS_RDS_ROUTE_ID_CLEANUP_REGEX = "gtfs_rts_route_id_cleanup_regex"; // do not change to avoid breaking compat w/ old modules private static final String GTFS_RDS_TRIP_ID_CLEANUP_REGEX = "gtfs_rts_trip_id_cleanup_regex"; // do not change to avoid breaking compat w/ old modules private static final String GTFS_RDS_STOP_ID_CLEANUP_REGEX = "gtfs_rts_stop_id_cleanup_regex"; // do not change to avoid breaking compat w/ old modules @@ -909,6 +910,10 @@ private static void dumpCommonValues(File dumpDirF, GAgencyTools gAgencyTools, M ow.write(Constants.NEW_LINE); ow.write(getRESOURCES_STRING(GTFS_RDS_COLOR, mSpec.getFirstAgency().getColor())); ow.write(Constants.NEW_LINE); + if (gAgencyTools.getServiceIdCleanupRegex() != null) { + ow.write(getRESOURCES_STRING(GTFS_RDS_SERVICE_ID_CLEANUP_REGEX, escapeResString(gAgencyTools.getServiceIdCleanupRegex()))); + ow.write(Constants.NEW_LINE); + } if (gAgencyTools.getRouteIdCleanupRegex() != null) { ow.write(getRESOURCES_STRING(GTFS_RDS_ROUTE_ID_CLEANUP_REGEX, escapeResString(gAgencyTools.getRouteIdCleanupRegex()))); ow.write(Constants.NEW_LINE); diff --git a/src/main/java/org/mtransit/parser/mt/MReader.kt b/src/main/java/org/mtransit/parser/mt/MReader.kt index f7b2d49e..fafc2e9f 100644 --- a/src/main/java/org/mtransit/parser/mt/MReader.kt +++ b/src/main/java/org/mtransit/parser/mt/MReader.kt @@ -147,23 +147,18 @@ object MReader { private const val GTFS_SCHEDULE_SERVICE_IDS = "gtfs_schedule_service_ids" @JvmStatic - fun loadServiceIds(fileBase: String): List? { - try { - val gtfsScheduleServiceIds = getResDirName(fileBase) + "/$RAW/${fileBase}$GTFS_SCHEDULE_SERVICE_IDS" - val gtfsScheduleServiceIdsFile = File(gtfsScheduleServiceIds) - if (!gtfsScheduleServiceIdsFile.exists()) { - MTLog.log("File not found '$gtfsScheduleServiceIds'!") - return null + fun loadServiceIds(fileBase: String) = try { + File(getResDirName(fileBase) + "/$RAW/${fileBase}$GTFS_SCHEDULE_SERVICE_IDS") + .takeIf { it.exists() } + .also { + if (it == null) MTLog.log("File not found '$it'!") } - val gtfsScheduleServiceIdsFileLines = gtfsScheduleServiceIdsFile.readLines() - val serviceIds = gtfsScheduleServiceIdsFileLines.mapNotNull { line -> + ?.readLines()?.mapNotNull { line -> MServiceId.fromFileLine(line) } - return serviceIds - } catch (e: Exception) { - MTLog.logNonFatal(e, "Error while reading '$fileBase' service ids!") - return null - } + } catch (e: Exception) { + MTLog.logNonFatal(e, "Error while reading '$fileBase' service ids!") + null } // endregion diff --git a/src/main/java/org/mtransit/parser/mt/data/MAgency.kt b/src/main/java/org/mtransit/parser/mt/data/MAgency.kt index 3e6d118b..b541adf4 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MAgency.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MAgency.kt @@ -29,9 +29,9 @@ data class MAgency( agencyTools.agencyRouteType, ) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val id = _id + @get:Discouraged(message = "Not memory efficient") + val id: String get() = _id private val _id: String get() { diff --git a/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt b/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt index 22f4984e..2f046203 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt @@ -15,9 +15,9 @@ data class MFrequency( private val headwayInSec: Int ) : Comparable { - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val serviceId = _serviceId + @get:Discouraged(message = "Not memory efficient") + val serviceId: String get() = _serviceId private val _serviceId: String get() = GIDs.getString(serviceIdInt) diff --git a/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt b/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt index 13964094..c1b84e92 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt @@ -45,16 +45,16 @@ data class MSchedule( accessible = accessible, ) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val serviceId = _serviceId + @get:Discouraged(message = "Not memory efficient") + val serviceId: String get() = _serviceId private val _serviceId: String get() = GIDs.getString(serviceIdInt) - @Discouraged(message = "Not memory efficient") @Suppress("unused") - val tripId = _tripId + @get:Discouraged(message = "Not memory efficient") + val tripId: String get() = _tripId private val _tripId: String get() = GIDs.getString(tripIdInt) diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt index 7dcf2954..c4fbf64e 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt @@ -16,7 +16,7 @@ data class MServiceDate( val exceptionType: Int, ) : Comparable { - constructor( + private constructor( serviceIdInt: Int, calendarDate: Int, exceptionType: MCalendarExceptionType @@ -26,9 +26,9 @@ data class MServiceDate( exceptionType.id ) - @Discouraged(message = "Not memory efficient") + @get:Discouraged(message = "Not memory efficient") @Suppress("unused") - val serviceId = _serviceId + val serviceId: String get() = _serviceId private val _serviceId: String get() = GIDs.getString(serviceIdInt) @@ -75,15 +75,28 @@ data class MServiceDate( return serviceDates.joinToString { it.toStringPlus() } } - fun fromFileLine(line: String) = line.split(Constants.COLUMN_SEPARATOR) - .takeIf { it.size == 3 } - ?.let { columns -> - MServiceDate( - serviceIdInt = GIDs.getInt(columns[0].unquotes()), - calendarDate = columns[1].toInt(), - exceptionType = columns[2].toInt(), - ) - } + @JvmStatic + fun fromCalendarDate(calendarDate: GCalendarDate) = + MServiceDate( + serviceIdInt = calendarDate.serviceIdInt, + calendarDate = calendarDate.date, + exceptionType = when (calendarDate.exceptionType) { + GCalendarDatesExceptionType.SERVICE_ADDED -> MCalendarExceptionType.ADDED + GCalendarDatesExceptionType.SERVICE_REMOVED -> MCalendarExceptionType.REMOVED + GCalendarDatesExceptionType.SERVICE_DEFAULT -> MCalendarExceptionType.DEFAULT + } + ) + + fun fromFileLine(line: String) = + line.split(Constants.COLUMN_SEPARATOR) + .takeIf { it.size == 3 } + ?.let { columns -> + MServiceDate( + serviceIdInt = MServiceIds.getInt(columns[0].unquotes()), + calendarDate = columns[1].toInt(), + exceptionType = columns[2].toInt(), + ) + } @Suppress("unused") @JvmStatic diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt index 1eea29f4..d9ae0636 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt @@ -10,13 +10,13 @@ data class MServiceId( ) : Comparable { fun toFile() = buildList { - add(serviceId.quotesEscape()) // service ID // already agencyTools.cleanServiceId(serviceId) before - add(serviceIdInt.toString()) // service ID Int + add(serviceIdInt.toString()) + add(serviceId.quotesEscape()) // already agencyTools.cleanServiceId(serviceId) before }.joinToString(Constants.COLUMN_SEPARATOR_) override fun compareTo(other: MServiceId) = compareBy( - MServiceId::serviceId, MServiceId::serviceIdInt, + MServiceId::serviceId, ).compare(this, other) companion object { diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt index 720a1c7e..047c8d76 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt @@ -30,7 +30,7 @@ object MServiceIds { @Suppress("unused") @JvmStatic - fun getId(serviceIdInt: Int): String { + fun getString(serviceIdInt: Int): String { return intToString[serviceIdInt] ?: throw MTLog.Fatal("Unexpected Service ID integer $serviceIdInt!") } @@ -44,5 +44,5 @@ object MServiceIds { stringToInt.forEach { id, idInt -> add(MServiceId(id, idInt)) } - } + }.sorted() } From 7f564be7d1c383a81b3e16a563256f3dfe446a88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Mon, 17 Nov 2025 21:42:15 -0500 Subject: [PATCH 06/13] fix --- .../mtransit/parser/mt/data/MServiceDate.kt | 6 +++--- .../org/mtransit/parser/mt/data/MServiceId.kt | 6 +++--- .../org/mtransit/parser/mt/data/MServiceIds.kt | 18 +++++++++--------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt index c4fbf64e..cb4c11e4 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt @@ -92,9 +92,9 @@ data class MServiceDate( .takeIf { it.size == 3 } ?.let { columns -> MServiceDate( - serviceIdInt = MServiceIds.getInt(columns[0].unquotes()), - calendarDate = columns[1].toInt(), - exceptionType = columns[2].toInt(), + serviceIdInt = GIDs.getInt(columns[0].unquotes()), // service ID + calendarDate = columns[1].toInt(), // calendar date + exceptionType = columns[2].toInt() ) } diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt index d9ae0636..29e24885 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt @@ -5,8 +5,8 @@ import org.mtransit.parser.db.SQLUtils.quotesEscape import org.mtransit.parser.db.SQLUtils.unquotes data class MServiceId( - val serviceId: String, // already agencyTools.cleanServiceId(serviceId) before val serviceIdInt: Int, + val serviceId: String, // already agencyTools.cleanServiceId(serviceId) before ) : Comparable { fun toFile() = buildList { @@ -24,8 +24,8 @@ data class MServiceId( .takeIf { it.size == 2 } ?.let { columns -> MServiceId( - serviceId = columns[0].unquotes(), - serviceIdInt = columns[1].toInt(), + serviceIdInt = columns[0].toInt(), + serviceId = columns[1].unquotes(), ) } } diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt index 047c8d76..2e6ba391 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt @@ -7,8 +7,8 @@ import org.mtransit.parser.MTLog object MServiceIds { private var increment = 0 - private val intToString = SparseArrayCompat() - private val stringToInt = mutableScatterMapOf() + private val idIntToId = SparseArrayCompat() + private val idToIdInt = mutableScatterMapOf() @JvmStatic fun addAll(lastServiceIds: List?) { @@ -16,14 +16,14 @@ object MServiceIds { } fun add(serviceId: MServiceId) { - intToString.put(serviceId.serviceIdInt, serviceId.serviceId) - stringToInt[serviceId.serviceId] = serviceId.serviceIdInt + idIntToId.put(serviceId.serviceIdInt, serviceId.serviceId) + idToIdInt[serviceId.serviceId] = serviceId.serviceIdInt increment = maxOf(increment, serviceId.serviceIdInt) } fun add(serviceId: String): Int { increment++ // move to next - val newServiceId = MServiceId(serviceId, increment) + val newServiceId = MServiceId(increment, serviceId) add(newServiceId) return newServiceId.serviceIdInt } @@ -31,18 +31,18 @@ object MServiceIds { @Suppress("unused") @JvmStatic fun getString(serviceIdInt: Int): String { - return intToString[serviceIdInt] ?: throw MTLog.Fatal("Unexpected Service ID integer $serviceIdInt!") + return idIntToId[serviceIdInt] ?: throw MTLog.Fatal("Unexpected Service ID integer $serviceIdInt!") } @JvmStatic fun getInt(serviceId: String): Int { - return stringToInt[serviceId] ?: add(serviceId) + return idToIdInt[serviceId] ?: add(serviceId) } @JvmStatic fun getAll() = buildList { - stringToInt.forEach { id, idInt -> - add(MServiceId(id, idInt)) + idToIdInt.forEach { id, idInt -> + add(MServiceId(idInt, id)) } }.sorted() } From 0274977ce3f99b9d82e90a6b16146869c4be029e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Mon, 17 Nov 2025 21:52:50 -0500 Subject: [PATCH 07/13] fix --- src/main/java/org/mtransit/parser/mt/MGenerator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index 0daac294..52a954b4 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -270,8 +270,6 @@ public static void dumpFiles(@NotNull GAgencyTools gAgencyTools, // STOPS Pair, Pair> minMaxLatLng = dumpRDSStops(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); - // SERVICE IDS - dumpServiceIds(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); // SCHEDULE SERVICE DATES Pair minMaxDates = dumpScheduleServiceDates(gAgencyTools, mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); @@ -279,6 +277,8 @@ public static void dumpFiles(@NotNull GAgencyTools gAgencyTools, dumpScheduleStops(gAgencyTools, mSpec, fileBase, deleteAll, rawDirF); // FREQUENCY ROUTES dumpFrequencyRoutes(gAgencyTools, mSpec, fileBase, deleteAll, rawDirF); + // SERVICE IDS + dumpServiceIds(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); // AFTER SCHEDULE STOPS & FREQUENCY ROUTES if (deleteAll) { dumpValues(rawDirF, fileBase, null, null, null, null, null, -1, -1, null, true); } else { @@ -547,7 +547,7 @@ private static void dumpServiceIds( dbStatement = dbConnection.createStatement(); sqlInsert = GTFSCommons.getT_SERVICE_IDS_SQL_INSERT(); } - for (MServiceId mServiceId : MServiceIds.getAll()) { // TODO? mSpec.getServiceIds() + for (MServiceId mServiceId : MServiceIds.getAll()) { final String serviceIdsInsert = mServiceId.toFile(); // gAgencyTools if (F_PRE_FILLED_DB) { From 2894d160bc77b32d6bdc005bad97d116b127871d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Tue, 18 Nov 2025 19:46:46 -0500 Subject: [PATCH 08/13] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/main/java/org/mtransit/parser/mt/MGenerator.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index 52a954b4..5d0d0370 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -193,8 +193,10 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age ); } + private static final boolean DEBUG_LOG_MERGING = false; // Set to true to enable merging logs + private static void logMerging(@NotNull String msg, long routeId) { - if (true) return; // DEBUG + if (!DEBUG_LOG_MERGING) return; MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (merging %s)", routeId, msg); } @@ -564,7 +566,7 @@ private static void dumpServiceIds( } } } catch (Exception ioe) { - throw new MTLog.Fatal(ioe, "I/O Error while writing service dates file!"); + throw new MTLog.Fatal(ioe, "I/O Error while writing service IDs file!"); } finally { CloseableUtils.closeQuietly(ow); } From c7d115be50f1466723eaa6e77413b10000ab98ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Tue, 18 Nov 2025 19:54:21 -0500 Subject: [PATCH 09/13] PR comments --- .../org/mtransit/parser/mt/data/MServiceId.kt | 18 +++++++++-------- .../mtransit/parser/mt/data/MServiceIds.kt | 20 ++++++++++++------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt index 29e24885..b74e2853 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt @@ -1,6 +1,7 @@ package org.mtransit.parser.mt.data import org.mtransit.parser.Constants +import org.mtransit.parser.MTLog import org.mtransit.parser.db.SQLUtils.quotesEscape import org.mtransit.parser.db.SQLUtils.unquotes @@ -20,13 +21,14 @@ data class MServiceId( ).compare(this, other) companion object { - fun fromFileLine(line: String) = line.split(Constants.COLUMN_SEPARATOR) - .takeIf { it.size == 2 } - ?.let { columns -> - MServiceId( - serviceIdInt = columns[0].toInt(), - serviceId = columns[1].unquotes(), - ) - } + fun fromFileLine(line: String) = + line.split(Constants.COLUMN_SEPARATOR) + .takeIf { it.size == 2 } + ?.let { columns -> + MServiceId( + serviceIdInt = columns[0].toInt(), + serviceId = columns[1].unquotes(), + ) + } ?: run { MTLog.log("Invalid service ID line: '$line'!") } } } diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt index 2e6ba391..c3142b61 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt @@ -6,6 +6,8 @@ import org.mtransit.parser.MTLog object MServiceIds { + private val incrementLock = Any() + private var increment = 0 private val idIntToId = SparseArrayCompat() private val idToIdInt = mutableScatterMapOf() @@ -16,16 +18,20 @@ object MServiceIds { } fun add(serviceId: MServiceId) { - idIntToId.put(serviceId.serviceIdInt, serviceId.serviceId) - idToIdInt[serviceId.serviceId] = serviceId.serviceIdInt - increment = maxOf(increment, serviceId.serviceIdInt) + synchronized(incrementLock) { + idIntToId.put(serviceId.serviceIdInt, serviceId.serviceId) + idToIdInt[serviceId.serviceId] = serviceId.serviceIdInt + increment = maxOf(increment, serviceId.serviceIdInt) + } } fun add(serviceId: String): Int { - increment++ // move to next - val newServiceId = MServiceId(increment, serviceId) - add(newServiceId) - return newServiceId.serviceIdInt + synchronized(incrementLock) { + increment++ // move to next + val newServiceId = MServiceId(increment, serviceId) + add(newServiceId) + return newServiceId.serviceIdInt + } } @Suppress("unused") From cec4531aacafaad0f63d579ba6526b2fb843407d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Tue, 18 Nov 2025 20:13:40 -0500 Subject: [PATCH 10/13] PR comments --- src/main/java/org/mtransit/parser/mt/MGenerator.java | 6 +----- src/main/java/org/mtransit/parser/mt/MReader.kt | 10 ++++++---- .../java/org/mtransit/parser/mt/data/MServiceId.kt | 8 ++++++-- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index 5d0d0370..d24b48fc 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -537,10 +537,8 @@ private static void dumpServiceIds( } File file = new File(F_PRE_FILLED_DB ? dataDirF : rawDirF, fileBase + GTFS_SCHEDULE_SERVICE_IDS); FileUtils.deleteIfExist(file); // delete previous - BufferedWriter ow = null; - try { + try (BufferedWriter ow = new BufferedWriter(new FileWriter(file))) { if (!deleteAll) { - ow = new BufferedWriter(new FileWriter(file)); MTLog.logPOINT(); // LOG Statement dbStatement = null; String sqlInsert = null; @@ -567,8 +565,6 @@ private static void dumpServiceIds( } } catch (Exception ioe) { throw new MTLog.Fatal(ioe, "I/O Error while writing service IDs file!"); - } finally { - CloseableUtils.closeQuietly(ow); } } diff --git a/src/main/java/org/mtransit/parser/mt/MReader.kt b/src/main/java/org/mtransit/parser/mt/MReader.kt index fafc2e9f..daecf660 100644 --- a/src/main/java/org/mtransit/parser/mt/MReader.kt +++ b/src/main/java/org/mtransit/parser/mt/MReader.kt @@ -150,12 +150,14 @@ object MReader { fun loadServiceIds(fileBase: String) = try { File(getResDirName(fileBase) + "/$RAW/${fileBase}$GTFS_SCHEDULE_SERVICE_IDS") .takeIf { it.exists() } - .also { - if (it == null) MTLog.log("File not found '$it'!") - } - ?.readLines()?.mapNotNull { line -> + ?.readLines() + ?.mapNotNull { line -> MServiceId.fromFileLine(line) } + ?: run { + MTLog.log("File not found '${"/$RAW/${fileBase}$GTFS_SCHEDULE_SERVICE_IDS"}'!") + null + } } catch (e: Exception) { MTLog.logNonFatal(e, "Error while reading '$fileBase' service ids!") null diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt index b74e2853..9ed78955 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt @@ -21,7 +21,7 @@ data class MServiceId( ).compare(this, other) companion object { - fun fromFileLine(line: String) = + fun fromFileLine(line: String): MServiceId? = line.split(Constants.COLUMN_SEPARATOR) .takeIf { it.size == 2 } ?.let { columns -> @@ -29,6 +29,10 @@ data class MServiceId( serviceIdInt = columns[0].toInt(), serviceId = columns[1].unquotes(), ) - } ?: run { MTLog.log("Invalid service ID line: '$line'!") } + } + ?: run { + MTLog.log("Invalid service ID line: '$line'!") + null + } } } From 810f8aaa137bb92f4fb7f946ebbfdc3da878d9e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Tue, 18 Nov 2025 20:23:31 -0500 Subject: [PATCH 11/13] wip --- .../org/mtransit/parser/mt/data/MFrequency.kt | 41 ++++++------------- .../org/mtransit/parser/mt/data/MSchedule.kt | 28 ++++--------- .../mtransit/parser/mt/data/MServiceDate.kt | 8 ++-- .../mtransit/parser/mt/data/MServiceIds.kt | 13 +++--- 4 files changed, 30 insertions(+), 60 deletions(-) diff --git a/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt b/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt index 2f046203..c63de4d2 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt @@ -26,41 +26,24 @@ data class MFrequency( fun toFile(agencyTools: GAgencyTools) = buildList { if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { - add(MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId))) // service ID int + add(MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId))) } else { - add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) // service ID + add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) } - add(directionId.toString()) // direction ID - add(startTime.toString()) // start time - add(endTime.toString()) // end time - add(headwayInSec.toString()) // headway in seconds + add(directionId.toString()) + add(startTime.toString()) + add(endTime.toString()) + add(headwayInSec.toString()) }.joinToString(Constants.COLUMN_SEPARATOR_) override fun compareTo(other: MFrequency?): Int { return when { - other !is MFrequency -> { - +1 - } - - serviceIdInt != other.serviceIdInt -> { - _serviceId.compareTo(other._serviceId) - } - - directionId != other.directionId -> { - directionId.compareTo(other.directionId) - } - - startTime != other.startTime -> { - startTime - other.startTime - } - - endTime != other.endTime -> { - endTime - other.endTime - } - - else -> { - headwayInSec - other.headwayInSec - } + other !is MFrequency -> +1 + serviceIdInt != other.serviceIdInt -> _serviceId.compareTo(other._serviceId) + directionId != other.directionId -> directionId.compareTo(other.directionId) + startTime != other.startTime -> startTime - other.startTime + endTime != other.endTime -> endTime - other.endTime + else -> headwayInSec - other.headwayInSec } } diff --git a/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt b/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt index c1b84e92..a5956f37 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt @@ -99,9 +99,9 @@ data class MSchedule( fun toFileNewServiceIdAndDirectionId(agencyTools: GAgencyTools) = buildList { if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { - add(MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId))) // service ID int + add(MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId))) } else { - add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) // service ID + add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) } // no route ID, just for file split add(directionId.toString()) @@ -165,25 +165,11 @@ data class MSchedule( override fun compareTo(other: MSchedule): Int { // sort by route_id => service_id => direction_id => stop_id => departure return when { - routeId != other.routeId -> { - routeId.compareTo(other.routeId) - } - - serviceIdInt != other.serviceIdInt -> { - _serviceId.compareTo(other._serviceId) - } - - directionId != other.directionId -> { - directionId.compareTo(other.directionId) - } - - stopId != other.stopId -> { - stopId - other.stopId - } - - else -> { - departure - other.departure - } + routeId != other.routeId -> routeId.compareTo(other.routeId) + serviceIdInt != other.serviceIdInt -> _serviceId.compareTo(other._serviceId) + directionId != other.directionId -> directionId.compareTo(other.directionId) + stopId != other.stopId -> stopId - other.stopId + else -> departure - other.departure } } diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt index cb4c11e4..ae6615a6 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt @@ -41,9 +41,9 @@ data class MServiceDate( fun toFile(agencyTools: GAgencyTools) = buildList { if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { - add(MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId))) // service ID int + add(MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId))) } else { - add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) // service ID + add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) } add(calendarDate.toString()) add(exceptionType.toString()) @@ -92,8 +92,8 @@ data class MServiceDate( .takeIf { it.size == 3 } ?.let { columns -> MServiceDate( - serviceIdInt = GIDs.getInt(columns[0].unquotes()), // service ID - calendarDate = columns[1].toInt(), // calendar date + serviceIdInt = GIDs.getInt(columns[0].unquotes()), + calendarDate = columns[1].toInt(), exceptionType = columns[2].toInt() ) } diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt index c3142b61..4671e91a 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt @@ -36,14 +36,15 @@ object MServiceIds { @Suppress("unused") @JvmStatic - fun getString(serviceIdInt: Int): String { - return idIntToId[serviceIdInt] ?: throw MTLog.Fatal("Unexpected Service ID integer $serviceIdInt!") - } + fun getString(serviceIdInt: Int) = + idIntToId[serviceIdInt] ?: throw MTLog.Fatal("Unexpected Service ID integer $serviceIdInt!") @JvmStatic - fun getInt(serviceId: String): Int { - return idToIdInt[serviceId] ?: add(serviceId) - } + fun getInt(serviceId: String): Int = + idToIdInt[serviceId] + ?: synchronized(incrementLock) { + return idToIdInt[serviceId] ?: add(serviceId) + } @JvmStatic fun getAll() = buildList { From 8f7a6d3b29c7ccc060176eedbb417f196eea886c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Tue, 18 Nov 2025 20:29:51 -0500 Subject: [PATCH 12/13] PR comments --- src/main/java/org/mtransit/parser/mt/data/MServiceId.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt index 9ed78955..bb346092 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt @@ -10,6 +10,9 @@ data class MServiceId( val serviceId: String, // already agencyTools.cleanServiceId(serviceId) before ) : Comparable { + /** + * same order as [org.mtransit.commons.GTFSCommons.T_SERVICE_IDS_SQL_INSERT] + */ fun toFile() = buildList { add(serviceIdInt.toString()) add(serviceId.quotesEscape()) // already agencyTools.cleanServiceId(serviceId) before From adcdca992a4678d4e82e83989b7baa56c376266e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Tue, 18 Nov 2025 20:35:19 -0500 Subject: [PATCH 13/13] wip --- src/main/java/org/mtransit/parser/mt/MGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index d24b48fc..9c3e570d 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -176,6 +176,7 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age MTLog.log("- Trips: %d", mTripsList.size()); MTLog.log("- Trip stops: %d", mDirectionStopsList.size()); MTLog.log("- Stops: %d", mStopsList.size()); + MTLog.log("- Service Ids: %d", MServiceIds.getAll().size()); MTLog.log("- Service Dates: %d", mServiceDatesList.size()); MTLog.log("- Route with Frequencies: %d", mRouteFrequencies.size()); MTLog.log("- First timestamp: %s", MTLog.formatDateTime(firstTimestamp)); @@ -549,7 +550,6 @@ private static void dumpServiceIds( } for (MServiceId mServiceId : MServiceIds.getAll()) { final String serviceIdsInsert = mServiceId.toFile(); - // gAgencyTools if (F_PRE_FILLED_DB) { SQLUtils.executeUpdate( dbStatement,