consistently treat utilities that would lead to zero choices in MC simulation

Author: Jan Zill

activitysim/abm/models/joint_tour_participation.py

@@ -213,19 +213,16 @@ def participants_chooser(
                 )
                 # anybody with probability > 0 is forced to join the joint tour
                 if state.settings.use_explicit_error_terms:
-                    # need "is valid choice", which requires minimum probability. This depends
-                    # on all other utilities, and we need to make sure that we set values such that we
-                    # certainly choose those with non-zero values, and do not choose other ones. Let's
-                    # use -999 as zero prob choice and 10 as definitive choice (based on diff and std gumbel).
-                    probs_from_utils = logit.utils_to_probs(state, probs_or_utils)
+                    # need "is valid choice" such that we certainly choose those with non-zero values,
+                    # and do not choose others. Let's use 3.0 as large value here.
                     probs_or_utils[choice_col] = np.where(
-                        probs_from_utils[choice_col] > 0, 10, -999
+                        probs_or_utils[choice_col] > logit.UTIL_MIN, 3.0, logit.UTIL_UNAVAILABLE
                     )
                     non_choice_col = [
                         col for col in probs_or_utils.columns if col != choice_col
                     ][0]
                     probs_or_utils[non_choice_col] = np.where(
-                        probs_or_utils[choice_col] == -999, 10, -999
+                        probs_or_utils[choice_col] <= logit.UTIL_MIN, 3.0, logit.UTIL_UNAVAILABLE
                     )
                 else:
                     probs_or_utils[choice_col] = np.where(

activitysim/abm/models/trip_departure_choice.py

@@ -314,7 +314,7 @@ def choose_tour_leg_pattern(
     chunk_sizer.log_df(trace_label, "sample_counts", None)
 
     # insert the zero-prob utilities to pad each alternative set to same size
-    padded_utilities = np.insert(interaction_utilities.utility.values, inserts, -999999)
+    padded_utilities = np.insert(interaction_utilities.utility.values, inserts, -999)
     del inserts
 
     del interaction_utilities
@@ -339,6 +339,9 @@ def choose_tour_leg_pattern(
         )
 
     if state.settings.use_explicit_error_terms:
+        utilities_df = logit.validate_utils(
+            state, utilities_df, trace_label=trace_label, trace_choosers=trip_segment
+        )
         # make choices
         # positions is series with the chosen alternative represented as a column index in probs
         # which is an integer between zero and num alternatives in the alternative sample

activitysim/abm/models/util/cdap.py

@@ -996,6 +996,8 @@ def household_activity_choices(
         utils = utils.add(joint_tour_utils)
 
     if state.settings.use_explicit_error_terms:
+        utils = logit.validate_utils(state, utils, trace_label=trace_label)
+
         idx_choices, rands = logit.make_choices_utility_based(
             state, utils, trace_label=trace_label
         )

activitysim/core/interaction_sample.py

@@ -43,6 +43,20 @@ def make_sample_choices_utility_based(
     assert isinstance(alternatives, pd.DataFrame)
     assert len(alternatives) == alternative_count
 
+    if allow_zero_probs:
+        zero_probs = (
+            utilities.sum(axis=1)
+            <= utilities.shape[1] * logit.UTIL_UNAVAILABLE
+        )
+        if zero_probs.all():
+            return pd.DataFrame(
+                columns=[alt_col_name, "rand", "prob", choosers.index.name]
+            )
+        if zero_probs.any():
+            # remove from sample
+            utilities = utilities[~zero_probs]
+            choosers = choosers[~zero_probs]
+
     utils_array = utilities.to_numpy()
     chunk_sizer.log_df(trace_label, "utils_array", utils_array)
     chosen_destinations = []
@@ -556,6 +570,14 @@ def _interaction_sample(
         return choices_df
 
     if use_eet:
+        utilities = logit.validate_utils(
+            state,
+            utilities,
+            allow_zero_probs=allow_zero_probs,
+            trace_label=trace_label,
+            trace_choosers=choosers,
+        )
+
         choices_df = make_sample_choices_utility_based(
             state,
             choosers,

activitysim/core/interaction_sample_simulate.py

@@ -262,14 +262,7 @@ def _interaction_sample_simulate(
     chunk_sizer.log_df(trace_label, "sample_counts", None)
 
     # insert the zero-prob utilities to pad each alternative set to same size
-    # -999 is not small enough for EET, there are edge cases where alternatives
-    # corresponding to padded utilities are chosen. TODO: proper zero-prob handling.
-    zero_probability_util = (
-        -999999
-    )  # min(interaction_utilities.utility.min().min() - 100, -999)
-    padded_utilities = np.insert(
-        interaction_utilities.utility.values, inserts, zero_probability_util
-    )
+    padded_utilities = np.insert(interaction_utilities.utility.values, inserts, -999)
     chunk_sizer.log_df(trace_label, "padded_utilities", padded_utilities)
     del inserts
 

activitysim/core/interaction_simulate.py

@@ -883,6 +883,9 @@ def _interaction_simulate(
     state.tracing.dump_df(DUMP, utilities, trace_label, "utilities")
 
     if state.settings.use_explicit_error_terms:
+        utilities = logit.validate_utils(
+            state, utilities, trace_label=trace_label, trace_choosers=choosers
+        )
         positions, rands = logit.make_choices_utility_based(
             state, utilities, trace_label=trace_label, trace_choosers=choosers
         )

activitysim/core/pathbuilder.py

@@ -991,45 +991,58 @@ def build_virtual_path(
                 utilities_df.index = orig.index
 
                 with memo("#TVPB build_virtual_path make_choices"):
-                    probs = logit.utils_to_probs(
-                        self.network_los.state,
-                        utilities_df,
-                        allow_zero_probs=True,
-                        trace_label=trace_label,
-                        overflow_protection=False,
-                    )
-                    chunk_sizer.log_df(trace_label, "probs", probs)
-
                     if trace:
+                        probs = logit.utils_to_probs(
+                            self.network_los.state,
+                            utilities_df,
+                            allow_zero_probs=True,
+                            trace_label=trace_label,
+                            overflow_protection=False,
+                        )
+                        chunk_sizer.log_df(trace_label, "probs", probs)
+
                         choices = override_choices
 
                         utilities_df["choices"] = choices
                         self.trace_df(utilities_df, trace_label, "utilities_df")
 
                         probs["choices"] = choices
                         self.trace_df(probs, trace_label, "probs")
+                        del probs
+                        chunk_sizer.log_df(trace_label, "probs", None)
                     else:
                         if self.network_los.state.settings.use_explicit_error_terms:
+                            utilities_df = logit.validate_utils(
+                                self.network_los.state, utilities_df, allow_zero_probs=True, trace_label=trace_label
+                            )
                             choices, rands = logit.make_choices_utility_based(
                                 self.network_los.state,
                                 utilities_df,
-                                allow_bad_probs=True,
                                 trace_label=trace_label,
                             )
                         else:
+                            probs = logit.utils_to_probs(
+                                self.network_los.state,
+                                utilities_df,
+                                allow_zero_probs=True,
+                                trace_label=trace_label,
+                                overflow_protection=False,
+                            )
+                            chunk_sizer.log_df(trace_label, "probs", probs)
+
                             choices, rands = logit.make_choices(
                                 self.network_los.state,
                                 probs,
                                 allow_bad_probs=True,
                                 trace_label=trace_label,
                             )
+                            del probs
+                            chunk_sizer.log_df(trace_label, "probs", None)
 
                         chunk_sizer.log_df(trace_label, "rands", rands)
                         del rands
                         chunk_sizer.log_df(trace_label, "rands", None)
 
-                    del probs
-                    chunk_sizer.log_df(trace_label, "probs", None)
 
                 # we need to get path_set, btap, atap from path_df row with same seq and path_num
                 # drop seq join column, but keep path_num of choice to override_choices when tracing

activitysim/core/simulate.py

@@ -1290,6 +1290,8 @@ def eval_mnl(
         )
 
     if state.settings.use_explicit_error_terms:
+        utilities = logit.validate_utils(state, utilities, trace_label=trace_label, trace_choosers=choosers)
+
         if custom_chooser:
             choices, rands = custom_chooser(
                 state, utilities, choosers, spec, trace_label
@@ -1427,6 +1429,11 @@ def eval_nl(
         )
 
     if state.settings.use_explicit_error_terms:
+        # TODO-EET: Nested utility zero choice probability
+        raw_utilities = logit.validate_utils(
+            state, raw_utilities, allow_zero_probs=True, trace_label=trace_label
+        )
+
         # utilities of leaves and nests
         nested_utilities = compute_nested_utilities(raw_utilities, nest_spec)
         chunk_sizer.log_df(trace_label, "nested_utilities", nested_utilities)

activitysim/core/test/test_logit.py

@@ -69,6 +69,7 @@ def utilities(choosers, spec, test_data):
         columns=test_data["probabilities"].columns,
     )
 
+# TODO-EET: Add tests here!
 
 def test_utils_to_probs(utilities, test_data):
     state = workflow.State().default_settings()