diff --git a/activitysim/abm/tables/shadow_pricing.py b/activitysim/abm/tables/shadow_pricing.py index b3c20108e8..9bbecd4c21 100644 --- a/activitysim/abm/tables/shadow_pricing.py +++ b/activitysim/abm/tables/shadow_pricing.py @@ -6,11 +6,13 @@ import logging import multiprocessing import time +import warnings from collections import OrderedDict from typing import Any, Literal import numpy as np import pandas as pd +from pydantic import model_validator from activitysim.abm.tables.size_terms import size_terms as get_size_terms from activitysim.abm.tables.size_terms import tour_destination_size_terms @@ -62,12 +64,6 @@ TALLY_CHECKOUT = (1, -1) TALLY_PENDING_PERSONS = (2, -1) -default_segment_to_name_dict = { - # model_selector : persons_segment_name - "school": "school_segment", - "workplace": "income_segment", -} - def size_table_name(model_selector): """ @@ -134,11 +130,20 @@ class ShadowPriceSettings(PydanticReadable, extra="forbid"): WRITE_ITERATION_CHOICES: bool = False - SEGMENT_TO_NAME: dict[str, str] = { - "school": "school_segment", - "workplace": "income_segment", - } # pydantic uses deep copy, so mutable default value is ok here - """Mapping from model_selector to persons_segment_name.""" + SEGMENT_TO_NAME: dict[str, str] | None = None + """Deprecated compatibility setting; chooser segmentation comes from model settings.""" + + @model_validator(mode="before") + @classmethod + def warn_deprecated_segment_to_name(cls, data: Any) -> Any: + if isinstance(data, dict) and "SEGMENT_TO_NAME" in data: + warnings.warn( + "SEGMENT_TO_NAME is deprecated and ignored; use " + "CHOOSER_SEGMENT_COLUMN_NAME in location choice settings instead", + DeprecationWarning, + stacklevel=2, + ) + return data class ShadowPriceCalculator: @@ -176,6 +181,7 @@ def __init__( ) self.model_selector = model_settings.MODEL_SELECTOR + self.chooser_segment_column = model_settings.CHOOSER_SEGMENT_COLUMN_NAME if (self.num_processes > 1) and not state.settings.fail_fast: # if we are multiprocessing, then fail_fast should be true or we will wait forever for failed processes @@ -869,13 +875,34 @@ def update_shadow_prices(self, state): sampled_persons = pd.DataFrame() persons_merged = state.get_dataframe("persons_merged") - # need to join the segment to the choices to sample correct persons - segment_to_name_dict = self.shadow_settings.SEGMENT_TO_NAME - segment_name = segment_to_name_dict[self.model_selector] + # Use the model chooser segmentation to keep shadow-pricing resampling + # consistent with segment_ids in location choice settings. + segment_name = self.chooser_segment_column + if segment_name not in persons_merged.columns: + raise SystemConfigurationError( + f"Missing chooser segment column '{segment_name}' in persons_merged " + f"for {self.model_selector} simulation shadow pricing" + ) if type(self.choices_synced) != pd.DataFrame: self.choices_synced = self.choices_synced.to_frame() + # Fail fast on obvious misconfiguration instead of silently sampling no one. + segment_values = set(self.segment_ids.values()) + chooser_values = set( + persons_merged[segment_name] + .reindex(self.choices_synced.index) + .dropna() + .unique() + ) + if not segment_values.intersection(chooser_values): + raise SystemConfigurationError( + f"No overlap between SEGMENT_IDS values ({sorted(segment_values)}) and " + f"modeled chooser values in persons_merged['{segment_name}'] " + f"for {self.model_selector} " + "simulation shadow pricing" + ) + choices_synced = self.choices_synced.merge( persons_merged[segment_name], how="left", diff --git a/activitysim/abm/test/test_misc/test_shadow_pricing_simulate.py b/activitysim/abm/test/test_misc/test_shadow_pricing_simulate.py index 6d8bcff6e6..288edc7000 100644 --- a/activitysim/abm/test/test_misc/test_shadow_pricing_simulate.py +++ b/activitysim/abm/test/test_misc/test_shadow_pricing_simulate.py @@ -12,6 +12,7 @@ from activitysim.abm.tables import shadow_pricing from activitysim.core import los, workflow from activitysim.core.configuration.logit import TourLocationComponentSettings +from activitysim.core.exceptions import SystemConfigurationError LAND_USE_FIELDS = [ "e01_nrm", @@ -333,6 +334,9 @@ def persons() -> pd.DataFrame: } ) + persons["school_segment_string"] = persons["school_segment"].astype(str) + persons["school_segment_string_99"] = "99" + return persons @@ -572,6 +576,163 @@ def test_shadow_pricing_simulate(state, model_settings, network_los): ) +def test_shadow_pricing_simulate_custom_segment(state, model_settings, network_los): + """Run simulation shadow pricing with string-valued chooser segments.""" + segment_ids = { + "university": "3", + "highschool": "2", + "gradeschool": "1", + } + custom_model_settings = model_settings.model_copy( + update={ + "CHOOSER_SEGMENT_COLUMN_NAME": "school_segment_string", + "SEGMENT_IDS": segment_ids, + } + ) + custom_model_settings.LOGSUM_SETTINGS = None + + spc = shadow_pricing.load_shadow_price_calculator(state, custom_model_settings) + + max_iterations = 5 + chooser_segment_column = "school_segment_string" + save_sample_df = choices_df = None + persons_merged = state.get_dataframe("persons_merged") + + for iteration in range(1, max_iterations + 1): + old_shadow_prices = spc.shadow_prices["highschool"].values + persons_merged_df_ = persons_merged.copy() + + if spc.use_shadow_pricing and iteration > 1: + spc.update_shadow_prices(state) + + if spc.shadow_settings.SHADOW_PRICE_METHOD == "simulation": + persons_merged_df_ = persons_merged_df_[ + persons_merged_df_.index.isin(spc.sampled_persons.index) + ].sort_index() + + choices_df_, save_sample_df = run_location_choice( + state, + persons_merged_df_, + network_los, + shadow_price_calculator=spc, + want_logsums=False, + want_sample_table=False, + estimator=None, + model_settings=custom_model_settings, + chunk_size=0, + chunk_tag="school_location_string_segment", + trace_label=f"school_location_string_segment_{iteration}", + ) + + if spc.use_shadow_pricing: + if ( + spc.shadow_settings.SHADOW_PRICE_METHOD == "simulation" + and iteration > 1 + ): + if len(choices_df_) != 0: + choices_df = pd.concat([choices_df, choices_df_], axis=0) + choices_df_index = choices_df_.index.name + choices_df = choices_df.reset_index() + choices_df = choices_df.drop_duplicates( + subset=[choices_df_index], keep="last" + ) + choices_df = choices_df.set_index(choices_df_index).sort_index() + else: + choices_df = choices_df_.copy() + + new_shadow_prices = spc.shadow_prices["highschool"].values + assert not any((old_shadow_prices == -999) & (new_shadow_prices != -999)) + check_shadow_prices(spc, iteration) + + spc.set_choices( + choices=choices_df["choice"], + segment_ids=persons_merged[chooser_segment_column].reindex( + choices_df.index + ), + ) + + +def test_shadow_pricing_simulate_segment_values_do_not_overlap( + state, model_settings, network_los +): + custom_segment_column = "school_segment_string_99" + custom_model_settings = model_settings.model_copy( + update={ + "CHOOSER_SEGMENT_COLUMN_NAME": custom_segment_column, + "SEGMENT_IDS": { + "university": "3", + "highschool": "2", + "gradeschool": "1", + }, + } + ) + persons_merged = state.get_dataframe("persons_merged") + state.settings.use_shadow_pricing = True + + try: + spc = shadow_pricing.load_shadow_price_calculator(state, custom_model_settings) + choices = pd.Series(22660, index=persons_merged.index, name="choice", dtype=int) + spc.set_choices(choices, persons_merged[custom_segment_column]) + + with pytest.raises(SystemConfigurationError) as error: + spc.update_shadow_prices(state) + + assert str(error.value) == ( + "No overlap between SEGMENT_IDS values (['1', '2', '3']) and " + "modeled chooser values in persons_merged['school_segment_string_99'] for school " + "simulation shadow pricing" + ) + finally: + persons_merged.pop(custom_segment_column) + + +def test_shadow_pricing_simulate_segment_overlap_ignores_unmodeled_choosers( + state, model_settings, network_los +): + custom_segment_column = "school_segment_string_99" + custom_model_settings = model_settings.model_copy( + update={ + "CHOOSER_SEGMENT_COLUMN_NAME": custom_segment_column, + "SEGMENT_IDS": { + "university": "3", + "highschool": "2", + "gradeschool": "1", + }, + } + ) + persons_merged = state.get_dataframe("persons_merged") + persons_merged[custom_segment_column] = "99" + unmodeled_chooser = persons_merged.index[0] + persons_merged.loc[unmodeled_chooser, custom_segment_column] = "1" + modeled_choosers = persons_merged.drop(index=unmodeled_chooser) + state.settings.use_shadow_pricing = True + + try: + spc = shadow_pricing.load_shadow_price_calculator(state, custom_model_settings) + choices = pd.Series( + 22660, index=modeled_choosers.index, name="choice", dtype=int + ) + spc.set_choices(choices, modeled_choosers[custom_segment_column]) + + with pytest.raises( + SystemConfigurationError, match="No overlap between SEGMENT_IDS" + ): + spc.update_shadow_prices(state) + finally: + persons_merged.pop(custom_segment_column) + + +def test_shadow_price_settings_accept_deprecated_segment_to_name(): + legacy_setting = {"school": "school_segment"} + + with pytest.warns(DeprecationWarning, match="SEGMENT_TO_NAME is deprecated"): + settings = shadow_pricing.ShadowPriceSettings.model_validate( + {"SEGMENT_TO_NAME": legacy_setting} + ) + + assert settings.SEGMENT_TO_NAME == legacy_setting + + def test_shadow_pricing_dedicated_rng_channel_eet_only( state, model_settings, network_los ): diff --git a/docs/dev-guide/changes.md b/docs/dev-guide/changes.md index 2afe63ff89..16361539b5 100644 --- a/docs/dev-guide/changes.md +++ b/docs/dev-guide/changes.md @@ -14,6 +14,18 @@ branch (i.e., the main branch on GitHub), but not yet released in a stable versi of ActivitySim. No unreleased changes are currently documented. +### Shadow Pricing Re-Simulation Segmentation + +In version 1.6.1, the segmentation used for shadow pricing re-simulation now +inherits the segmentation used in the work and school location choice simulation, +e.g., defined as `CHOOSER_SEGMENT_COLUMN_NAME` in the `workplace_location.yaml`. +The legacy setting `SEGMENT_TO_NAME` in the `shadow_pricing.yaml` is now deprecated +and ignored. This ensures consistency between the segmentation used for shadow +pricing re-simulation and the segmentation used for work and school location +choice simulation. See +[Issue #1082](https://github.com/ActivitySim/activitysim/issues/1082) and +[PR #1083](https://github.com/ActivitySim/activitysim/pull/1083). + ## v1.6 diff --git a/docs/dev-guide/components/shadow_pricing.md b/docs/dev-guide/components/shadow_pricing.md index c618858fe8..9a9c3e760c 100644 --- a/docs/dev-guide/components/shadow_pricing.md +++ b/docs/dev-guide/components/shadow_pricing.md @@ -50,10 +50,32 @@ zone on or off for selection in the subsequent iterations. For this reason, warm for this approach is not applicable.) This process repeats until the overall convergence criteria is met or the maximum number of allowed iterations is reached. +Note that the simulation approach compares the modeled employment/enrollment share against the target +employment/enrollment share for each zone, rather than the absolute number. Therefore, it does not require +the absolute total workers/students to match the regional employment/enrollment totals. + Because the simulation approach only re-simulates workers / students who were over-assigned in the previous iteration, run time is significantly less (~90%) than the CTRAMP or DaySim approaches which re-simulate all workers and students at each iteration. +When choosing workers or students for re-simulation, the simulation approach selects individuals from +overassigned zones by shadow pricing segment. In version 1.6.0 and prior, the segmentation was defaulted +to use income groups for workers and school segments for students, and changable via the `SEGMENT_TO_NAME` +setting in the `shadow_pricing.yaml`. However, this setting was easily overlooked or misconfigured, +which could cause the shadow pricing to be accidentally skipped or cause a mismatch between the segmentation +used for shadow pricing re-simulation and the segmentation used for work and school location choice simulation, +if the latter uses a different segmentation scheme from the default income and school segments. +In version 1.6.1, the segmentation used for shadow pricing re-simulation now inherits the segmentation +used in the work and school location choice simulation, e.g., defined as `CHOOSER_SEGMENT_COLUMN_NAME` in +the `workplace_location.yaml`. The legacy setting `SEGMENT_TO_NAME` in the `shadow_pricing.yaml` is now +deprecated and ignored. + +The segmentation in shadow pricing re-simulation, e.g., occupation or income groups, accommodates but +does not require occupation- or income-specific land use employment controls. For example, +the example SANDAG model segments workplace location choice by occupation, but all occupation segments +share the same `total_emp_revised` target. Under the SANDAG configuration, ActivitySim combines the modeled +workers across segments sharing that target when evaluating whether a zone is over- or under-assigned. + ## Configuration ```{eval-rst}