diff --git a/.gitignore b/.gitignore index bf50b1f..d2cf20b 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ /demo/logs /demo/data /tests/images +/tests/data /tests/logs /tests/integration/data /tests/integration/logs diff --git a/docs/source/conf.py b/docs/source/conf.py index a073f17..a48aaaf 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,11 +28,11 @@ source_suffix = '.rst' master_doc = 'index' project = u'UrbanAccess' -author = u'UrbanSim Inc.' +author = u'UrbanSim Inc. and Samuel D. Blanchard' copyright = u'{}, {}'.format(datetime.now().year, author) version = u'0.2.2' -release = u'0.2.2' -language = None +release = version +language = 'en' # List of patterns to ignore when looking for source files. exclude_patterns = ['_build'] @@ -54,7 +54,7 @@ # Output file base name for HTML help builder. htmlhelp_basename = 'UrbanAccessdoc' html_show_sphinx = False -html_show_sourcelink = True +html_show_sourcelink = False # -- Options for LaTeX output --------------------------------------------- latex_elements = { diff --git a/urbanaccess/config.py b/urbanaccess/config.py index 67bea78..5c62b2f 100644 --- a/urbanaccess/config.py +++ b/urbanaccess/config.py @@ -1,7 +1,8 @@ -import os -import yaml +import itertools import numpy as np +from urbanaccess.utils import _dict_to_yaml, _yaml_to_dict + def _format_check(settings): """ @@ -15,20 +16,37 @@ def _format_check(settings): ------- Nothing """ - - valid_keys = ['data_folder', 'images_folder', 'image_filename', 'logs_folder', - 'log_file', 'log_console', 'log_name', 'log_filename', - 'txt_encoding', 'gtfs_api'] - - for key in settings.keys(): - if key not in valid_keys: - raise ValueError('{} not found in list of valid configuration ' - 'keys: {}.'.format(key, valid_keys)) - if not isinstance(key, str): - raise ValueError('{} must be a string.'.format(key)) - if key == 'log_file' or key == 'log_console': + gtfs_api_schema_keys = ['gtfsdataexch'] + str_keys = ['data_folder', 'images_folder', 'image_filename', + 'logs_folder', 'log_name', 'log_filename', 'txt_encoding'] + bool_keys = ['log_file', 'log_console'] + dict_keys = ['gtfs_api'] + valid_keys = list(itertools.chain(str_keys, bool_keys, dict_keys)) + valid_keys = sorted(valid_keys) + settings_keys = list(settings.keys()) + + if set(settings_keys) != set(valid_keys): + raise ValueError("Configuration keys: {} do not match required " + "keys: {}.".format(settings_keys, valid_keys)) + for key in settings_keys: + if key in str_keys: + if not isinstance(settings[key], str): + raise ValueError( + "Key: '{}' value must be string.".format(key)) + if key in bool_keys: if not isinstance(settings[key], bool): - raise ValueError('{} must be boolean.'.format(key)) + raise ValueError( + "Key: '{}' value must be boolean.".format(key)) + if key in dict_keys: + for gtfs_api_key in settings['gtfs_api'].keys(): + if gtfs_api_key not in gtfs_api_schema_keys: + raise ValueError( + "gtfs_api key: '{}' does not match valid key(s):" + " {}.".format(gtfs_api_key, gtfs_api_schema_keys)) + if not isinstance(settings['gtfs_api'][gtfs_api_key], str): + raise ValueError( + "gtfs_api key: '{}' value must be string.".format( + gtfs_api_key)) # TODO: make class CamelCase @@ -105,19 +123,7 @@ def from_yaml(cls, configdir='configs', ------- urbanaccess_config """ - - if not isinstance(configdir, str): - raise ValueError('configdir must be a string') - if not os.path.exists(configdir): - raise ValueError('{} does not exist or was not found'.format( - configdir)) - if not isinstance(yamlname, str): - raise ValueError('yaml must be a string') - - yaml_file = os.path.join(configdir, yamlname) - - with open(yaml_file, 'r') as f: - yaml_config = yaml.safe_load(f) + yaml_config = _yaml_to_dict(yaml_dir=configdir, yaml_name=yamlname) settings = cls( data_folder=yaml_config.get('data_folder', 'data'), @@ -168,29 +174,14 @@ def to_yaml(self, configdir='configs', yamlname='urbanaccess_config.yaml', yamlname : str or file like, optional File name to which to save a YAML file. overwrite : bool, optional - if true, overwrite an existing same name YAML - file in specified directory + if true, will overwrite an existing YAML + file in specified directory if file names are the same Returns ------- Nothing """ - - if not isinstance(configdir, str): - raise ValueError('configdir must be a string') - if not os.path.exists(configdir): - raise ValueError('{} does not exist or was not found'.format( - configdir)) - os.makedirs(configdir) - if not isinstance(yamlname, str): - raise ValueError('yaml must be a string') - yaml_file = os.path.join(configdir, yamlname) - if overwrite is False and os.path.isfile(yaml_file) is True: - raise ValueError( - '{} already exists. Rename or turn overwrite to True'.format( - yamlname)) - else: - with open(yaml_file, 'w') as f: - yaml.dump(self.to_dict(), f, default_flow_style=False) + _dict_to_yaml(dictionary=self.to_dict(), yaml_dir=configdir, + yaml_name=yamlname, overwrite=overwrite) # set global variables @@ -279,7 +270,7 @@ def to_yaml(self, configdir='configs', yamlname='urbanaccess_config.yaml', _GTFS_TXT_FILE_TYPES = { 'required_files': ['stops.txt', 'routes.txt', 'trips.txt', 'stop_times.txt'], - 'optional_files': ['agency.txt'], + 'optional_files': ['agency.txt', 'shapes.txt'], 'calendar_files': ['calendar.txt', 'calendar_dates.txt']} _SUPPORTED_GTFS_TXT_FILES = [] @@ -300,7 +291,10 @@ def to_yaml(self, configdir='configs', yamlname='urbanaccess_config.yaml', 'remove_whitespace': ['stop_id'], 'min_required_cols': ['stop_id', 'stop_lat', 'stop_lon']}, 'routes': {'req_dtypes': {'route_id': object}, - 'opt_dtypes': None, + 'opt_dtypes': {'route_short_name': object, + 'route_long_name': object, + 'route_color': object, + 'route_text_color': object}, 'numeric_converter': None, 'remove_whitespace': ['route_id'], 'min_required_cols': ['route_id']}, @@ -309,7 +303,8 @@ def to_yaml(self, configdir='configs', yamlname='urbanaccess_config.yaml', 'route_id': object}, 'opt_dtypes': {'shape_id': object}, 'numeric_converter': None, - 'remove_whitespace': ['trip_id', 'service_id', 'route_id'], + 'remove_whitespace': ['trip_id', 'service_id', 'route_id', + 'shape_id'], 'min_required_cols': ['trip_id', 'service_id', 'route_id']}, 'stop_times': {'req_dtypes': {'trip_id': object, @@ -318,10 +313,20 @@ def to_yaml(self, configdir='configs', yamlname='urbanaccess_config.yaml', 'arrival_time': object}, 'opt_dtypes': None, 'numeric_converter': None, - 'remove_whitespace': ['trip_id', 'stop_id'], + 'remove_whitespace': ['trip_id', 'stop_id', + 'departure_time', 'arrival_time'], 'min_required_cols': ['trip_id', 'stop_id', 'departure_time', 'arrival_time']}, - 'calendar': {'req_dtypes': {'service_id': object}, + 'shapes': {'req_dtypes': {'shape_id': object}, + 'opt_dtypes': None, + 'numeric_converter': ['shape_pt_lat', 'shape_pt_lon', + 'shape_pt_sequence'], + 'remove_whitespace': ['shape_id'], + 'min_required_cols': ['shape_id', 'shape_pt_lat', + 'shape_pt_lon', 'shape_pt_sequence']}, + 'calendar': {'req_dtypes': {'service_id': object, + 'start_date': object, + 'end_date': object}, 'opt_dtypes': None, 'numeric_converter': ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', @@ -331,7 +336,9 @@ def to_yaml(self, configdir='configs', yamlname='urbanaccess_config.yaml', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday', 'start_date', 'end_date']}, - 'calendar_dates': {'req_dtypes': {'service_id': object}, + 'calendar_dates': {'req_dtypes': {'service_id': object, + 'date': object, + 'exception_type': object}, 'opt_dtypes': None, 'numeric_converter': None, 'remove_whitespace': ['service_id'], diff --git a/urbanaccess/gtfs/headways.py b/urbanaccess/gtfs/headways.py index e0ac0e3..eecc2d0 100644 --- a/urbanaccess/gtfs/headways.py +++ b/urbanaccess/gtfs/headways.py @@ -2,7 +2,7 @@ import pandas as pd import time -from urbanaccess.utils import log +from urbanaccess.utils import log, _add_unique_trip_id, _add_unique_route_id from urbanaccess.gtfs.utils_validation import _check_time_range_format from urbanaccess.gtfs.network import _time_selector @@ -24,35 +24,44 @@ def _calc_headways_by_route_stop(df): DataFrame : pandas.DataFrame DataFrame of statistics of route stop headways in units of minutes """ - # TODO: Optimize for speed - + # TODO: Update to incorporate optional direction_id into group, if + # direction_id doesnt exist do not use it since its an optional col + # TODO: Assess if we can simplify the results to return only unique records start_time = time.time() - df['unique_stop_route'] = ( - df['unique_stop_id'].str.cat( - df['unique_route_id'].astype('str'), sep=',')) + df = _add_unique_stop_route(df) stop_route_groups = df.groupby('unique_stop_route') log('Starting route stop headway calculation for {:,} route ' 'stops...'.format(len(stop_route_groups))) - results = {} - # suppress RuntimeWarning: Mean of empty slice. for this code block with warnings.catch_warnings(): warnings.simplefilter("ignore", category='RuntimeWarning') - + results = {} + col = 'departure_time_sec_interpolate' for unique_stop_route, stop_route_group in stop_route_groups: - stop_route_group.sort_values(['departure_time_sec_interpolate'], - ascending=True, inplace=True) - next_bus_time = (stop_route_group['departure_time_sec_interpolate'] - .iloc[1:].values) - prev_bus_time = (stop_route_group['departure_time_sec_interpolate'] - .iloc[:-1].values) - stop_route_group_headways = (next_bus_time - prev_bus_time) / 60 - results[unique_stop_route] = (pd.Series(stop_route_group_headways) - .describe()) + stop_route_group_headways = pd.DataFrame() + if 'direction_id' not in stop_route_group.columns: + stop_route_group.sort_values([col], ascending=True, inplace=True) + next_veh_time = (stop_route_group[col].iloc[1:].values) + prev_veh_time = (stop_route_group[col].iloc[:-1].values) + # compute headway with: + # (next veh arrival time - previous veh arrival time) / 60 seconds + # = headway in min + stop_route_group_headways = (next_veh_time - prev_veh_time) / 60 + results[unique_stop_route] = (pd.Series(stop_route_group_headways) + .describe()) + else: + for direction in stop_route_group.direction_id.unique(): + stop_route_group_dir = stop_route_group[stop_route_group['direction_id']==direction] + stop_route_group_dir.sort_values([col], ascending=True, inplace=True) + next_veh_time = (stop_route_group_dir[col].iloc[1:].values) + prev_veh_time = (stop_route_group_dir[col].iloc[:-1].values) + headways_dir = pd.DataFrame((next_veh_time - prev_veh_time) / 60, columns=['headway']) + stop_route_group_headways = pd.concat([stop_route_group_headways, headways_dir]) + results[unique_stop_route] = (stop_route_group_headways['headway'].describe()) log('Route stop headway calculation complete. Took {:,.2f} seconds'.format( time.time() - start_time)) @@ -84,15 +93,13 @@ def _headway_handler(interpolated_stop_times_df, trips_df, DataFrame of statistics of route stop headways in units of minutes with relevant route and stop information """ + # TODO: reconsider the use of inplace or copy incoming dfs + # to memory to avoid changing in memory tables start_time = time.time() # add unique trip and route ID - trips_df['unique_trip_id'] = ( - trips_df['trip_id'].str.cat( - trips_df['unique_agency_id'].astype('str'), sep='_')) - trips_df['unique_route_id'] = ( - trips_df['route_id'].str.cat( - trips_df['unique_agency_id'].astype('str'), sep='_')) + trips_df = _add_unique_trip_id(trips_df) + trips_df = _add_unique_route_id(trips_df) columns = ['unique_route_id', 'service_id', 'unique_trip_id', 'unique_agency_id'] @@ -105,9 +112,7 @@ def _headway_handler(interpolated_stop_times_df, trips_df, trips_df = trips_df[columns] # add unique route ID - routes_df['unique_route_id'] = ( - routes_df['route_id'].str.cat( - routes_df['unique_agency_id'].astype('str'), sep='_')) + routes_df = _add_unique_route_id(routes_df) columns = ['unique_route_id', 'route_long_name', 'route_type', 'unique_agency_id'] @@ -133,9 +138,7 @@ def _headway_handler(interpolated_stop_times_df, trips_df, merge_df[['unique_stop_route', 'unique_stop_id', 'unique_route_id']], how='left', left_index=True, right_on='unique_stop_route', sort=False) headway_by_routestop_df.drop('unique_stop_route', axis=1, inplace=True) - headway_by_routestop_df['node_id_route'] = ( - headway_by_routestop_df['unique_stop_id'].str.cat( - headway_by_routestop_df['unique_route_id'].astype('str'), sep='_')) + headway_by_routestop_df = _add_node_id_route(headway_by_routestop_df) log('Headway calculation complete. Took {:,.2f} seconds.'.format( time.time() - start_time)) @@ -182,3 +185,43 @@ def headways(gtfsfeeds_df, headway_timerange): gtfsfeeds_df.headways = headways_df return gtfsfeeds_df + + +def _add_unique_stop_route(df): + """ + Create 'unique_stop_route' column and values in a pandas.DataFrame. + Intended for use with headways. + + Parameters + ---------- + df : pandas.DataFrame + pandas.DataFrame to generate 'unique_stop_route' column + + Returns + ------- + df : pandas.DataFrame + pandas.DataFrames with 'unique_stop_route' column added + """ + df['unique_stop_route'] = df['unique_stop_id'].str.cat( + df['unique_route_id'].astype('str'), sep=',') + return df + + +def _add_node_id_route(df): + """ + Create 'node_id_route' column and values in a pandas.DataFrame. + Intended for use with headways. + + Parameters + ---------- + df : pandas.DataFrame + pandas.DataFrame to generate 'node_id_route' column + + Returns + ------- + df : pandas.DataFrame + pandas.DataFrames with 'node_id_route' column added + """ + df['node_id_route'] = df['unique_stop_id'].str.cat( + df['unique_route_id'].astype('str'), sep='_') + return df diff --git a/urbanaccess/gtfs/load.py b/urbanaccess/gtfs/load.py index ea2a59b..ee2951a 100644 --- a/urbanaccess/gtfs/load.py +++ b/urbanaccess/gtfs/load.py @@ -143,23 +143,25 @@ def _txt_header_whitespace_check( file_path, encoding=txt_encoding) as f: lines = f.readlines() - line_wo_whitespace = re.sub(r'\s+', '', lines[0]) + '\n' - # only write the file if there are changes to be made - if lines[0] != line_wo_whitespace: - msg = 'Removing whitespace from header(s) in: {}...' - log(msg.format(file_path)) - lines[0] = line_wo_whitespace - # Write to file - if six.PY2: - with open( - file_path, 'w') as f: - f.writelines(lines) - else: - # write with default 'utf-8' encoding - with open( - file_path, 'w', - encoding=txt_encoding) as f: - f.writelines(lines) + if len(lines) != 0: + line_wo_whitespace = re.sub(r'\s+', '', lines[0]) + '\n' + # only write the file if there are changes to be made + if lines[0] != line_wo_whitespace: + msg = ('Removing whitespace from ' + 'header(s) in: {}...') + log(msg.format(file_path)) + lines[0] = line_wo_whitespace + # Write to file + if six.PY2: + with open( + file_path, 'w') as f: + f.writelines(lines) + else: + # write with default 'utf-8' encoding + with open( + file_path, 'w', + encoding=txt_encoding) as f: + f.writelines(lines) except Exception as e: msg = 'Unable to process: {}. Exception: {}' raise Exception(log(msg.format(file_path, e), @@ -210,6 +212,7 @@ def gtfsfeed_to_df(gtfsfeed_path=None, validation=False, verbose=True, gtfsfeeds_dfs.routes : pandas.DataFrame gtfsfeeds_dfs.trips : pandas.DataFrame gtfsfeeds_dfs.stop_times : pandas.DataFrame + gtfsfeeds_dfs.shapes : pandas.DataFrame gtfsfeeds_dfs.calendar : pandas.DataFrame gtfsfeeds_dfs.calendar_dates : pandas.DataFrame """ @@ -218,25 +221,30 @@ def gtfsfeed_to_df(gtfsfeed_path=None, validation=False, verbose=True, merged_routes_df = pd.DataFrame() merged_trips_df = pd.DataFrame() merged_stop_times_df = pd.DataFrame() + merged_shapes_df = pd.DataFrame() merged_calendar_df = pd.DataFrame() merged_calendar_dates_df = pd.DataFrame() start_time = time.time() + dir_error_msg = "Directory: '{}' does not exist." if gtfsfeed_path is None: gtfsfeed_path = os.path.join( config.settings.data_folder, 'gtfsfeed_text') if not os.path.exists(gtfsfeed_path): - raise ValueError('{} does not exist.'.format(gtfsfeed_path)) + raise ValueError(dir_error_msg.format(gtfsfeed_path)) else: if not os.path.exists(gtfsfeed_path): - raise ValueError('{} does not exist.'.format(gtfsfeed_path)) + raise ValueError(dir_error_msg.format(gtfsfeed_path)) if not isinstance(gtfsfeed_path, str): raise ValueError('gtfsfeed_path must be a string.') if validation: - if bbox is None or remove_stops_outsidebbox is None or verbose is \ - None: + has_bbox_param = bbox is not None + has_remove_stops_param = remove_stops_outsidebbox is not None + has_verbose_param = verbose is not None + if not has_bbox_param or not has_remove_stops_param or not \ + has_verbose_param: raise ValueError( 'Attempted to run validation but bbox, verbose, and or ' 'remove_stops_outsidebbox were set to None. These parameters ' @@ -332,25 +340,34 @@ def gtfsfeed_to_df(gtfsfeed_path=None, validation=False, verbose=True, textfile=textfile) else: agency_df = pd.DataFrame() + if textfile == 'shapes.txt': + if textfile in textfilelist: + shapes_df = utils_format._read_gtfs_file( + textfile_path=os.path.join(gtfsfeed_path, folder), + textfile=textfile) + else: + shapes_df = pd.DataFrame() - stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ - calendar_dates_df = utils_format._add_unique_agencyid( + stops_df, routes_df, trips_df, stop_times_df, shapes_df, calendar_df, \ + calendar_dates_df = utils_format._add_unique_agency_id( agency_df=agency_df, stops_df=stops_df, routes_df=routes_df, trips_df=trips_df, stop_times_df=stop_times_df, + shapes_df=shapes_df, calendar_df=calendar_df, calendar_dates_df=calendar_dates_df, feed_folder=os.path.join(gtfsfeed_path, folder), nulls_as_folder=True) - stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ + stops_df, routes_df, trips_df, stop_times_df, shapes_df, calendar_df, \ calendar_dates_df = utils_format._add_unique_gtfsfeed_id( stops_df=stops_df, routes_df=routes_df, trips_df=trips_df, stop_times_df=stop_times_df, + shapes_df=shapes_df, calendar_df=calendar_df, calendar_dates_df=calendar_dates_df, feed_folder=folder, @@ -381,18 +398,20 @@ def gtfsfeed_to_df(gtfsfeed_path=None, validation=False, verbose=True, trips_df=trips_df[['trip_id', 'route_id']], info_to_append='route_type_to_stop_times') - merged_stops_df = merged_stops_df.append( - stops_df, ignore_index=True) - merged_routes_df = merged_routes_df.append( - routes_df, ignore_index=True) - merged_trips_df = merged_trips_df.append( - trips_df, ignore_index=True) - merged_stop_times_df = merged_stop_times_df.append( - stop_times_df, ignore_index=True) - merged_calendar_df = merged_calendar_df.append( - calendar_df, ignore_index=True) - merged_calendar_dates_df = merged_calendar_dates_df.append( - calendar_dates_df, ignore_index=True) + merged_stops_df = pd.concat([merged_stops_df, + stops_df], ignore_index=True) + merged_routes_df = pd.concat([merged_routes_df, + routes_df], ignore_index=True) + merged_trips_df = pd.concat([merged_trips_df, + trips_df], ignore_index=True) + merged_stop_times_df = pd.concat([merged_stop_times_df, + stop_times_df], ignore_index=True) + merged_shapes_df = pd.concat([merged_shapes_df, + shapes_df], ignore_index=True) + merged_calendar_df = pd.concat([merged_calendar_df, + calendar_df], ignore_index=True) + merged_calendar_dates_df = pd.concat([merged_calendar_dates_df, + calendar_dates_df], ignore_index=True) # print break to visually separate each GTFS feed log log('--------------------------------') @@ -413,6 +432,7 @@ def gtfsfeed_to_df(gtfsfeed_path=None, validation=False, verbose=True, gtfsfeeds_dfs.routes = merged_routes_df gtfsfeeds_dfs.trips = merged_trips_df gtfsfeeds_dfs.stop_times = merged_stop_times_df + gtfsfeeds_dfs.shapes = merged_shapes_df gtfsfeeds_dfs.calendar = merged_calendar_df gtfsfeeds_dfs.calendar_dates = merged_calendar_dates_df diff --git a/urbanaccess/gtfs/network.py b/urbanaccess/gtfs/network.py index 35a3d18..cf83b8c 100644 --- a/urbanaccess/gtfs/network.py +++ b/urbanaccess/gtfs/network.py @@ -5,8 +5,12 @@ from datetime import datetime, timedelta import logging as lg -from urbanaccess.utils import log, df_to_hdf5, hdf5_to_df +from urbanaccess.utils import log, df_to_hdf5, hdf5_to_df, \ + _add_unique_trip_id, _add_unique_stop_id, _add_unique_route_id + from urbanaccess.gtfs.utils_validation import _check_time_range_format +from urbanaccess.gtfs.utils_calendar import _calendar_service_id_selector, \ + _trip_selector, _highest_freq_trips_date from urbanaccess.network import ua_network from urbanaccess import config from urbanaccess.gtfs.gtfsfeeds_dataframe import gtfsfeeds_dfs, \ @@ -17,8 +21,8 @@ def create_transit_net( gtfsfeeds_dfs, - day, - timerange, + day=None, + timerange=None, calendar_dates_lookup=None, overwrite_existing_stop_times_int=False, use_existing_stop_times_int=False, @@ -26,7 +30,12 @@ def create_transit_net( save_dir=config.settings.data_folder, save_filename=None, timerange_pad=None, - time_aware=False): + time_aware=False, + date=None, + date_range=None, + use_highest_freq_trips_date=False, + simplify=False +): """ Create a travel time weight network graph in units of minutes from GTFS data @@ -38,9 +47,18 @@ def create_transit_net( stop_times, calendar, calendar_dates (optional) and stop_times_int (optional) day : {'monday', 'tuesday', 'wednesday', 'thursday', - 'friday', 'saturday', 'sunday'} - day of the week to extract transit schedule from that - corresponds to the day in the GTFS calendar + 'friday', 'saturday', 'sunday'}, optional + day of the week to extract active service IDs in calendar and or + calendar_dates tables. If GTFS feeds have only calendar, service IDs + that are active on day specified will be extracted. If GTFS feeds + have only calendar_dates, service IDs that are active on dates + that match the day of the week specified will be extracted where + exception_type = 1 (service is added for date). If GTFS feeds have + both calendar and calendar_dates, calendar service IDs that are + active on day specified will be extracted and calendar_dates + service IDs that are active on dates that match the day of the week + specified will be added to those found in the calendar where + exception_type = 1 (service is added for date). timerange : list time range to extract transit schedule from in a list with time 1 and time 2 as strings. It is suggested the time range @@ -52,10 +70,12 @@ def create_transit_net( calendar_dates_lookup : dict, optional dictionary of the lookup column (key) as a string and corresponding string (value) as string or list of strings to use to subset trips - using the calendar_dates DataFrame. Search will be exact. If none, - then the calendar_dates DataFrame will not be used to select trips - that are not in the calendar DataFrame. Note search will select all - records that meet each key value pair criteria. + using the calendar_dates DataFrame. Parameter should only be used if + there is a high degree of certainly that the GTFS feed in use has + service_ids listed in calendar_dates that would otherwise not be + selected if using any of the other calendar_dates parameters such as: + 'day', 'date' or 'date_range'. Search will be exact and will select + all records that meet each key value pair criteria. Example: {'schedule_type' : 'WD'} or {'schedule_type' : ['WD', 'SU']} overwrite_existing_stop_times_int : bool, optional if true, and if there is an existing stop_times_int @@ -83,6 +103,63 @@ def create_transit_net( from the stop_times table will be included in the transit edge table where 'departure_time' is the departure time at node_id_from stop and 'arrival_time' is the arrival time at node_id_to stop + date : str, optional + date to extract active service IDs in calendar and or + calendar_dates tables specified as a string in YYYY-MM-DD format, + e.g. '2013-03-09'. If GTFS feeds have only calendar, service IDs + that are active on the date specified (where the date is within the + date range given by the start_date and end_date columns) and + that date's day of the week will be extracted. If GTFS feeds have + only calendar_dates, service IDs that are active on the date + specified will be extracted where exception_type = 1 + (service is added for date). If GTFS feeds have both calendar + and calendar_dates, calendar service IDs that are active on the date + specified (where the date is within the date range given by the + start_date and end_date columns) and that date's day of the week + will be extracted and calendar_dates service IDs that are active + on the date specified will be added to those found in the + calendar where exception_type = 1 (service is added for date) + and service IDs that are inactive on the date specified will be + removed from those found in the calendar where exception_type = 2 + (service is removed for date). + date_range : list, optional + date range to extract active service IDs in calendar and or + calendar_dates tables specified as a 2 element list of strings + where the first element is the first date and the second element + is the last date in the date range. Dates should be specified + as strings in YYYY-MM-DD format, e.g. ['2013-03-09', '2013-09-01']. + If GTFS feeds have only calendar, service IDs that are active + within the date range specified (where the date range is within + the date range given by the start_date and end_date columns) + will be extracted. If GTFS feeds have only calendar_dates, + service IDs that are active within the date range specified + will be extracted where exception_type = 1 + (service is added for date). If GTFS feeds have both calendar + and calendar_dates, calendar service IDs that are active within + the date range specified (where the date range is within the date + range given by the state_date and end_date columns) will be + extracted and calendar_dates service IDs that are active within + the date range specified will be added to those found in the + calendar where exception_type = 1 (service is added for date). + This parameter is best utilized if its known that the GTFS + feed calendar table has seasonal service IDs. + use_highest_freq_trips_date : boolean, optional + when True, the date that contains the highest number of active trips + will be determined and will be used to select active service IDs. + See the docstring for the 'date' parameter to understand how the + 'date' is then used to select service IDs. If multiple dates are found + to contain the highest number of trips, and they contain the same + trip IDs, one date from those found will be used. If they do not + contain the same trip IDs, an error will be raised to prompt you + to explicitly set one of the dates found as the date in the 'date' + parameter. + simplify : boolean + when True, the transit network is significantly reduced in size by + collapsing trips that share identical characteristics including + agency, travel time, stop sequence, and route into one unique + representative trip with its corresponding edges. It is suggested to + use simplification when using networks with network analysis tools + such as Pandana for computational efficiency. Default is False. Returns ------- @@ -118,6 +195,11 @@ def create_transit_net( if overwrite_existing_stop_times_int and use_existing_stop_times_int: raise ValueError('overwrite_existing_stop_times_int and ' 'use_existing_stop_times_int cannot both be True.') + if use_highest_freq_trips_date and any([day, date, date_range]): + raise ValueError("Only one parameter: 'use_highest_freq_trips_date' " + "or one of 'day', 'date', or 'date_range' can be " + "used at a time or both 'day' and 'date_range' can " + "be used.") columns = ['route_id', 'direction_id', @@ -128,21 +210,50 @@ def create_transit_net( if 'direction_id' not in gtfsfeeds_dfs.trips.columns: columns.remove('direction_id') - # TODO: support use case where only calendar_dates is in use: make 'day' - # optional as None but require either day or calendar_dates_lookup - # to exist but both are not required - calendar_selected_trips_df = _trip_schedule_selector( - input_trips_df=gtfsfeeds_dfs.trips[columns], - input_calendar_df=gtfsfeeds_dfs.calendar, - input_calendar_dates_df=gtfsfeeds_dfs.calendar_dates, - day=day, - calendar_dates_lookup=calendar_dates_lookup) + # since data can be composed of multiple feeds and agencies, + # run calendar service ID operations on each individual agency to + # be transparent on what service IDs are being extracted from + # each agency + active_service_ids = [] + agency_id_col = 'unique_agency_id' + agency_ids = gtfsfeeds_dfs.trips[agency_id_col].unique() + for agency in agency_ids: + log('--------------------------------') + log('Running active service ID selection operation for ' + 'agency: {}'.format(agency)) + agency_trips = gtfsfeeds_dfs.trips.loc[ + gtfsfeeds_dfs.trips[agency_id_col] == agency] + agency_calendar = gtfsfeeds_dfs.calendar.loc[ + gtfsfeeds_dfs.calendar[agency_id_col] == agency] + agency_calendar_dates = gtfsfeeds_dfs.calendar_dates.loc[ + gtfsfeeds_dfs.calendar_dates[agency_id_col] == agency] + + if use_highest_freq_trips_date: + date = _highest_freq_trips_date( + agency_trips, + agency_calendar, + agency_calendar_dates) + + agency_active_service_ids = _calendar_service_id_selector( + calendar_df=agency_calendar, + calendar_dates_df=agency_calendar_dates, + day=day, + date=date, + date_range=date_range, + cal_dates_lookup=calendar_dates_lookup) + # combine each agency's active service IDs + active_service_ids.extend(agency_active_service_ids) + + # use aggregated active_service_ids to get all active trip IDs + calendar_selected_trips_df = _trip_selector( + trips_df=gtfsfeeds_dfs.trips[columns], + service_ids=active_service_ids) # proceed to calc stop_times_int if stop_times_int is already empty, or # overwrite existing is True, or use existing is False - if gtfsfeeds_dfs.stop_times_int.empty or \ - overwrite_existing_stop_times_int or use_existing_stop_times_int \ - is False: + existing_stop_times_int = gtfsfeeds_dfs.stop_times_int.empty + if existing_stop_times_int or overwrite_existing_stop_times_int or \ + use_existing_stop_times_int is False: if overwrite_existing_stop_times_int: log(' Overwriting existing stop_times_int DataFrame...') gtfsfeeds_dfs.stop_times_int = _interpolate_stop_times( @@ -184,6 +295,24 @@ def create_transit_net( transit_edges = _route_id_to_edge( transit_edge_df=transit_edges, trips_df=gtfsfeeds_dfs.trips) + transit_nodes = _remove_nodes_not_in_edges( + nodes=transit_nodes, edges=transit_edges, + from_id_col='node_id_from', to_id_col='node_id_to') + + if transit_edges.empty: + msg = ( + "Resulting transit network edges have 0 record(s). Most likely " + "the 'timerange' and or calendar selection parameters resulted in " + "a network with no active trips within the schedule window " + "specified. It is suggested to expand the 'timerange' and or " + "alter the calendar selection parameters to enlarge the time " + "window for selecting active trips.") + raise ValueError(msg) + + if simplify: + transit_edges, transit_nodes = _simplify_transit_net( + transit_edges, transit_nodes) + # assign node and edge net type transit_nodes['net_type'] = 'transit' transit_edges['net_type'] = 'transit' @@ -198,277 +327,100 @@ def create_transit_net( return ua_network -def _trip_schedule_selector(input_trips_df, input_calendar_df, - input_calendar_dates_df, day, - calendar_dates_lookup=None): +def _simplify_transit_net(edges, nodes): """ - Select trips that correspond to a specific schedule in either calendar.txt - and or calendar_dates.txt by finding service_ids that correspond to the - specified search parameters and the trips related to those service_ids + Simplifies edge and node tables by removing trips from the edge table + that have identical properties that result in duplicate edge attributes + for use in routing engines such as Pandana. Edges are grouped by + columns: 'node_id_from', 'node_id_to', 'weight', 'unique_agency_id', + 'sequence', 'route_type', and 'unique_route_id'. If multiple trip IDs are + found to have the exact same attributes in those columns, only one + representative trip ID and its corresponding edges will be preserved out + of the group. Parameters ---------- - input_trips_df : pandas.DataFrame - trips DataFrame - input_calendar_df : pandas.DataFrame - calendar DataFrame - input_calendar_dates_df : pandas.DataFrame - calendar_dates DataFrame - day : {'monday', 'tuesday', 'wednesday', 'thursday', - 'friday', 'saturday', 'sunday'} - day of the week to extract transit schedule that corresponds to the - day in the GTFS calendar - calendar_dates_lookup : dict, optional - dictionary of the lookup column (key) as a string and corresponding - string (value) as string or list of strings to use to subset trips - using the calendar_dates DataFrame. Search will be exact. If none, - then the calendar_dates DataFrame will not be used to select trips - that are not in the calendar DataFrame. Note search will select all - records that meet each key value pair criteria. - Example: {'schedule_type' : 'WD'} or {'schedule_type' : ['WD', 'SU']} + edges : pandas.DataFrame + edges DataFrame to simplify + nodes : pandas.DataFrame + nodes DataFrame to simplify Returns ------- - calendar_selected_trips_df : pandas.DataFrame - + simp_edges : pandas.DataFrame + simplified edge DataFrame + simp_nodes : pandas.DataFrame + simplified node DataFrame """ + # TODO: add option for groups that are duplicates, place trip ids + # of the groups that are removed in a new column as list of + # strings for downstream debugging or for informative trip counts + # TODO: can simplify even more by reducing along edge shared attr in + # addition to trips, must ensure that stop ids are connected start_time = time.time() - valid_days = ['monday', 'tuesday', 'wednesday', 'thursday', - 'friday', 'saturday', 'sunday'] - - if day not in valid_days: - valid_days_str = str(valid_days).replace('[', '').replace(']', '') - raise ValueError('Incorrect day specified. Must be one of lowercase ' - 'strings: {}.'.format(valid_days_str)) - - # check format of calendar_dates_lookup - if calendar_dates_lookup is not None: - if not isinstance(calendar_dates_lookup, dict): - raise ValueError( - 'calendar_dates_lookup parameter must be a dictionary.') - for key in calendar_dates_lookup.keys(): - if not isinstance(key, str): - raise ValueError('calendar_dates_lookup key: {} ' - 'must be a string.'.format(key)) - - if isinstance(calendar_dates_lookup[key], str): - value = [calendar_dates_lookup[key]] - else: - if not isinstance(calendar_dates_lookup[key], list): - raise ValueError( - 'calendar_dates_lookup value: {} must be a string or ' - 'a list of strings.'.format( - calendar_dates_lookup[key])) - else: - value = calendar_dates_lookup[key] - - for string in value: - if not isinstance(string, str): - raise ValueError('calendar_dates_lookup value: {} ' - 'must contain strings.'.format(value)) - - # check if calendar dfs and related params are empty or not to determine - # what will be used in processing - has_calendar = input_calendar_df.empty is False - has_calendar_param = day is not None - has_calendar_dates = input_calendar_dates_df.empty is False - has_calendar_dates_param = calendar_dates_lookup is not None - - if not has_calendar: - log('calendar table has no records and will not be used to ' - 'select trips.') - if has_calendar_param: - log("Warning: calendar is empty. " - "Unable to use the 'day' parameter.", level=lg.WARNING) - if has_calendar_dates: - if not has_calendar_dates_param: - log("calendar_dates table has records however the " - "'calendar_dates_lookup' parameter is None, no trips will be " - "selected using calendar_dates.") - else: - log('calendar_dates table has no records and will not be used to ' - 'select trips.') - if has_calendar_dates_param: - raise ValueError("calendar_dates is empty. Unable to use the " - "'calendar_dates_lookup' parameter. Set to None.") - - # create unique service IDs for dfs in list if they are not empty - df_list = [input_trips_df] - if has_calendar: - df_list.extend([input_calendar_df]) - if has_calendar_dates: - df_list.extend([input_calendar_dates_df]) - for index, df in enumerate(df_list): - df['unique_service_id'] = (df['service_id'].str.cat( - df['unique_agency_id'].astype('str'), sep='_')) - df_list[index] = df - - service_ids_df = pd.DataFrame() - - # collect service IDs that match search parameters in calendar.txt - if has_calendar and has_calendar_param: - # select service IDs where day specified has a 1 = service - # runs on that day - log('Using calendar to extract service_ids to select trips...') - service_ids_df = input_calendar_df[(input_calendar_df[day] == 1)] - service_ids_df = service_ids_df[['unique_service_id']] - num_cal_service_ids_extracted = len(service_ids_df) - log('{:,} service_ids were extracted from calendar.'.format( - num_cal_service_ids_extracted)) - - # generate information needed to tell user the status of their trips in - # terms of service_ids in calendar table - trips_in_calendar = input_trips_df.loc[ - input_trips_df['unique_service_id'].isin( - service_ids_df['unique_service_id'])] - trips_notin_calendar = input_trips_df.loc[ - ~input_trips_df['unique_service_id'].isin( - service_ids_df['unique_service_id'])] - cnt_input_trips_df = len(input_trips_df) - cnt_trips_in_calendar = len(trips_in_calendar) - pct_trips_in_calendar = round(cnt_trips_in_calendar / len( - input_trips_df) * 100, 2) - - feeds_wtrips_in_cal = trips_in_calendar['unique_feed_id'].unique() - print_feed_ids = [' '.join(feed_id.split('_')[:-1]) for feed_id in - feeds_wtrips_in_cal] - feeds_wotrips_in_cal = trips_notin_calendar['unique_feed_id'].unique() - if print_feed_ids: - log('{:,} trip(s) {:.2f} percent of {:,} total trip records were ' - 'found in calendar for GTFS feed(s): {}.'.format( - cnt_trips_in_calendar, pct_trips_in_calendar, - cnt_input_trips_df, print_feed_ids)) - - feed_id_not_in_cal = [x for x in feeds_wotrips_in_cal if - x not in feeds_wtrips_in_cal] - for feed_id in feed_id_not_in_cal: - trip_feed_name = ' '.join(feed_id.split('_')[:-1]) - log('0 trip(s) 0 percent of {:,} total trip records were ' - 'found in calendar for GTFS feed: {}.'.format( - cnt_input_trips_df, trip_feed_name)) - - # warn user that if they have a calendar_dates table and they - # expected more trips to be selected from the calendar table that - # they should consider using the calendar_dates table to supplement - # the selection of trips - if has_calendar_dates and len(trips_notin_calendar) > 0 and \ - has_calendar_dates_param is False: - warning_msg = ( - 'NOTE: If you expected more trips to have been extracted and ' - 'your GTFS feed(s) have a calendar_dates file, consider ' - 'utilizing the calendar_dates_lookup parameter in order to ' - 'add additional trips based on information inside of ' - 'calendar_dates. This should only be done if you know the ' - 'corresponding GTFS feed is using calendar_dates instead of ' - 'calendar to specify service_ids. When in doubt do not use ' - 'the calendar_dates_lookup parameter.') - log(warning_msg, level=lg.WARNING) - - if len(feeds_wtrips_in_cal) != len(feeds_wotrips_in_cal) and \ - calendar_dates_lookup is None: - for feed_id in feeds_wotrips_in_cal: - trip_feed_name = ' '.join(feed_id.split('_')[:-1]) - log('{:,} trip(s) {:.2f} percent of {:,} total trip records ' - 'were not found in calendar for GTFS feed: {}.'.format( - cnt_trips_in_calendar, pct_trips_in_calendar, - cnt_input_trips_df, trip_feed_name)) - if feed_id not in feeds_wtrips_in_cal: - log('Warning: GTFS feed: {} no trips were selected using ' - 'calendar. It is suggested you use the ' - 'calendar_dates_lookup parameter to utilize this ' - 'feed\'s calendar_dates file.'.format(trip_feed_name), - level=lg.WARNING) + log('Running transit network simplification...') + + if edges.empty: + raise ValueError( + 'Unable to simplify transit network. Edges have 0 records to ' + 'simplify.') + + # columns to use for group to group value comparison to identify + # groups of similar values for simplification + col_list = ['node_id_from', 'node_id_to', 'weight', 'unique_agency_id', + 'sequence', 'route_type', 'unique_route_id'] + + # group records that have identical values for each col in col_list + # tag each record in group as True if the group exists more than once + id_col = 'unique_trip_id' + edges_wdup = edges.groupby(id_col)[col_list].agg(tuple).sum(1).duplicated() + + # remove records where their group was duplicated + simp_edges = edges.loc[edges[id_col].isin(edges_wdup[~edges_wdup].index)] + + # simplify nodes by removing nodes that do not exist in the simplified + # edge table, this catches edges cases but normally there should never be + # node ids in the node table that are not in the edge table + simp_nodes = _remove_nodes_not_in_edges( + nodes=nodes, edges=simp_edges, + from_id_col='node_id_from', to_id_col='node_id_to') + + # calculate various data len and print statistics + trip_remove_cnt = len(edges_wdup.loc[edges_wdup == True]) # noqa + trip_org_tot_cnt = len(edges[id_col].unique()) + trip_proc_tot_cnt = len(simp_edges[id_col].unique()) + trip_remove_pct = (trip_remove_cnt / trip_org_tot_cnt) * 100 + edge_org_rec_tot_cnt = len(edges) + edge_proc_rec_tot_cnt = len(simp_edges) + edge_remove_cnt = edge_org_rec_tot_cnt - edge_proc_rec_tot_cnt + edge_remove_pct = (edge_remove_cnt / edge_org_rec_tot_cnt) * 100 + msg = ('Transit edges have been simplified removing {:,} trip(s) ' + '({:.2f} percent) (reduced from {:,} to {:,} trip(s)) ' + 'resulting in the removal of {:,} edge(s) ({:.2f} percent) ' + '(reduced from {:,} to {:,} edges(s)).') + if edge_remove_cnt > 0: + log(msg.format(trip_remove_cnt, trip_remove_pct, + trip_org_tot_cnt, trip_proc_tot_cnt, + edge_remove_cnt, edge_remove_pct, + edge_org_rec_tot_cnt, edge_proc_rec_tot_cnt)) else: - num_cal_service_ids_extracted = 0 - cnt_input_trips_df = 0 - - # collect service IDs that match search parameters in calendar_dates.txt - if has_calendar_dates and has_calendar_dates_param: - # look for service_ids inside of calendar_dates if calendar does not - # supply enough service_ids to select trips by - if has_calendar: - if len(trips_notin_calendar) > 0: - log('Using calendar_dates to supplement service_ids extracted ' - 'from calendar to select trips...') - - subset_result_df = pd.DataFrame() - - for col_name_key, string_value in calendar_dates_lookup.items(): - if col_name_key not in input_calendar_dates_df.columns: - raise ValueError('Column: {} not found in calendar_dates ' - 'DataFrame.'.format(col_name_key)) - if col_name_key not in input_calendar_dates_df.select_dtypes( - include=[object]).columns: - raise ValueError('Column: {} must be object type.'.format( - col_name_key)) - - if not isinstance(string_value, list): - string_value = [string_value] - - for text in string_value: - # TODO: modify this in order to allow subset based on GTFS - # feed name or a or/and condition - subset_result = input_calendar_dates_df[ - input_calendar_dates_df[col_name_key].str.match( - text, case=False, na=False)] - cnt_subset_result = len(subset_result) - if cnt_subset_result != 0: - feed_id_list = subset_result['unique_feed_id'].unique() - for index, id in enumerate(feed_id_list): - feed_id_list[index] = ' '.join(id.split('_')[:-1]) - - log('Found {:,} record(s) that matched query: column: {} ' - 'and string: {} for GTFS feed(s): {}.'.format( - cnt_subset_result, col_name_key, text, feed_id_list)) - - subset_result_df = subset_result_df.append(subset_result) - - subset_result_df.drop_duplicates(inplace=True) - subset_result_df = subset_result_df[['unique_service_id']] - - num_caldates_service_ids_extracted = len(subset_result_df) - tot_service_ids_extracted = \ - num_caldates_service_ids_extracted + num_cal_service_ids_extracted - log('An additional {:,} service_id(s) were extracted from ' - 'calendar_dates. Total service_id(s) extracted: {:,}.'.format( - num_caldates_service_ids_extracted, tot_service_ids_extracted)) - service_ids_df = service_ids_df.append(subset_result_df) - service_ids_df.drop_duplicates(inplace=True) - - if service_ids_df.empty: - raise ValueError('No service_id(s) were found with ' - 'the specified calendar and or calendar_dates ' - 'search parameters.') - - # select and create df of trips that match the service IDs for the day of - # the week specified merge calendar df that has service IDs for - # specified day with trips df - calendar_selected_trips_df = input_trips_df.loc[ - input_trips_df['unique_service_id'].isin( - service_ids_df['unique_service_id'])] - - sort_columns = ['route_id', 'trip_id', 'direction_id'] - if 'direction_id' not in calendar_selected_trips_df.columns: - sort_columns.remove('direction_id') - calendar_selected_trips_df.sort_values(by=sort_columns, inplace=True) - calendar_selected_trips_df.reset_index(drop=True, inplace=True) - calendar_selected_trips_df.drop('unique_service_id', axis=1, inplace=True) - - calendar_selected_trips_count = len(calendar_selected_trips_df) - if calendar_dates_lookup is None: - log('{:,} of {:,} total trips were extracted representing calendar ' - 'day: {}. Took {:,.2f} seconds.'.format( - calendar_selected_trips_count, cnt_input_trips_df, day, - time.time() - start_time)) - else: - log('{:,} of {:,} total trips were extracted representing calendar ' - 'day: {} and calendar_dates search parameters: {}. ' - 'Took {:,.2f} seconds.'.format( - calendar_selected_trips_count, cnt_input_trips_df, day, - calendar_dates_lookup, time.time() - start_time)) - - return calendar_selected_trips_df + log('Transit edges cannot be further simplified.') + + node_org_tot_cnt = len(nodes) + node_proc_tot_cnt = len(simp_nodes) + # if nodes have a different count after simplification notify the user, + # however this should never occur + if node_org_tot_cnt != node_proc_tot_cnt: + node_remove_cnt = node_org_tot_cnt - node_proc_tot_cnt + node_remove_pct = (node_remove_cnt / node_org_tot_cnt) * 100 + msg = ('Transit nodes have been simplified removing {:,} nodes(s) ' + '({:.2f} percent) (reduced from {:,} to {:,} nodes(s)).') + log(msg.format(node_remove_cnt, node_remove_pct, node_org_tot_cnt, + node_proc_tot_cnt)) + log('Transit edge simplification complete. ' + 'Took {:,.2f} seconds.'.format(time.time() - start_time)) + return simp_edges, simp_nodes def _interpolate_stop_times(stop_times_df, calendar_selected_trips_df): @@ -488,15 +440,13 @@ def _interpolate_stop_times(stop_times_df, calendar_selected_trips_df): final_stop_times_df : pandas.DataFrame """ - start_time = time.time() # create unique trip IDs df_list = [calendar_selected_trips_df, stop_times_df] for index, df in enumerate(df_list): - df['unique_trip_id'] = (df['trip_id'].str.cat( - df['unique_agency_id'].astype('str'), sep='_')) + df = _add_unique_trip_id(df) df_list[index] = df # sort stop times inplace based on first to last stop in @@ -682,9 +632,7 @@ def _interpolate_stop_times(stop_times_df, calendar_selected_trips_df): final_stop_times_df['departure_time_sec_interpolate'].astype(int) # add unique stop ID - final_stop_times_df['unique_stop_id'] = ( - final_stop_times_df['stop_id'].str.cat( - final_stop_times_df['unique_agency_id'].astype('str'), sep='_')) + final_stop_times_df = _add_unique_stop_id(final_stop_times_df) if missing_stop_times_count > 0: log('Departure stop time interpolation complete. ' @@ -730,7 +678,7 @@ def _time_selector(df, starttime, endtime, timerange_pad=None): starttime : str 24 hour clock formatted time 1 endtime : str - 24 hour clock formatted time 2, + 24 hour clock formatted time 2 timerange_pad: str, optional string indicating the number of hours minutes seconds to pad after the end of the time interval specified in 'timerange'. Must follow format @@ -835,6 +783,16 @@ def _format_transit_net_edge(stop_times_df, time_aware=False): log('Starting transformation process for {:,} ' 'total trips...'.format(len(stop_times_df['unique_trip_id'].unique()))) + if stop_times_df.empty: + raise ValueError( + "Unable to continue processing transit network. stop_times " + "table has 0 records. Most likely the 'timerange' and or " + "calendar selection parameters resulted in a network with no " + "active trips within the schedule window specified. It is " + "suggested to expand the 'timerange' and or alter the calendar " + "selection parameters to enlarge the time window for selecting " + "active trips.") + # subset to only columns needed for processing cols_of_interest = ['unique_trip_id', 'stop_id', 'unique_stop_id', 'timediff', 'stop_sequence', 'unique_agency_id', @@ -854,7 +812,7 @@ def _format_transit_net_edge(stop_times_df, time_aware=False): for trip, tmp_trip_df in stop_times_df.groupby(['unique_trip_id']): # if 'time_aware', also create arrival and departure time cols if time_aware: - edge_df = pd.DataFrame({ + edge_df = pd.DataFrame.from_dict({ "node_id_from": tmp_trip_df['unique_stop_id'].iloc[:-1].values, "node_id_to": tmp_trip_df['unique_stop_id'].iloc[1:].values, "weight": tmp_trip_df['timediff'].iloc[1:].values, @@ -869,9 +827,9 @@ def _format_transit_net_edge(stop_times_df, time_aware=False): # arrival_time at node_id_to stop "arrival_time": tmp_trip_df['arrival_time'].iloc[1:].values - }) + }, orient='index').T.ffill() else: - edge_df = pd.DataFrame({ + edge_df = pd.DataFrame.from_dict({ "node_id_from": tmp_trip_df['unique_stop_id'].iloc[:-1].values, "node_id_to": tmp_trip_df['unique_stop_id'].iloc[1:].values, "weight": tmp_trip_df['timediff'].iloc[1:].values, @@ -880,7 +838,7 @@ def _format_transit_net_edge(stop_times_df, time_aware=False): # set unique trip ID without edge order to join other data # later "unique_trip_id": trip - }) + }, orient='index').T.ffill().dropna() # Set current trip ID to edge ID column adding edge order at # end of string @@ -959,9 +917,7 @@ def _stops_in_edge_table_selector(input_stops_df, input_stop_times_df): start_time = time.time() # add unique stop ID - input_stops_df['unique_stop_id'] = ( - input_stops_df['stop_id'].str.cat( - input_stops_df['unique_agency_id'].astype('str'), sep='_')) + input_stops_df = _add_unique_stop_id(input_stops_df) # Select stop IDs that match stop IDs in the subset stop time data that # match day and time selection @@ -995,9 +951,7 @@ def _format_transit_net_nodes(df): # add unique stop ID if 'unique_stop_id' not in df.columns: - df['unique_stop_id'] = ( - df['stop_id'].str.cat( - df['unique_agency_id'].astype('str'), sep='_')) + df = _add_unique_stop_id(df) final_node_df = pd.DataFrame() final_node_df['node_id'] = df['unique_stop_id'] @@ -1042,22 +996,14 @@ def _route_type_to_edge(transit_edge_df, stop_time_df): start_time = time.time() # create unique trip IDs - stop_time_df['unique_trip_id'] = ( - stop_time_df['trip_id'].str.cat( - stop_time_df['unique_agency_id'].astype('str'), sep='_')) + stop_time_df = _add_unique_trip_id(stop_time_df) # join route_id to the edge table - merged_df = pd.merge( - transit_edge_df, stop_time_df[['unique_trip_id', 'route_type']], - how='left', on='unique_trip_id', sort=False, copy=False) - merged_df.drop_duplicates( - subset='unique_trip_id', keep='first', inplace=True) - # need to get unique records here to have a one to one join - - # this serves as the look up table - # join the look up table created above to the table of interest - transit_edge_df_w_routetype = pd.merge( - transit_edge_df, merged_df[['route_type', 'unique_trip_id']], - how='left', on='unique_trip_id', sort=False, copy=False) + transit_edge_df = transit_edge_df.reset_index().set_index('unique_trip_id') + stop_time_df = stop_time_df[['unique_trip_id', 'route_type']].set_index('unique_trip_id') + stop_time_df = stop_time_df[~stop_time_df.index.duplicated(keep='first')] + merged_df = transit_edge_df.join(stop_time_df[['route_type']]) + transit_edge_df_w_routetype = merged_df.reset_index() log('Route type successfully joined to transit edges. ' 'Took {:,.2f} seconds.'.format(time.time() - start_time)) @@ -1085,12 +1031,8 @@ def _route_id_to_edge(transit_edge_df, trips_df): if 'unique_route_id' not in transit_edge_df.columns: # create unique trip and route IDs - trips_df['unique_trip_id'] = ( - trips_df['trip_id'].str.cat( - trips_df['unique_agency_id'].astype('str'), sep='_')) - trips_df['unique_route_id'] = ( - trips_df['route_id'].str.cat( - trips_df['unique_agency_id'].astype('str'), sep='_')) + trips_df = _add_unique_trip_id(trips_df) + trips_df = _add_unique_route_id(trips_df) transit_edge_df_with_routes = pd.merge( transit_edge_df, trips_df[['unique_trip_id', 'unique_route_id']], @@ -1179,8 +1121,8 @@ def edge_impedance_by_route_type( # used in this function in case changes are made in the config obj if set(sorted(route_type_dict.keys())) != set( sorted(var_mode_id_lookup.keys())): - ValueError('ROUTES_MODE_TYPE_LOOKUP keys do not match keys in ' - 'var_mode_id_lookup. Keys must match.') + raise ValueError('ROUTES_MODE_TYPE_LOOKUP keys do not match keys in ' + 'var_mode_id_lookup. Keys must match.') for key, value in route_type_dict.items(): route_type_dict[key] = {'name': value, 'multiplier': var_mode_id_lookup[key]} @@ -1359,3 +1301,29 @@ def _check_if_index_name_in_cols(df): cols = df.columns.values iname = df.index.name return (iname in cols) + + +def _remove_nodes_not_in_edges(nodes, edges, from_id_col, to_id_col): + """ + Helper function to remove nodes from node DataFrame that are not found + in the edges DataFrame + + Parameters + ---------- + nodes : pandas.DataFrame + nodes DataFrame + edges : pandas.DataFrame + edges DataFrame + from_id_col : str + name of from ID column + to_id_col : str + name of to ID column + + Returns + ------- + nodes_df : pandas.DataFrame + node DataFrame with only the nodes that exist in the edge DataFrame + """ + edge_node_list = set(edges[from_id_col]) | set(edges[to_id_col]) + nodes_df = nodes.loc[nodes.index.isin(edge_node_list)] + return nodes_df diff --git a/urbanaccess/gtfs/utils_calendar.py b/urbanaccess/gtfs/utils_calendar.py new file mode 100644 index 0000000..daaf868 --- /dev/null +++ b/urbanaccess/gtfs/utils_calendar.py @@ -0,0 +1,1438 @@ +from __future__ import division +import pandas as pd +import time +from datetime import datetime as dt +import logging as lg + +from urbanaccess.utils import log, _add_unique_trip_id + + +def _trip_schedule_selector_validate_params(calendar_dates_df, params): + """ + Validate parameters passed to the _calendar_service_id_selector() + function. + + Parameters + ---------- + calendar_dates_df : pandas.DataFrame + Calendar dates DataFrame + params : dict + Parameters to validate as a dict + + Returns + ------- + Nothing + + """ + has_cal = params['has_cal'] + has_cal_dates = params['has_cal_dates'] + + day = params['day'] + date = params['date'] + date_range = params['date_range'] + cal_dates_lookup = params['cal_dates_lookup'] + + has_day_param = params['has_day_param'] + has_date_param = params['has_date_param'] + has_date_range_param = params['has_date_range_param'] + has_cal_dates_param = params['has_cal_dates_param'] + has_day_and_date_range_param = params['has_day_and_date_range_param'] + + # only one search parameter can be used at a time + all_param_list = [has_day_param, has_date_param, has_date_range_param] + all_param_true_cnt = all_param_list.count(True) + + # Note: combination of day and date_range is valid + if all_param_true_cnt > 1 and not has_day_and_date_range_param: + raise ValueError( + "Only one parameter: 'day', 'date', or 'date_range' can be used " + "at a time or both 'day' and 'date_range' can be used.") + + if has_day_param: + if not isinstance(day, str): + raise ValueError('Day must be a string.') + valid_days = ['monday', 'tuesday', 'wednesday', 'thursday', + 'friday', 'saturday', 'sunday'] + if day not in valid_days: + valid_days_str = str(valid_days).replace('[', '').replace(']', '') + raise ValueError( + 'Day: {} is not a supported day. Must be one of lowercase ' + 'strings: {}.'.format(day, valid_days_str)) + + if has_date_param: + if not isinstance(date, str): + raise ValueError('Date must be a string.') + try: + dt.strptime(date, '%Y-%m-%d') + except ValueError: + raise ValueError("Date: {} is not a supported date format. " + "Expected format: 'YYYY-MM-DD'.".format(date)) + + if has_date_range_param: + if len(date_range) != 2: + raise ValueError("Date range {} must have a " + "length of 2.".format(date_range)) + for date_item in date_range: + if not isinstance(date_item, str): + raise ValueError('Dates in date range must be a string.') + try: + dt.strptime(date_item, '%Y-%m-%d') + except ValueError: + raise ValueError("Date: {} in date range: {} is not a " + "supported date format. Expected format: " + "'YYYY-MM-DD'.".format(date_item, date_range)) + item_1 = dt.strptime(date_range[0], '%Y-%m-%d') + item_2 = dt.strptime(date_range[1], '%Y-%m-%d') + if item_1 > item_2: + raise ValueError("First date in date range {} must be less than " + "the last date.".format(date_range)) + + if has_cal_dates_param: + if has_cal_dates: + if not isinstance(cal_dates_lookup, dict): + raise ValueError( + 'calendar_dates_lookup parameter must be a dictionary.') + for key in cal_dates_lookup.keys(): + if not isinstance(key, str): + raise ValueError('calendar_dates_lookup key: {} ' + 'must be a string.'.format(key)) + if isinstance(cal_dates_lookup[key], str): + value = [cal_dates_lookup[key]] + else: + if not isinstance(cal_dates_lookup[key], list): + raise ValueError( + 'calendar_dates_lookup value: {} must be a ' + 'string or a list of strings.'.format( + cal_dates_lookup[key])) + else: + value = cal_dates_lookup[key] + + for string in value: + if not isinstance(string, str): + raise ValueError('calendar_dates_lookup value: {} ' + 'must contain strings.'.format(value)) + for col_name_key, string_value in cal_dates_lookup.items(): + if col_name_key not in calendar_dates_df.columns: + raise ValueError('Column: {} not found in calendar_dates ' + 'DataFrame.'.format(col_name_key)) + if col_name_key not in calendar_dates_df.select_dtypes( + include=[object]).columns: + raise ValueError('Calendar_dates column: {} must be ' + 'object type.'.format(col_name_key)) + else: + raise ValueError("Calendar_dates is empty. Unable to use the " + "'calendar_dates_lookup' parameter. Set to None.") + + if not has_cal: + log(' Calendar table has no records and will not be used to ' + 'select service_ids.') + if not has_cal_dates: + log(' Calendar dates table has no records and will not be used to ' + 'select service_ids.') + + +def _cal_date_dt_conversion(df, date_cols): + """ + Convert columns with dates as strings to datetime64[ns] format. Expected + incoming dates in format:'%y%m%d' (YYYY-MM-DD) + + Parameters + ---------- + df : pandas.DataFrame + DataFrame that contains date columns to convert to DateTime + format. Typically this will be the calendar DataFrame + date_cols : list + List of strings denoting date columns with string values to convert + to DateTime format. For example: ['start_date', 'end_date'] + + Returns + ------- + df : pandas.DataFrame + Returns the same passed df DataFrame with the columns specified in + date_cols converted to datetime64[ns] format. + + """ + for col in date_cols: + try: + df[col] = pd.to_datetime( + df[col], format='mixed') + except ValueError: + raise ValueError("Column: {} has values that are not in a " + "supported date format. Expected format: " + "'YYYY-MM-DD'.".format(col)) + return df + + +def _select_calendar_service_ids(calendar_df, params): + """ + Wrapper function to select service IDs from the calendar.txt that are + active according to the parameters specified + + Parameters + ---------- + calendar_df : pandas.DataFrame + Calendar DataFrame + params : dict + Parameters denoting how service IDs should be selected from table + and supporting information to guide the selection process + + Returns + ------- + srvc_ids : list + returns a list of service IDs from the calendar.txt that are + active according to the parameters specified + + """ + # collect service IDs that match search parameters in calendar.txt + msg = 'Selecting service_ids from calendar' + srvc_ids = [] + + day = params['day'] + date = params['date'] + date_range = params['date_range'] + has_day_param = params['has_day_param'] + has_date_param = params['has_date_param'] + has_date_range_param = params['has_date_range_param'] + has_day_and_date_range_param = params['has_day_and_date_range_param'] + + # convert cols to datetime expected format: 'yyyymmdd' e.g.: '20130825' + date_cols = ['start_date', 'end_date'] + calendar_df = _cal_date_dt_conversion(df=calendar_df, date_cols=date_cols) + + if has_day_param: + srvc_ids_day = _select_calendar_service_ids_by_day( + calendar_df, msg, day) + srvc_ids = srvc_ids_day.copy() + + if has_date_range_param: + srvc_ids_date_range = _select_calendar_service_ids_by_date_range( + calendar_df, msg, date_range) + srvc_ids = srvc_ids_date_range.copy() + + if has_date_param: + srvc_ids_date = _select_calendar_service_ids_by_date( + calendar_df, msg, date) + srvc_ids = srvc_ids_date.copy() + + if has_day_and_date_range_param: + # get the intersection between service IDs found via 'day' and + # 'date_range' params + print_msg = '{} that are active within date_range: {} on day: {}...' + log(print_msg.format(msg, date_range, day)) + intersect_srvc_ids = _intersect_cal_service_ids( + dict_1={'day': srvc_ids_day}, + dict_2={'date range': srvc_ids_date_range}) + srvc_ids = intersect_srvc_ids.copy() + + return srvc_ids + + +def _print_cal_service_ids_len(srvc_ids, table_name): + """ + Helper function to print counts of service IDs + + Parameters + ---------- + srvc_ids : list + list of selected service IDs to generate counts for print message + table_name : str + name of table where service IDs were selected from to include + in print message + + Returns + ------- + Nothing + + """ + cnt_msg = ' {:,} service_id(s) were selected from {}.' + warning_msg = ' Warning: No service_ids were selected from {}.' + srvc_ids_cnt = len(srvc_ids) + uni_srvc_ids_cnt = len(set(srvc_ids)) + # TODO: print count of records and unique IDs + if srvc_ids_cnt > 0: + log(cnt_msg.format(uni_srvc_ids_cnt, table_name)) + else: + log(warning_msg.format(table_name), level=lg.WARNING) + + +def _intersect_cal_service_ids(dict_1, dict_2, verbose=True): + """ + Return the intersection of two lists of service IDs that are set as + the values in two dicts + + Parameters + ---------- + dict_1 : dict + dict 1 where the key is the origin of the service IDs for informative + purposes and the value is a list of service IDs. + Example: {'date': srvc_ids_date} + dict_2 : dict + dict 2 where the key is the origin of the service IDs for informative + purposes and the value is a list of service IDs. + Example: {'day': srvc_ids_date} + verbose : boolean, optional + if False, turns off logging and print statements in this function + Returns + ------- + srvc_ids : list + list of intersected and unique selected service IDs + """ + srvc_id_1_name, srvc_id_1 = list(dict_1.items())[0] + srvc_id_2_name, srvc_id_2 = list(dict_2.items())[0] + srvc_ids = list(set(srvc_id_1) & set(srvc_id_2)) + if verbose: + srvc_ids_len = len(srvc_ids) + list_1_len = len(srvc_id_1) + list_2_len = len(srvc_id_2) + msg = (' Intersection between service_ids selected with: ' + '{} parameter (count: {:,}) and {} parameter (count: {:,}) ' + 'returned {:,} total service_id(s).'.format( + srvc_id_1_name, list_1_len, srvc_id_2_name, list_2_len, + srvc_ids_len)) + log(msg) + return srvc_ids + + +def _select_calendar_service_ids_by_day( + calendar_df, msg, day=None, verbose=True): + """ + Global search using day in calendar.txt to select service IDs where + day specified = 1 where 1 = service is active; 0 = service inactive + + Parameters + ---------- + calendar_df : pandas.DataFrame + Calendar DataFrame + msg : str + string to prepend to print statement for informative purposes + day : {'monday', 'tuesday', 'wednesday', 'thursday', + 'friday', 'saturday', 'sunday'} + day of the week to extract active service IDs in calendar. + verbose : boolean, optional + if False, turns off logging and print statements in this function + Returns + ------- + srvc_ids : list + list of active service IDs + """ + if verbose: + log(' {} that are active on day: {}...'.format(msg, day)) + subset_cal_df = calendar_df.loc[calendar_df[day] == 1] + srvc_ids = subset_cal_df['unique_service_id'].to_list() + + if verbose: + _print_cal_service_ids_len(srvc_ids, table_name='calendar') + return srvc_ids + + +def _select_calendar_service_ids_by_date_range( + calendar_df, msg, date_range=None): + """ + Global search using date_range to subset records in calendar.txt + using 'start_date' and 'end_date' columns when GTFS feed has + seasonal service_ids select service IDs where + 'start_date' and 'end_date' are within the date_range + + Parameters + ---------- + calendar_df : pandas.DataFrame + Calendar DataFrame + msg : str + string to prepend to print statement for informative purposes + date_range : list + date range to extract active service IDs in calendar table + specified as a 2 element list of strings where the first element + is the first date and the second element is the last date in the + date range. Dates should be specified as strings in YYYY-MM-DD format, + e.g. ['2013-03-09', '2013-09-01']. Service IDs that are active + within the date range specified (where the date range is within + the date range given by the start_date and end_date columns) + will be extracted. + + Returns + ------- + srvc_ids : list + list of active service IDs + """ + log(' {} that are active within date_range: {}...'.format( + msg, date_range)) + + # select service_ids that are within the date range + service_ids = calendar_df['unique_service_id'] + start_d_mask = service_ids.loc[calendar_df['start_date'].between( + date_range[0], date_range[1], inclusive=True)].to_list() + end_d_mask = service_ids.loc[calendar_df['end_date'].between( + date_range[0], date_range[1], inclusive=True)].to_list() + ids_in_range = service_ids.loc[ + service_ids.isin(start_d_mask) & service_ids.isin(end_d_mask)] + subset_cal_df = calendar_df.loc[service_ids.isin(ids_in_range)] + srvc_ids = subset_cal_df['unique_service_id'].to_list() + + if len(srvc_ids) == 0: + date_rngs = _calendar_date_ranges(calendar_df) + print_msg = ( + ' Warning: Date range: {} does not contain any of the ' + 'date ranges available in the calendar. Date ranges available ' + 'are: \n {}'.format(date_range, date_rngs)) + log(print_msg, level=lg.WARNING) + else: + date_rngs = _calendar_date_ranges( + calendar_df.loc[calendar_df['unique_service_id'].isin(srvc_ids)]) + print_msg = ( + ' Date range: {} contains the following date ranges ' + 'available in the calendar. Date ranges available ' + 'are: \n {}'.format(date_range, date_rngs)) + log(print_msg) + + _print_cal_service_ids_len(srvc_ids, table_name='calendar') + return srvc_ids + + +def _calendar_date_ranges(calendar_df, for_print=True): + """ + Helper function to return date ranges inside of the calendar.txt table + using the columns 'start_date' and 'end_date'. Returns date ranges + in string format to be used in print statements and in pd.Timestamp + format to be used in downstream functions. + + Parameters + ---------- + calendar_df : pandas.DataFrame + Calendar DataFrame + for_print : boolean, optional + If True, returns date ranges in string format to be used in print + statements where each element takes format: + '{start timestamp as string} to {start timestamp as string}'. + If False, returns date ranges in pd.Timestamp format as a + list of dicts where keys are the 'start_date' and + 'end_date' for each range: {'start_date': pd.Timestamp, + 'end_date': pd.Timestamp}. + Returns + ------- + date_rngs : list + If for_print is True, returns date ranges in string format to be used + in print statements where each element takes format: + '{start timestamp as string} to {end timestamp as string}'. + If False, returns date ranges in pd.Timestamp format as a + list of dicts where keys are the 'start_date' and + 'end_date' for each range: {'start_date': pd.Timestamp, + 'end_date': pd.Timestamp}. + """ + date_cols = ['start_date', 'end_date'] + unique_range = calendar_df[date_cols].drop_duplicates(subset=date_cols) + # build list of ranges that exist in table to print to user + if for_print: + for col in date_cols: + # convert to string for prints + unique_range[col] = unique_range[col].dt.strftime('%Y-%m-%d') + unique_range[col] = unique_range[col].astype('str') + date_rngs = list(unique_range['start_date'].str.cat( + unique_range['end_date'], sep=' to ').values) + # build dict of ranges that exist in table to use elsewhere + else: + # keep in datetime format + date_rngs = unique_range.to_dict('records') + return date_rngs + + +def _select_calendar_service_ids_by_date( + calendar_df, msg, date=None, verbose=True): + """ + Global search using date to subset records in calendar.txt + using 'start_date' and 'end_date' columns when GTFS feed has + seasonal service_ids, select service IDs where date is within the + 'start_date' and 'end_date' and then select IDs that are active on + the day of the week the date represents + + Parameters + ---------- + calendar_df : pandas.DataFrame + Calendar DataFrame + msg : str + string to prepend to print statement for informative purposes + date : str + date to extract active service IDs in calendar table specified as + a string in YYYY-MM-DD format, e.g. '2013-03-09'. service IDs + that are active on the date specified (where the date is within the + date range given by the start_date and end_date columns) and + that date's day of the week will be extracted. + verbose : boolean, optional + if False, turns off logging and print statements in this function + Returns + ------- + srvc_ids : list + list of active service IDs + """ + # convert date to day of the week + day = dt.strptime(date, '%Y-%m-%d').strftime('%A').lower() + + if verbose: + log(' {} that are active on date: {} ' + '(using a {} schedule)...'.format(msg, date, day)) + + # select date ranges where the 'date' is within the bounds of + # 'start_date' and 'end_date' + service_ids = calendar_df['unique_service_id'] + date_dt = pd.to_datetime(date) + start_d_mask = service_ids.loc[ + calendar_df['start_date'] <= date_dt].to_list() + end_d_mask = service_ids.loc[ + calendar_df['end_date'] >= date_dt].to_list() + ids_in_range = service_ids.loc[ + service_ids.isin(start_d_mask) & service_ids.isin(end_d_mask)] + subset_cal_df = calendar_df.loc[service_ids.isin(ids_in_range)] + srvc_ids_date = subset_cal_df['unique_service_id'].to_list() + + if len(srvc_ids_date) == 0: + date_rngs = _calendar_date_ranges(calendar_df) + if verbose: + print_msg = ( + ' Warning: Date: {} does not fall within any ' + 'date range(s) available in the calendar. ' + 'Available date range(s) found in the calendar ' + 'are: \n {}') + log(print_msg.format(date, date_rngs), level=lg.WARNING) + else: + date_rngs = _calendar_date_ranges( + calendar_df.loc[ + calendar_df['unique_service_id'].isin(srvc_ids_date)]) + if verbose: + print_msg = ( + ' Date: {} is within the available date range(s) ' + 'in the calendar. Available date range(s) found in the ' + 'calendar are: \n {}') + log(print_msg.format(date, date_rngs)) + + # use the date's day of the week to then select service_ids that are + # active on that day + srvc_ids_day = _select_calendar_service_ids_by_day( + calendar_df, msg, day, verbose=False) + # find the intersection of where service_ids are active on the date and + # the day of week to get the final list of IDs + srvc_ids = _intersect_cal_service_ids( + dict_1={'date': srvc_ids_date}, + dict_2={'day': srvc_ids_day}, + verbose=verbose) + if verbose: + _print_cal_service_ids_len(srvc_ids, table_name='calendar') + return srvc_ids + + +def _parse_cal_dates_exception_type(cal_dates_df, verbose=True): + """ + Helper function to parse selected calendar_dates.txt service IDs + from unique_service_id column by exception_type + + Parameters + ---------- + cal_dates_df : pandas.DataFrame + Calendar dates DataFrame + verbose : boolean, optional + if False, turns off logging and print statements in this function + Returns + ------- + srvc_ids_add : list + list of selected service IDs to add + srvc_ids_del : list + list of selected service IDs to remove + """ + # 1 - Service has been added for the specified date. + # 2 - Service has been removed for the specified date. + id_col = 'unique_service_id' + exception_col = 'exception_type' + + srvc_ids_add = cal_dates_df[id_col].loc[ + cal_dates_df[exception_col] == '1'].to_list() + srvc_ids_del = cal_dates_df[id_col].loc[ + cal_dates_df[exception_col] == '2'].to_list() + srvc_ids_add = list(set(srvc_ids_add)) + srvc_ids_del = list(set(srvc_ids_del)) + if verbose: + log(' {:,} service_id(s) were selected from calendar_dates ' + 'with exception_type 1 ' + '(service added for specified date).'.format(len(srvc_ids_add))) + log(' {:,} service_id(s) were selected from calendar_dates ' + 'with exception_type 2 ' + '(service removed for specified date).'.format(len(srvc_ids_del))) + return srvc_ids_add, srvc_ids_del + + +def _select_calendar_dates_service_ids_by_day( + calendar_dates_df, msg, day=None): + """ + Select service IDs in calendar_dates.txt that are active on dates + that match the day of the week specified will be where + exception_type = 1 (service is added for date). + + Parameters + ---------- + calendar_dates_df : pandas.DataFrame + Calendar dates DataFrame + msg : str + string to prepend to print statement for informative purposes + day : {'monday', 'tuesday', 'wednesday', 'thursday', + 'friday', 'saturday', 'sunday'} + day of the week to extract service IDs in calendar_dates table. + Service IDs that are active on dates that match the day of the week + specified will be extracted where exception_type = 1 (service is added + for date). + + Returns + ------- + srvc_ids : list + list of active service IDs + """ + + # select all service ids that are active on this day of week, + # that are inclusive + + # determine day of the week for each date + log(' {} that are active on day: {}...'.format(msg, day)) + # TODO: check if there is an issue with other versions of pandas such that + # requiring pandas > 0.18.1 is needed to run the line below: + calendar_dates_df['_ua_weekday'] = calendar_dates_df[ + 'date'].dt.day_name().str.lower() + subset_df = calendar_dates_df.loc[calendar_dates_df['_ua_weekday'] == day] + srvc_ids = subset_df['unique_service_id'].loc[ + subset_df['exception_type'] == '1'].to_list() + _print_cal_service_ids_len(srvc_ids, table_name='calendar dates') + + return srvc_ids + + +def _select_calendar_dates_service_ids_by_date( + calendar_dates_df, msg, date=None, verbose=True): + """ + Search using date to subset records in calendar_dates.txt + using 'date' column to select service IDs regardless of exception_type + + Parameters + ---------- + calendar_dates_df : pandas.DataFrame + Calendar dates DataFrame + msg : str + string to prepend to print statement for informative purposes + date : str + date to extract service IDs in calendar dates table specified as + a string in YYYY-MM-DD format, e.g. '2013-03-09'. calendar_dates + service IDs that are active within the date range specified will + be selected where exception_type = 1 + (service is added for date) and inactive within + the date range specified will be selected where exception_type = 2 + (service is removed for date). + verbose : boolean, optional + if False, turns off logging and print statements in this function + Returns + ------- + srvc_ids_add : list + list of selected service IDs to add + srvc_ids_del : list + list of selected service IDs to remove + """ + # select all that are on this day, that are inclusive + # remove from those found in calendar that are exclusive + if verbose: + log(' {} that are active on date: {}...'.format(msg, date)) + subset_df = calendar_dates_df.loc[calendar_dates_df['date'] == date] + + srvc_ids_add, srvc_ids_del = _parse_cal_dates_exception_type( + subset_df, verbose) + + if verbose: + if len(srvc_ids_add) == 0: + print_msg = (' Warning: No active service_ids were ' + 'found on date: {}.') + log(print_msg.format(date), level=lg.WARNING) + + return srvc_ids_add, srvc_ids_del + + +def _select_calendar_dates_service_ids_by_date_range( + calendar_dates_df, msg, date_range=None, verbose=True): + """ + Search using date_range to subset records in calendar_dates.txt + using the 'date' column to select service IDs within the date range + regardless of exception_type + + Parameters + ---------- + calendar_dates_df : pandas.DataFrame + Calendar dates DataFrame + msg : str + string to prepend to print statement for informative purposes + date_range : list + date range to extract active service IDs in calendar_dates table + specified as a 2 element list of strings where the first element + is the first date and the second element is the last date in the date + range. Dates should be specified as strings in YYYY-MM-DD format, + e.g. ['2013-03-09', '2013-09-01']. + calendar_dates service IDs that are active within + the date range specified will be selected where exception_type = 1 + (service is added for date) and inactive within + the date range specified will be selected where exception_type = 2 + (service is removed for date). + verbose : boolean, optional + if False, turns off logging and print statements in this function + Returns + ------- + srvc_ids_add : list + list of selected service IDs to add + srvc_ids_del : list + list of selected service IDs to remove + """ + if verbose: + log(' {} that are active within date_range: {}...'.format( + msg, date_range)) + subset_df = calendar_dates_df.loc[ + calendar_dates_df['date'].between( + date_range[0], date_range[1], inclusive=True)] + + srvc_ids_add, srvc_ids_del = _parse_cal_dates_exception_type(subset_df) + + if verbose: + if len(srvc_ids_add) == 0: + earliest_date = calendar_dates_df[ + 'date'].min().strftime('%Y-%m-%d') + latest_date = calendar_dates_df[ + 'date'].max().strftime('%Y-%m-%d') + print_msg = (' Warning: No active service_ids were ' + 'found within date range: {}. ' + 'Dates only exist between: {} and {}.') + log(print_msg.format(date_range, earliest_date, latest_date), + level=lg.WARNING) + + return srvc_ids_add, srvc_ids_del + + +def _add_exception_type_service_id_lists(srvc_ids_list_dict): + """ + Helper function to combine selected calendar_dates.txt service IDs + together by exception_type that have been selected with different + parameters + + Parameters + ---------- + srvc_ids_list_dict : dict + dict with keys 'add' (exception_type = 1) and 'del' + (exception_type = 2) with values as a list of lists of service IDs + for example: + {'add': [['service ID 1, 'service ID 2], ['service ID 3']], + {'del': [['service ID 4, 'service ID 5], ['service ID 6']]} + Returns + ------- + srvc_ids_add : list + list of combined selected service IDs to add + srvc_ids_del : list + list of combined selected service IDs to remove + """ + print_dict = {} + for list_type, srvc_id_lists in srvc_ids_list_dict.items(): + srvc_ids_cnt_1 = len(srvc_id_lists[0]) + srvc_id_lists[0].extend(srvc_id_lists[1]) + srvc_ids_wo_dups = set(srvc_id_lists[0]) + result_srvc_ids = list(srvc_ids_wo_dups) + srvc_ids_cnt_2 = len(srvc_id_lists[0]) + result_cnt = abs(srvc_ids_cnt_1 - srvc_ids_cnt_2) + print_dict.update( + {list_type: [result_cnt, srvc_ids_cnt_1, srvc_ids_cnt_2]}) + + if list_type == 'add': + srvc_ids_add = result_srvc_ids + if list_type == 'del': + srvc_ids_del = result_srvc_ids + + msg_print = ( + ' Added {:,} active and {:,} inactive service_id(s) ' + 'found with calendar_dates_lookup parameter to the existing {:,} ' + 'active and {:,} inactive service_id(s) selected in calendar_' + 'dates to total: {:,} active and {:,} inactive service_id(s).') + log(msg_print.format( + print_dict['add'][0], print_dict['del'][0], + print_dict['add'][1], print_dict['del'][1], + print_dict['add'][2], print_dict['del'][2])) + return srvc_ids_add, srvc_ids_del + + +def _select_calendar_dates_service_ids(calendar_dates_df, params): + """ + Wrapper function to select service IDs from the calendar_dates.txt + that are active according to the parameters specified + + Parameters + ---------- + calendar_dates_df : pandas.DataFrame + Calendar dates DataFrame + params : dict + Parameters denoting how service IDs should be selected from table + and supporting information to guide the selection process + + Returns + ------- + srvc_ids_add : list + list of selected service IDs to add + srvc_ids_del : list + list of selected service IDs to remove + + """ + # collect service IDs that match search parameters in calendar_dates.txt + msg = ' Selecting service_ids from calendar_dates' + + day = params['day'] + date = params['date'] + date_range = params['date_range'] + cal_dates_lookup = params['cal_dates_lookup'] + has_day_param = params['has_day_param'] + has_date_param = params['has_date_param'] + has_date_range_param = params['has_date_range_param'] + has_cal_dates_param = params['has_cal_dates_param'] + has_day_and_date_range_param = params['has_day_and_date_range_param'] + + # convert 'date' to datetime expected format: 'yyyymmdd' e.g.: '20130825' + calendar_dates_df = _cal_date_dt_conversion( + df=calendar_dates_df, date_cols=['date']) + + # No service_ids can be removed when using day of the week since its + # a global selection parameter over the entire calendar + srvc_ids_add = [] + srvc_ids_del = [] + + if has_day_param: + srvc_ids_add = _select_calendar_dates_service_ids_by_day( + calendar_dates_df, msg, day) + if has_day_and_date_range_param: + srvc_ids_add_day = srvc_ids_add.copy() + + if has_date_param: + srvc_ids_add, srvc_ids_del = \ + _select_calendar_dates_service_ids_by_date( + calendar_dates_df, msg, date) + + if has_date_range_param: + srvc_ids_add, srvc_ids_del = \ + _select_calendar_dates_service_ids_by_date_range( + calendar_dates_df, msg, date_range) + if has_day_and_date_range_param: + srvc_ids_add_date_range = srvc_ids_add.copy() + # since we are also using 'day' as a param we cannot remove + # service_ids so set it to empty list + srvc_ids_del = [] + + if has_day_and_date_range_param: + # get the intersection between service IDs found via 'day' and + # 'date_range' params; since 'day' is a global param we + # cannot use any exception_type = 2 records to remove records + print_msg = '{} that are active within date_range: {} on day: {}...' + log(print_msg.format(msg, date_range, day)) + intersect_srvc_ids = _intersect_cal_service_ids( + dict_1={'day': srvc_ids_add_day}, + dict_2={'date range': srvc_ids_add_date_range}) + srvc_ids_add = intersect_srvc_ids.copy() + + if has_cal_dates_param: + query_srvc_ids_add, query_srvc_ids_del = \ + _select_calendar_dates_str_match( + calendar_dates_df, msg, cal_dates_lookup) + # add the service_ids found from query to any existing + # active service_ids or inactive service_ids + merge_dict = {'add': [srvc_ids_add, query_srvc_ids_add], + 'del': [srvc_ids_del, query_srvc_ids_del]} + srvc_ids_add, srvc_ids_del = _add_exception_type_service_id_lists( + merge_dict) + + return srvc_ids_add, srvc_ids_del + + +def _merge_service_ids_cal_dates_w_cal( + srvc_ids, srvc_ids_add, srvc_ids_del, verbose=True): + """ + Helper function to combine selected calendar.txt and calendar_dates.txt + service IDs together where service IDs from calendar dates table with + exception type 1 are added to those found in calendar table and those + with exception type 2 are removed from those found in calendar table + + Parameters + ---------- + srvc_ids : list + list of selected service IDs from calendar table + srvc_ids_add : list + list of selected service IDs to add from calendar dates table + srvc_ids_del : list + list of selected service IDs to remove from calendar dates table + verbose : boolean, optional + if False, turns off logging and print statements in this function + + Returns + ------- + active_srvc_ids : list + list of combined selected service IDs + """ + if verbose: + log(' Reconciling service_id(s) between calendar and ' + 'calendar_dates based on exception_type... ') + srvc_ids_add = list(set(srvc_ids_add)) + srvc_ids_del = list(set(srvc_ids_del)) + srvc_ids_cnt_before_add = len(set(srvc_ids)) + + # add IDs that had exception_type = 1 in calendar_dates + srvc_ids.extend(srvc_ids_add) + active_srvc_ids = list(set(srvc_ids)) + active_srvc_ids_cnt_1 = len(active_srvc_ids) + + diff_cnt = abs(active_srvc_ids_cnt_1 - srvc_ids_cnt_before_add) + # TODO: clean up this section to make sure that the calculation of the + # subtraction works the same as the value difference shown in the print + # statements + if verbose: + msg_print = (' Adding {:,} service_id(s) to those found in ' + 'calendar. Total active service_id(s): {:,}.') + log(msg_print.format(diff_cnt, active_srvc_ids_cnt_1)) + # remove IDs that had exception_type = 2 in calendar_dates + active_srvc_ids = list(set(active_srvc_ids) - set(srvc_ids_del)) + active_srvc_ids_cnt_2 = len(active_srvc_ids) + if verbose: + msg_print = (' Removing {:,} service_id(s) from those found ' + 'in calendar. Total active service_id(s): {:,}.') + log(msg_print.format( + abs(active_srvc_ids_cnt_1 - active_srvc_ids_cnt_2), + active_srvc_ids_cnt_2)) + return active_srvc_ids + + +def _select_calendar_dates_str_match( + calendar_dates_df, msg, cal_dates_lookup): + """ + Selects service IDs from calendar_dates.txt that match the specified + query composed of a dict of column name and string value selection key + value pair criteria. Search will be exact and will select all records + that meet each key value pair criteria. + + Parameters + ---------- + calendar_dates_df : pandas.DataFrame + Calendar dates DataFrame + msg : str + message to prepend to print statement in function for informative + purposes + cal_dates_lookup : dict + dictionary of the lookup column (key) as a string and corresponding + string (value) as string or list of strings to use to subset service + IDs using the calendar_dates DataFrame. Search will be exact and will + select all records that meet each key value pair criteria. + Example: {'schedule_type' : 'WD'} or {'schedule_type' : ['WD', 'SU']} + + Returns + ------- + srvc_ids_add : list + list of selected service IDs to add + srvc_ids_del : list + list of selected service IDs to remove + """ + # collect service IDs that match search parameters in calendar_dates.txt + log('{} that match calendar_date_lookup parameter(s) for ' + 'column(s) and string(s): {}...'.format(msg, cal_dates_lookup)) + srvc_ids = [] + for col_name_key, string_value in cal_dates_lookup.items(): + if not isinstance(string_value, list): + string_value = [string_value] + for text in string_value: + # TODO: support or/and condition + result_df = calendar_dates_df[ + calendar_dates_df[col_name_key].str.match( + text, case=False, na=False)] + result_srvc_ids = result_df['unique_service_id'].to_list() + result_srvc_ids = list(set(result_srvc_ids)) + result_cnt = len(result_srvc_ids) + log(' Found {:,} unique service_id(s) that matched query:' + ' column: {} and string: {}.'.format( + result_cnt, col_name_key, text)) + srvc_ids.extend(result_srvc_ids) + srvc_ids = list(set(srvc_ids)) + + subset_df = calendar_dates_df.loc[ + calendar_dates_df['unique_service_id'].isin(srvc_ids)] + srvc_ids_add, srvc_ids_del = _parse_cal_dates_exception_type(subset_df) + + if len(srvc_ids_add) == 0: + print_msg = (' Warning: No active service_ids were ' + 'found with query: {}.') + log(print_msg.format(cal_dates_lookup), level=lg.WARNING) + + return srvc_ids_add, srvc_ids_del + + +def _print_count_service_ids(df_dict, subset_ids): + """ + Helper function to provide information to user on the results of the + active service ID selection process by printing counts of service IDs + in tables prior and post-processing. + + Parameters + ---------- + df_dict : dict + dict where key is a string denoting the name of the dataframe to use + for prints and the value is a pandas.DataFrame containing the + original DataFrame + subset_ids : list + list of processed active service IDs + Returns + ------- + Nothing + """ + # TODO: flip this around to have: + # {agency: {table: {org_count: n, proc_count: n}}}} + # if agency has records in both tables print differently + log(' In summary, active service IDs were derived from:') + unique_col = 'unique_service_id' + results = {'original_cnts': {}, 'processed_cnts': {}} + for table, df in df_dict.items(): + if not df.empty: + # only count unique IDs + unique_ids = df.drop_duplicates(subset=unique_col) + subset_df = unique_ids.loc[unique_ids[unique_col].isin(subset_ids)] + original_cnts = len(unique_ids) + processed_cnts = len(subset_df) + results['original_cnts'].update({table: original_cnts}) + results['processed_cnts'].update({table: processed_cnts}) + # NOTE: for feeds that use both calendar and + # calendar_dates tables, the sum between that agency's calendar + # count and calendar_dates count will not total to the sum of + # active service IDs given that calendar_dates IDs are used to add + # or remove IDs from the calendar table + start_count = results['original_cnts'][table] + end_count = results['processed_cnts'][table] + msg_print = ( + ' {:,} out of {:,} unique service_id(s) from {}.') + log(msg_print.format(end_count, start_count, table)) + + +def _calendar_service_id_selector( + calendar_df, + calendar_dates_df, + day=None, + date=None, + date_range=None, + cal_dates_lookup=None): + """ + Select trips that correspond to a specific schedule in either calendar.txt + and or calendar_dates.txt by finding service_ids that correspond to the + specified search parameters and the trips related to those service_ids + + Parameters + ---------- + calendar_df : pandas.DataFrame + calendar DataFrame + calendar_dates_df : pandas.DataFrame + calendar_dates DataFrame + day : {'monday', 'tuesday', 'wednesday', 'thursday', + 'friday', 'saturday', 'sunday'}, optional + day of the week to extract active service IDs in calendar and or + calendar_dates tables. If GTFS feeds have only calendar, service IDs + that are active on day specified will be extracted. If GTFS feeds + have only calendar_dates, service IDs that are active on dates + that match the day of the week specified will be extracted where + exception_type = 1 (service is added for date). If GTFS feeds have + both calendar and calendar_dates, calendar service IDs that are + active on day specified will be extracted and calendar_dates + service IDs that are active on dates that match the day of the week + specified will be added to those found in the calendar where + exception_type = 1 (service is added for date). + date : str, optional + date to extract active service IDs in calendar and or + calendar_dates tables specified as a string in YYYY-MM-DD format, + e.g. '2013-03-09'. If GTFS feeds have only calendar, service IDs + that are active on the date specified (where the date is within the + date range given by the start_date and end_date columns) and + that date's day of the week will be extracted. If GTFS feeds have + only calendar_dates, service IDs that are active on the date + specified will be extracted where exception_type = 1 + (service is added for date). If GTFS feeds have both calendar + and calendar_dates, calendar service IDs that are active on the date + specified (where the date is within the date range given by the + start_date and end_date columns) and that date's day of the week + will be extracted and calendar_dates service IDs that are active + on the date specified will be added to those found in the + calendar where exception_type = 1 (service is added for date) + and service IDs that are inactive on the date specified will be + removed from those found in the calendar where exception_type = 2 + (service is removed for date). + date_range : list, optional + date range to extract active service IDs in calendar and or + calendar_dates tables specified as a 2 element list of strings + where the first element is the first date and the second element + is the last date in the date range. Dates should be specified + as strings in YYYY-MM-DD format, e.g. ['2013-03-09', '2013-09-01']. + If GTFS feeds have only calendar, service IDs that are active + within the date range specified (where the date range is within + the date range given by the start_date and end_date columns) + will be extracted. If GTFS feeds have only calendar_dates, + service IDs that are active within the date range specified + will be extracted where exception_type = 1 + (service is added for date). If GTFS feeds have both calendar + and calendar_dates, calendar service IDs that are active within + the date range specified (where the date range is within the date + range given by the state_date and end_date columns) will be + extracted and calendar_dates service IDs that are active within + the date range specified will be added to those found in the + calendar where exception_type = 1 (service is added for date). + This parameter is best utilized if its known that the GTFS + feed calendar table has seasonal service IDs. + cal_dates_lookup : dict, optional + dictionary of the lookup column (key) as a string and corresponding + string (value) as string or list of strings to use to subset trips + using the calendar_dates DataFrame. Parameter should only be used if + there is a high degree of certainly that the GTFS feed in use has + service_ids listed in calendar_dates that would otherwise not be + selected if using any of the other calendar_dates parameters such as: + 'day', 'date' or 'date_range'. Search will be exact and will select + all records that meet each key value pair criteria. + Example: {'schedule_type' : 'WD'} or {'schedule_type' : ['WD', 'SU']} + + Returns + ------- + active_srvc_ids : list + list of strings of active unique service IDs that match the calendar + parameters to use to select active trip IDs + + """ + start_time = time.time() + + log('Processing active service_ids that match the specified ' + 'parameters for trip selection...') + + # check if calendar dfs and related params are empty or not to determine + # what will be used in processing + has_cal = calendar_df.empty is False + has_cal_dates = calendar_dates_df.empty is False + has_day_param = day is not None + has_date_param = date is not None + has_date_range_param = date_range is not None + has_cal_dates_param = cal_dates_lookup is not None + has_day_and_date_range_param = \ + has_day_param and has_date_range_param is True + + params = {'day': day, + 'date': date, + 'date_range': date_range, + 'cal_dates_lookup': cal_dates_lookup, + 'has_cal': has_cal, + 'has_day_param': has_day_param, + 'has_date_param': has_date_param, + 'has_date_range_param': has_date_range_param, + 'has_cal_dates': has_cal_dates, + 'has_cal_dates_param': has_cal_dates_param, + 'has_day_and_date_range_param': has_day_and_date_range_param} + + _trip_schedule_selector_validate_params(calendar_dates_df, params) + + # create unique service IDs for dfs in list if they are not empty + df_list = [] + if has_cal: + df_list.extend([calendar_df]) + if has_cal_dates: + df_list.extend([calendar_dates_df]) + df_list = _add_unique_service_id(df_list) + + if has_cal: + srvc_ids = _select_calendar_service_ids( + calendar_df, params) + + if has_cal_dates: + srvc_ids_add, srvc_ids_del = _select_calendar_dates_service_ids( + calendar_dates_df, params) + + if has_cal and has_cal_dates: + active_srvc_ids = _merge_service_ids_cal_dates_w_cal( + srvc_ids, srvc_ids_add, srvc_ids_del) + elif has_cal: + active_srvc_ids = srvc_ids.copy() + elif has_cal_dates: + active_srvc_ids = srvc_ids_add.copy() + active_srvc_ids_cnt = len(active_srvc_ids) + + if active_srvc_ids_cnt == 0: + msg = ( + 'Warning: No active service_ids were found matching the specified ' + 'parameters. No trips can be selected. Suggest modifying the ' + 'calendar parameters and or reviewing the GTFS calendar and or ' + 'calendar_dates tables.') + log(msg, level=lg.WARNING) + else: + _print_count_service_ids(df_dict={'calendar': calendar_df, + 'calendar_dates': calendar_dates_df}, + subset_ids=active_srvc_ids) + + msg_print = ( + '{:,} active service_id(s) were found that match the specified ' + 'parameters for trip selection. Took {:,.2f} seconds.') + log(msg_print.format(active_srvc_ids_cnt, time.time() - start_time)) + + return active_srvc_ids + + +def _trip_selector(trips_df, service_ids, verbose=True): + """ + Select trips that correspond to a specific schedule in either calendar.txt + and or calendar_dates.txt using the specified active service_ids + + Parameters + ---------- + trips_df : pandas.DataFrame + trips DataFrame + service_ids : list + list of strings that represent active service IDs that will be used + to select trips. Trips that correspond to these service IDs will + be exported and considered active during the specified schedule. + verbose : boolean, optional + if False, turns off logging and print statements in this function + + Returns + ------- + subset_trip_df : pandas.DataFrame + trip DataFrame comprised of trips that are active during the + specified schedule represented in the passed list of active service IDs + + """ + start_time = time.time() + if verbose: + log('--------------------------------') + log('Selecting trip_id(s) with active service_ids that match ' + 'the specified calendar and or calendar date parameters...') + df_list = _add_unique_service_id([trips_df]) + subset_trip_df = trips_df.loc[ + trips_df['unique_service_id'].isin(service_ids)] + + sort_columns = ['route_id', 'trip_id', 'direction_id'] + if 'direction_id' not in subset_trip_df.columns: + sort_columns.remove('direction_id') + subset_trip_df.sort_values(by=sort_columns, inplace=True) + subset_trip_df.reset_index(drop=True, inplace=True) + subset_trip_df.drop('unique_service_id', axis=1, inplace=True) + + tot_trip_cnt = len(trips_df) + tot_subset_trip_cnt = len(subset_trip_df) + tot_trip_pct = (tot_subset_trip_cnt / tot_trip_cnt) * 100 + + if tot_subset_trip_cnt == 0: + raise ValueError( + 'No trips were found that matched the active service_ids ' + 'identified by the specified calendar parameters. Suggest ' + 'modifying the calendar parameters and or reviewing the GTFS ' + 'trips and calendar and or calendar_dates tables.') + if verbose: + agency_id_col = 'unique_agency_id' + agency_ids = trips_df[agency_id_col].unique() + for agency in agency_ids: + agency_trip_cnt = len(subset_trip_df.loc[ + subset_trip_df[agency_id_col] == agency]) + agency_tot_trip_cnt = len(trips_df.loc[ + trips_df[agency_id_col] == agency]) + agency_trip_pct = (agency_trip_cnt / agency_tot_trip_cnt) * 100 + + if agency_trip_cnt > 0: + msg = ('{:,} of {:,} ({:.2f} percent) of trips for agency: {} ' + 'were selected using service_ids that matched the ' + 'specified calendar parameters.') + log(msg.format(agency_trip_cnt, agency_tot_trip_cnt, + agency_trip_pct, agency)) + else: + msg = ('Warning: For agency: {}, no trips were found that ' + 'matched the active service_ids identified by the ' + 'specified calendar parameters. Suggest modifying the ' + 'calendar parameters and or reviewing the GTFS ' + 'trips and calendar and or calendar_dates tables.') + log(msg.format(agency), level=lg.WARNING) + + if len(agency_ids) != 1: + msg = ('In total: {:,} of {:,} ({:.2f} percent) of trips were ' + 'selected using service_ids that matched the specified ' + 'calendar parameters.') + log(msg.format(tot_subset_trip_cnt, tot_trip_cnt, tot_trip_pct)) + log('Took {:,.2f} seconds.'.format(time.time() - start_time)) + log('--------------------------------') + return subset_trip_df + + +def _highest_freq_trips_date(trips_df, calendar_df, calendar_dates_df): + """ + Counts the number of trips active on each calendar day of active service + and returns the date with the maximum number of trips. + + Parameters + ---------- + trips_df : pandas.DataFrame + trips DataFrame + calendar_df : pandas.DataFrame + calendar DataFrame + calendar_dates_df : pandas.DataFrame + calendar dates DataFrame + + Returns + ------- + max_date : str + date as a string where the maximum number of trips occur in a schedule + + """ + start_time = time.time() + # get service ids for each date + # count the trips that have those service ids + # the date that has the max is the date to choose + # use to select service ids by date with the date found here + log('Finding date with the most frequent trips...') + has_cal = calendar_df.empty is False + has_cal_dates = calendar_dates_df.empty is False + + df_list = [trips_df] + if has_cal: + df_list.extend([calendar_df]) + if has_cal_dates: + df_list.extend([calendar_dates_df]) + df_list = _add_unique_service_id(df_list) + + if has_cal: + # convert cols to datetime expected format: 'yyyymmdd' e.g.: '20130825' + date_cols = ['start_date', 'end_date'] + calendar_df = _cal_date_dt_conversion( + df=calendar_df, date_cols=date_cols) + date_ranges = _calendar_date_ranges(calendar_df, for_print=False) + date_list = [] + for date_range in date_ranges: + dates = pd.date_range( + date_range['start_date'], + date_range['end_date'], closed=None, freq='d') + dates_dt = dates.to_pydatetime() + dates_str = [date.strftime('%Y-%m-%d') for date in dates_dt] + date_list.extend(dates_str) + unique_date_list_cal = list(set(date_list)) + + date_srv_id_dict = {} + for date in unique_date_list_cal: + srvc_ids_date = _select_calendar_service_ids_by_date( + calendar_df, msg='', date=date, verbose=False) + date_srv_id_dict.update({date: srvc_ids_date}) + + if has_cal_dates: + calendar_dates_df = _cal_date_dt_conversion( + df=calendar_dates_df, date_cols=['date']) + unique_date_list_cal_dates = list( + calendar_dates_df['date'].dt.strftime('%Y-%m-%d').unique()) + + date_add_rmv_srv_id_dict = {} + for date in unique_date_list_cal_dates: + srvc_ids_add, srvc_ids_del = \ + _select_calendar_dates_service_ids_by_date( + calendar_dates_df, msg='', date=date, verbose=False) + date_add_rmv_srv_id_dict.update({date: { + 'add': srvc_ids_add, 'remove': srvc_ids_del}}) + + if has_cal and has_cal_dates: + # combine both list of dates into one list of unique dates + unique_date_list_cal.extend(unique_date_list_cal_dates) + unique_date_list = set(unique_date_list_cal) + + for date in unique_date_list: + if date in date_srv_id_dict.keys(): + srvc_ids = date_srv_id_dict[date].copy() + else: + srvc_ids = [] + if date in date_add_rmv_srv_id_dict.keys(): + srvc_ids_add = date_add_rmv_srv_id_dict[date]['add'] + srvc_ids_del = date_add_rmv_srv_id_dict[date]['remove'] + else: + srvc_ids_add = [] + srvc_ids_del = [] + active_srvc_ids = _merge_service_ids_cal_dates_w_cal( + srvc_ids, srvc_ids_add, srvc_ids_del, verbose=False) + date_srv_id_dict[date] = active_srvc_ids.copy() + elif has_cal and not has_cal_dates: + unique_date_list = unique_date_list_cal.copy() + elif has_cal_dates and not has_cal: + unique_date_list = unique_date_list_cal_dates.copy() + date_srv_id_dict = {} + for date in unique_date_list: + # only consider service IDs to add (exception type == 1) + date_srv_id_dict.update( + {date: date_add_rmv_srv_id_dict[date]['add']}) + + # select the trips and count + trips_df = _add_unique_trip_id(trips_df) + date_trip_cnt = {} + for date in unique_date_list: + subset_trips_df = _trip_selector( + trips_df, service_ids=date_srv_id_dict[date], verbose=False) + trip_id_cnt = len(subset_trips_df['unique_trip_id'].to_list()) + date_trip_cnt.update({date: trip_id_cnt}) + + # find the date that has the max number of trips + # sort so the max is always the same between runs of the same data + date_trip_cnt_sorted = dict(sorted(date_trip_cnt.items())) + max_date = max(date_trip_cnt_sorted, key=lambda k: date_trip_cnt_sorted[k]) + # get the max number of trips number + max_trips = max(date_trip_cnt_sorted.values()) + # check to see if there are more than 1 dates that have the same max number + # of trips value + dates_w_max_trips = [k for k, v in + date_trip_cnt_sorted.items() if v == max_trips] + # get the service_ids for dates that share the max number of trips value + reduce_list = [date_srv_id_dict[x] for x in dates_w_max_trips] + # check to see if any of the dates have different service_ids from + # one another and make a list of the IDs that are not shared between dates + service_ids_not_shared = list(set.difference(*map(set, reduce_list))) + + time_msg = 'Took {:,.2f} seconds.'.format(time.time() - start_time) + if len(dates_w_max_trips) == 1: + msg = ("The date with the most frequent trips ({:,} active trips): {} " + "will be used to select service ids. {}".format( + max_trips, max_date, time_msg)) + log(msg) + # if there are more than 1 dates with the same max number of trips value, + # print informative messages and ValueError depending on if service_ids + # differ between the dates + else: + if len(service_ids_not_shared) == 0: + msg = ("The following dates: {} have identical service_ids " + "representing the date with the most frequent trips " + "({:,} active trips). The service ids active on these dates" + " will be used to select active trips. Selection of " + "service ids will be made using date: {}. {}") + log(msg.format( + sorted(dates_w_max_trips), max_trips, max_date, time_msg)) + # if there are different service_ids between the dates that have the + # same max number of trips value, throw error to make user explicitly + # pick one of the dates in the list to use since we cannot arbitrary + # pick one for them + else: + msg = ("The following dates: {} have different service ids " + "representing the date with the most frequent trips " + "({:,} active trips). Unable to use the " + "'use_highest_freq_trips_date' parameter. " + "Pick one date from this list as the 'date' parameter " + "to proceed.") + raise ValueError(msg.format(sorted(dates_w_max_trips), max_trips)) + + # return the date of the max trip value instead of the date's service IDs + # in order to utilize the informative debugging prints in the + # _calendar_service_id_selector() function to be transparent in how IDs + # were selected and to help catch potential data issues + return max_date + + +def _add_unique_service_id(df_list): + """ + Create 'unique_service_id' column and values for a list of + pandas.DataFrames + + Parameters + ---------- + df_list : list + list of pandas.DataFrames to generate 'unique_service_id' column for + + Returns + ------- + df_list : list + list of pandas.DataFrames with 'unique_service_id' column added + """ + for index, df in enumerate(df_list): + df['unique_service_id'] = df['service_id'].str.cat( + df['unique_agency_id'].astype('str'), sep='_') + df_list[index] = df + return df_list diff --git a/urbanaccess/gtfs/utils_format.py b/urbanaccess/gtfs/utils_format.py index a0e8b4f..8d5af44 100644 --- a/urbanaccess/gtfs/utils_format.py +++ b/urbanaccess/gtfs/utils_format.py @@ -65,24 +65,36 @@ def _read_gtfs_file(textfile_path, textfile): '{} is missing required column(s): {}.'.format( textfile, missing_req_cols)) - # if optional dtype col exists include it in req_dtypes dict - if opt_dtypes is not None: - for col_name, dtype in opt_dtypes.items(): - if col_name in col_list: - req_dtypes.update({col_name: dtype}) + # build dtypes dict + if req_dtypes is not None or opt_dtypes is not None: + dtype_dict = {} + dtype_dict.update(req_dtypes) + # if optional dtype col exists include it in dtypes dict + if opt_dtypes is not None: + for col_name, dtype in opt_dtypes.items(): + if col_name in col_list: + dtype_dict.update({col_name: dtype}) + else: + dtype_dict = None - df = pd.read_csv(file_path, dtype=req_dtypes, low_memory=False) + df = pd.read_csv(file_path, dtype=dtype_dict, low_memory=False) # print warning or raise error when table is empty depending on the table if df.empty: cal_file_warnings = {'calendar': 'calendar_dates.txt', 'calendar_dates': 'calendar.txt'} + shape_file_warnings = {'shapes': 'shapes.txt'} if file_name in cal_file_warnings.keys(): warning_msg = ( ' {} has no records. This could indicate that this feed ' 'is using {} instead of {}.txt for service_ids.') log(warning_msg.format(textfile, cal_file_warnings[file_name], file_name), level=lg.WARNING) + elif file_name in shape_file_warnings.keys(): + warning_msg = ( + ' {} has no records. This feed does not use {}.txt.') + log(warning_msg.format(textfile, shape_file_warnings[file_name], + file_name), level=lg.WARNING) else: raise ValueError('{} has no records. ' 'This file cannot be empty.'.format(textfile)) @@ -92,9 +104,19 @@ def _read_gtfs_file(textfile_path, textfile): df = _remove_whitespace( df=df, textfile=textfile, col_list=remove_whitespace) + # convert dtypes here in case col name was incorrect due to + # whitespace when read in pd.read_csv() + if req_dtypes is not None: + for col_name, dtype in req_dtypes.items(): + if df[col_name].dtype != dtype: + if dtype == object: + dtype = str # force to string instead of object + df[col_name] = df[col_name].astype(dtype) + if numeric_converter is not None: for col in numeric_converter: - df[col] = pd.to_numeric(df[col]) + if col in df.columns: + df[col] = pd.to_numeric(df[col]) record_cnt = len(df) msg = (' Successfully read: {} with {:,} record(s). ' @@ -140,7 +162,6 @@ def _calendar_dates_agencyid(calendar_dates_df, routes_df, 'service_id', 'unique_agency_id']).size() group_counts = group.reset_index(level=1) # check if service_ids are associated with more than one agency - if any(group_counts.index.value_counts().values > 1): feed_name = os.path.split(feed_folder)[1] log('GTFS feed: {!s}, calendar_dates uses the same service_id across ' @@ -384,10 +405,65 @@ def _stop_times_agencyid(stop_times_df, routes_df, trips_df, return merged_df -def _add_unique_agencyid(agency_df, stops_df, routes_df, - trips_df, stop_times_df, calendar_df, - calendar_dates_df, feed_folder, - nulls_as_folder=True): +def _shapes_agencyid(shapes_df, trips_df, routes_df, agency_df, feed_folder): + """ + Assign unique agency ID to shapes DataFrame + + Parameters + ---------- + shapes_df : pandas:DataFrame + shapes DataFrame + trips_df : pandas:DataFrame + trips DataFrame + routes_df : pandas:DataFrame + routes DataFrame + agency_df : pandas:DataFrame + agency DataFrame + feed_folder : str + name of GTFS feed folder + + Returns + ------- + merged_df : pandas.DataFrame + """ + tmp1 = pd.merge(routes_df, agency_df, how='left', on='agency_id', + sort=False, copy=False) + merged_df = pd.merge(trips_df[['shape_id', 'trip_id', 'route_id']], + tmp1, how='left', + on='route_id', sort=False, copy=False) + merged_df['unique_agency_id'] = _generate_unique_agency_id( + merged_df, 'agency_name') + + group = merged_df[['shape_id', 'unique_agency_id']].groupby([ + 'shape_id', 'unique_agency_id']).size() + group_counts = group.reset_index(level=1) + # check if shape_ids are associated with more than one agency + if any(group_counts.index.value_counts().values > 1): + feed_name = os.path.split(feed_folder)[1] + log('GTFS feed: {!s}, shapes uses the same shape_id across ' + 'multiple agency_ids. This feed shapes table will be ' + 'modified from its original format to provide shape_ids for ' + 'each agency using a one to many join.'.format(feed_name)) + + tmp = merged_df[['shape_id', 'unique_agency_id']].drop_duplicates( + ['shape_id', 'unique_agency_id'], inplace=False) + merged_df = tmp.merge(shapes_df, 'left', on='shape_id') + + else: + merged_df.drop_duplicates( + subset='shape_id', keep='first', inplace=True) + + merged_df = pd.merge(shapes_df, + merged_df[['unique_agency_id', 'shape_id']], + how='left', on='shape_id', + sort=False, copy=False) + return merged_df + + +def _add_unique_agency_id(agency_df, stops_df, routes_df, + trips_df, stop_times_df, shapes_df, calendar_df, + calendar_dates_df, feed_folder, + nulls_as_folder=True): """ Create an unique agency ID for all GTFS feed DataFrames to enable unique relational table keys. Pathways to create the unique agency ID are: @@ -404,7 +480,7 @@ def _add_unique_agencyid(agency_df, stops_df, routes_df, tables; 4) If GTFS feed has an agency.txt file and it has more than one agency (it must have an 'agency_id' and 'agency_name' column and values), however if there is also a mismatch between the 'agency_id' - in aganecy.txt and routes.txt, then assume records tied to the mismatched + in agency.txt and routes.txt, then assume records tied to the mismatched 'agency_id'(s) are from multiple agencies and label the unique agency ID as such by concatenating 'multiple_operators_' and the GTFS feed directory folder name. @@ -421,6 +497,8 @@ def _add_unique_agencyid(agency_df, stops_df, routes_df, trips DataFrame stop_times_df : pandas:DataFrame stop times DataFrame + shapes_df : pandas:DataFrame + shapes DataFrame calendar_df : pandas:DataFrame calendar DataFrame calendar_dates_df : pandas:DataFrame @@ -432,7 +510,7 @@ def _add_unique_agencyid(agency_df, stops_df, routes_df, name will be used as the unique agency ID Returns ------- - stops_df, routes_df, trips_df, stop_times_df, calendar_df, + stops_df, routes_df, trips_df, stop_times_df, shapes_df, calendar_df, calendar_dates_df : pandas.DataFrame Returns all input GTFS DataFrames with a unique agency ID column and value for all tables and records. @@ -452,7 +530,8 @@ def _add_unique_agencyid(agency_df, stops_df, routes_df, 'stop_times': stop_times_df} optional_df_dict = {'calendar': calendar_df, - 'calendar_dates': calendar_dates_df} + 'calendar_dates': calendar_dates_df, + 'shapes': shapes_df} # if optional calendar or calendar_dates_df are not empty then add it # to the processing list for name, df in optional_df_dict.items(): @@ -495,11 +574,13 @@ def _add_unique_agencyid(agency_df, stops_df, routes_df, # replace nans with '' agency_df[cols] = agency_df[cols].fillna('') # determine extent of missing values - all_missing = (agency_df['agency_id'].str.isspace().all() and - agency_df['agency_name'].str.isspace().all()) or \ + agency_id_str = agency_df['agency_id'].astype(str) + agency_name_str = agency_df['agency_name'].astype(str) + all_missing = (agency_id_str.str.isspace().all() and + agency_name_str.str.isspace().all()) or \ (agency_df[cols].values == '').all() - any_missing = (agency_df['agency_id'].str.isspace().any() and - agency_df['agency_name'].str.isspace().any()) or \ + any_missing = (agency_id_str.str.isspace().any() and + agency_name_str.str.isspace().any()) or \ (agency_df[cols].values == '').any() # if 'agency_name' and 'agency_id' both have no records or both contain @@ -550,6 +631,16 @@ def _add_unique_agencyid(agency_df, stops_df, routes_df, # below, apply the unique agency ID to all GTFS dfs via # merge operations + # if optional shapes_df is not empty then process it + if shapes_df.empty is False: + subset_trips_df_w_shpid = trips_df[ + ['trip_id', 'route_id', 'shape_id']] + shapes_replacement_df = _shapes_agencyid( + shapes_df=shapes_df, + routes_df=subset_routes_df, + trips_df=subset_trips_df_w_shpid, + agency_df=subset_agency_df, + feed_folder=feed_folder) # if optional calendar_dates_df is not empty then process it if calendar_dates_df.empty is False: @@ -598,8 +689,11 @@ def _add_unique_agencyid(agency_df, stops_df, routes_df, 'trips': trips_replacement_df, 'stop_times': stop_times_replacement_df} - # if optional calendar or calendar_dates_df are not empty then - # add it to the processing list + # if optional shapes, calendar, or calendar_dates_df are not + # empty then add it to the processing list + if shapes_df.empty is False: + df_dict.update( + {'shapes': shapes_replacement_df}) if calendar_df.empty is False: df_dict.update( {'calendar': calendar_replacement_df}) @@ -631,18 +725,21 @@ def _add_unique_agencyid(agency_df, stops_df, routes_df, feed_folder_name)) df_dict[name] = df + # TODO: this dict may be redundant check if can refactor optional_df_dict = { 'calendar': calendar_df, - 'calendar_dates': calendar_dates_df} + 'calendar_dates': calendar_dates_df, + 'shapes': shapes_df} # if optional calendar or calendar_dates_df are empty then return - # the original empty df + # the original empty df with an empty agency ID column for name, df in optional_df_dict.items(): if df.empty: + df['unique_agency_id'] = np.nan df_dict.update({name: df}) # ensure returned items in list are always in the same expected order df_list = [df_dict['stops'], df_dict['routes'], df_dict['trips'], - df_dict['stop_times'], df_dict['calendar'], + df_dict['stop_times'], df_dict['shapes'], df_dict['calendar'], df_dict['calendar_dates']] log('Unique agency ID operation complete. ' @@ -651,8 +748,8 @@ def _add_unique_agencyid(agency_df, stops_df, routes_df, def _add_unique_gtfsfeed_id(stops_df, routes_df, trips_df, - stop_times_df, calendar_df, calendar_dates_df, - feed_folder, feed_number): + stop_times_df, shapes_df, calendar_df, + calendar_dates_df, feed_folder, feed_number): """ Create an unique GTFS feed specific ID for all GTFS feed DataFrames to enable tracking of specific feeds @@ -667,6 +764,8 @@ def _add_unique_gtfsfeed_id(stops_df, routes_df, trips_df, trips DataFrame stop_times_df : pandas:DataFrame stop times DataFrame + shapes_df : pandas:DataFrame + shapes DataFrame calendar_df : pandas:DataFrame calendar DataFrame calendar_dates_df : pandas:DataFrame @@ -677,27 +776,44 @@ def _add_unique_gtfsfeed_id(stops_df, routes_df, trips_df, current number iteration of GTFS feed being read in root directory Returns ------- - stops_df, routes_df, trips_df, stop_times_df, calendar_df, + stops_df, routes_df, trips_df, stop_times_df, shapes_df, calendar_df, calendar_dates_df : pandas.DataFrame """ start_time = time.time() - df_list = [stops_df, routes_df, trips_df, stop_times_df, calendar_df] - # if calendar_dates_df is not empty then add it to the processing list - if calendar_dates_df.empty is False: - df_list.extend([calendar_dates_df]) + df_dict = {'stops': stops_df, + 'routes': routes_df, + 'trips': trips_df, + 'stop_times': stop_times_df} + + optional_df_dict = {'calendar': calendar_df, + 'calendar_dates': calendar_dates_df, + 'shapes': shapes_df} + # if optional calendar or calendar_dates_df are not empty then add it + # to the processing list + for name, df in optional_df_dict.items(): + if df.empty is False: + df_dict.update({name: df}) # standardize feed_folder name feed_folder = _generate_unique_feed_id(feed_folder) - for index, df in enumerate(df_list): + for name, df in df_dict.items(): # create new unique_feed_id column based on the name of the feed folder df['unique_feed_id'] = '_'.join([feed_folder, str(feed_number)]) - df_list[index] = df + df_dict[name] = df + + # if optional calendar or calendar_dates_df are empty then return + # the original empty df with an empty agency ID column + for name, df in optional_df_dict.items(): + if df.empty: + df['unique_feed_id'] = np.nan + df_dict.update({name: df}) - # if calendar_dates_df is empty then return the original empty df - if calendar_dates_df.empty: - df_list.extend([calendar_dates_df]) + # ensure returned items in list are always in the same expected order + df_list = [df_dict['stops'], df_dict['routes'], df_dict['trips'], + df_dict['stop_times'], df_dict['shapes'], df_dict['calendar'], + df_dict['calendar_dates']] log('Unique GTFS feed ID operation complete. ' 'Took {:,.2f} seconds.'.format(time.time() - start_time)) @@ -768,7 +884,7 @@ def _timetoseconds(df, time_cols): series_df = col_series.to_frame(name=''.join([col, '_sec'])) # check if times are negative if so display warning - if series_df.values.any() < 0: + if (series_df < 0).any()[0]: log('Warning: Some stop times in {} column are negative. ' 'Time should be positive. Suggest checking original ' 'GTFS feed stop_time file before proceeding.'.format(col), @@ -830,8 +946,7 @@ def _apply_gtfs_definition(df, desc_dict): return df -def _add_txt_definitions(stops_df, routes_df, stop_times_df, - trips_df): +def _add_txt_definitions(stops_df, routes_df, stop_times_df, trips_df): """ Append GTFS definitions to stops, routes, stop times, and trips DataFrames @@ -1009,8 +1124,11 @@ def _remove_whitespace(df, textfile, col_list=None): # remove leading and trailing spaces in values for columns in list if col_list: + # rebuild col_list so that it doesnt fail on expected columns that were + # not present in the file + present_col_list = [col for col in col_list if col in after_cols] df_copy = df.copy() - for col in col_list: + for col in present_col_list: before_count = df_copy[col].str.len().sum() df_copy[col] = df_copy[col].str.rstrip().str.lstrip() after_count = df_copy[col].str.len().sum() @@ -1037,5 +1155,9 @@ def _list_raw_txt_columns(file): df : list list of columns in txt file """ - df = pd.read_csv(file) + try: + df = pd.read_csv(file) + except Exception as e: + msg = 'Unable to process: {}. Exception: {}' + raise ValueError(msg.format(file, e)) return list(df.columns) diff --git a/urbanaccess/gtfs/utils_validation.py b/urbanaccess/gtfs/utils_validation.py index e9c7dff..897113c 100644 --- a/urbanaccess/gtfs/utils_validation.py +++ b/urbanaccess/gtfs/utils_validation.py @@ -186,7 +186,8 @@ def _validate_gtfs(stops_df, feed_folder, def _check_time_range_format(timerange): """ - Check time range value format for expected schema + Check time range value format for expected schema and raise value error + if format is invalid Parameters ---------- @@ -197,8 +198,10 @@ def _check_time_range_format(timerange): Returns ------- - None + Nothing """ + if timerange is None: + raise ValueError('timerange cannot be None.') time_error_statement = ( '{} starttime and endtime are not in the correct format. ' 'Format should be a 24 hour clock in the following format: 08:00:00 ' diff --git a/urbanaccess/gtfsfeeds.py b/urbanaccess/gtfsfeeds.py index 2ab110d..96b255c 100644 --- a/urbanaccess/gtfsfeeds.py +++ b/urbanaccess/gtfsfeeds.py @@ -1,14 +1,16 @@ -import yaml import pandas as pd import traceback import zipfile import os import logging as lg import time +import ssl from six.moves.urllib import request +import shutil from urbanaccess.utils import log from urbanaccess import config +from urbanaccess.utils import _dict_to_yaml, _yaml_to_dict # TODO: make class CamelCase @@ -22,21 +24,21 @@ class urbanaccess_gtfsfeeds(object): ---------- gtfs_feeds : dict dictionary of the name of the transit service or agency GTFS feed - as the key -note: this name will be used as the feed folder name. + as the key (Note: this name will be used as the feed folder name. If the GTFS feed does not have a agency name in the agency.txt file - this key will be used to name the agency- and + this key will be used to name the agency) and the GTFS feed URL as the value to pass to the GTFS downloader as: {unique name of GTFS feed or transit service/agency : URL of feed} """ - def __init__(self, - gtfs_feeds={}): + def __init__(self, gtfs_feeds={}): self.gtfs_feeds = gtfs_feeds @classmethod - def from_yaml(cls, gtfsfeeddir=os.path.join(config.settings.data_folder, - 'gtfsfeeds'), + def from_yaml(cls, + gtfsfeeddir=os.path.join( + config.settings.data_folder, 'gtfsfeeds'), yamlname='gtfsfeeds.yaml'): """ Create an urbanaccess_gtfsfeeds instance from a saved YAML. @@ -47,56 +49,46 @@ def from_yaml(cls, gtfsfeeddir=os.path.join(config.settings.data_folder, Directory to load a YAML file. yamlname : str or file like, optional File name from which to load a YAML file. + Returns ------- - urbanaccess_gtfsfeeds + gtfsfeeds : object """ - - if not isinstance(gtfsfeeddir, str): - raise ValueError('gtfsfeeddir must be a string') - if not os.path.exists(gtfsfeeddir): - raise ValueError('{} does not exist or was not found'.format( - gtfsfeeddir)) - if not isinstance(yamlname, str): - raise ValueError('yaml must be a string') - - yaml_file = os.path.join(gtfsfeeddir, yamlname) - - with open(yaml_file, 'r') as f: - yaml_config = yaml.safe_load(f) - - if not isinstance(yaml_config, dict): - raise ValueError('{} yamlname is not a dict'.format(yamlname)) + yaml_config = _yaml_to_dict(yaml_dir=gtfsfeeddir, yaml_name=yamlname) validkey = 'gtfs_feeds' if validkey not in yaml_config.keys(): raise ValueError('key gtfs_feeds was not found in YAML file') + dtype_raise_error_msg = '{} must be a string.' for key in yaml_config['gtfs_feeds'].keys(): if not isinstance(key, str): - raise ValueError('{} must be a string'.format(key)) - for value in yaml_config['gtfs_feeds'][key]: - if not isinstance(value, str): - raise ValueError('{} must be a string'.format(value)) + raise ValueError(dtype_raise_error_msg.format(key)) + value = yaml_config['gtfs_feeds'][key] + if not isinstance(value, str): + raise ValueError(dtype_raise_error_msg.format(value)) unique_url_count = len( - pd.DataFrame.from_dict(yaml_config['gtfs_feeds'], orient='index')[ - 0].unique()) + pd.DataFrame.from_dict( + yaml_config['gtfs_feeds'], orient='index')[0].unique()) url_count = len(yaml_config['gtfs_feeds']) if unique_url_count != url_count: raise ValueError( - 'duplicate values were found when the passed add_dict ' - 'dictionary was added to the existing dictionary. Feed URL ' - 'values must be unique.') + 'duplicate values were found in YAML file: {}. Feed URL ' + 'values must be unique.'.format(yamlname)) gtfsfeeds = cls(gtfs_feeds=yaml_config.get('gtfs_feeds', {})) - log('{} YAML successfully loaded with {} feeds.'.format(yaml_file, len( - yaml_config['gtfs_feeds']))) + log('{} YAML successfully loaded with {:,} feeds.'.format( + yamlname, len(yaml_config['gtfs_feeds']))) return gtfsfeeds def to_dict(self): """ Return a dict representation of an urbanaccess_gtfsfeeds instance. + + Returns + ------- + {'gtfs_feeds': urbanaccess_gtfsfeeds.gtfs_feeds} : dict """ return {'gtfs_feeds': self.gtfs_feeds} @@ -113,35 +105,41 @@ def add_feed(self, add_dict, replace=False): as: {unique name of GTFS feed or transit service/agency : URL of feed} replace : bool, optional - If key of dict is already in the UrbanAccess replace - the existing dict value with the value passed + If key of dict is already in the urbanaccess_gtfsfeeds object, + replace the existing dict value with the value passed + + Returns + ------- + urbanaccess_gtfsfeeds : object """ if not isinstance(add_dict, dict): raise ValueError('add_dict is not a dict') if not isinstance(replace, bool): raise ValueError('replace is not bool') + dtype_raise_error_msg = '{} must be a string' - if replace is not True: - + # TODO: refactor to remove the need to have repeat code if + # replace True/False + if replace is False: for key in add_dict.keys(): if key in self.gtfs_feeds.keys(): - raise ValueError( - '{} passed in add_dict already exists in gtfs_feeds. ' - 'Only unique keys are allowed to be added.'.format( - key)) + msg = ('{} passed in add_dict already exists in ' + 'gtfs_feeds. Only unique keys are allowed to be ' + 'added.') + raise ValueError(msg.format(key)) if not isinstance(key, str): - raise ValueError('{} must be a string'.format(key)) - for value in add_dict[key]: - if not isinstance(value, str): - raise ValueError('{} must be a string'.format(value)) + raise ValueError(dtype_raise_error_msg.format(key)) + value = add_dict[key] + if not isinstance(value, str): + raise ValueError(dtype_raise_error_msg.format(value)) for key, value in add_dict.items(): if value in self.gtfs_feeds.values(): - raise ValueError('duplicate values were found when the ' - 'passed add_dict dictionary was added to ' - 'the existing dictionary. Feed URL ' - 'values must be unique.') + msg = ('duplicate values were found when the passed ' + 'add_dict dictionary was added to the existing ' + 'dictionary. Feed URL values must be unique.') + raise ValueError(msg) gtfs_feeds = self.gtfs_feeds.update(add_dict) else: @@ -150,14 +148,14 @@ def add_feed(self, add_dict, replace=False): log('{} passed in add_dict will replace existing {} feed ' 'in gtfs_feeds.'.format(key, key)) if not isinstance(key, str): - raise ValueError('{} must be a string'.format(key)) - for value in add_dict[key]: - if not isinstance(value, str): - raise ValueError('{} must be a string'.format(value)) - + raise ValueError(dtype_raise_error_msg.format(key)) + value = add_dict[key] + if not isinstance(value, str): + raise ValueError(dtype_raise_error_msg.format(value)) gtfs_feeds = self.gtfs_feeds.update(add_dict) - log('Added {} feeds to gtfs_feeds: {}'.format(len(add_dict), add_dict)) + log('Added {:,} feeds to gtfs_feeds: {}'.format( + len(add_dict), add_dict)) return gtfs_feeds @@ -173,17 +171,19 @@ def remove_feed(self, del_key=None, remove_all=False): remove_all : bool, optional if true, remove all keys from existing urbanaccess_gtfsfeeds instance - """ + Returns + ------- + Nothing + + """ if not isinstance(remove_all, bool): raise ValueError('remove_all is not bool') if del_key is None and remove_all: self.gtfs_feeds = {} log('Removed all feeds from gtfs_feeds') - else: - if not isinstance(del_key, (list, str)): raise ValueError('del_key must be a string or list of strings') if remove_all: @@ -195,16 +195,14 @@ def remove_feed(self, del_key=None, remove_all=False): for key in del_key: if key not in self.gtfs_feeds.keys(): - raise ValueError( - '{} key to delete was not found in gtfs_feeds'.format( - key)) + msg = '{} key to delete was not found in gtfs_feeds' + raise ValueError(msg.format(key)) del self.gtfs_feeds[key] log('Removed {} feed from gtfs_feeds'.format(key)) - def to_yaml(self, gtfsfeeddir=os.path.join(config.settings.data_folder, - 'gtfsfeeds'), - yamlname='gtfsfeeds.yaml', - overwrite=False): + def to_yaml(self, gtfsfeeddir=os.path.join( + config.settings.data_folder, 'gtfsfeeds'), + yamlname='gtfsfeeds.yaml', overwrite=False): """ Save an urbanaccess_gtfsfeeds representation to a YAML file. @@ -215,32 +213,14 @@ def to_yaml(self, gtfsfeeddir=os.path.join(config.settings.data_folder, yamlname : str or file like, optional File name to which to save a YAML file. overwrite : bool, optional - if true, overwrite an existing same name YAML file in specified - directory + if true, will overwrite an existing YAML + file in specified directory if file names are the same Returns ------- Nothing """ - - if not isinstance(gtfsfeeddir, str): - raise ValueError('gtfsfeeddir must be a string') - if not os.path.exists(gtfsfeeddir): - log( - '{} does not exist or was not found and will be ' - 'created'.format( - gtfsfeeddir)) - os.makedirs(gtfsfeeddir) - if not isinstance(yamlname, str): - raise ValueError('yaml must be a string') - yaml_file = os.path.join(gtfsfeeddir, yamlname) - if overwrite is False and os.path.isfile(yaml_file) is True: - raise ValueError( - '{} already exists. Rename or turn overwrite to True'.format( - yamlname)) - else: - with open(yaml_file, 'w') as f: - yaml.dump(self.to_dict(), f, default_flow_style=False) - log('{} file successfully created'.format(yaml_file)) + _dict_to_yaml(dictionary=self.to_dict(), yaml_dir=gtfsfeeddir, + yaml_name=yamlname, overwrite=overwrite) # instantiate the UrbanAccess GTFS feed object @@ -279,12 +259,10 @@ def search(api='gtfsdataexch', search_text=None, search_field=None, search_result_df : pandas.DataFrame Dataframe of search results displaying full feed metadata """ - - log( - 'Note: Your use of a GTFS feed is governed by each GTFS feed author ' - 'license terms. It is suggested you read the respective license ' - 'terms for the appropriate use of a GTFS feed.', - level=lg.WARNING) + msg = ('Note: Your use of a GTFS feed is governed by each GTFS feed author' + ' license terms. It is suggested you read the respective license ' + 'terms for the appropriate use of a GTFS feed.') + log(msg, level=lg.WARNING) if not isinstance(api, str): raise ValueError('{} must be a string'.format(api)) @@ -292,30 +270,27 @@ def search(api='gtfsdataexch', search_text=None, search_field=None, raise ValueError('{} is not currently a supported API'.format(api)) if config.settings.gtfs_api[api] is None or not isinstance( config.settings.gtfs_api[api], str): - raise ValueError('{} is not defined or defined incorrectly'.format( - api)) + raise ValueError('{} API is not defined or is ' + 'defined incorrectly'.format(api)) if not isinstance(match, str) or match not in ['contains', 'exact']: raise ValueError('match must be either: contains or exact') if not isinstance(add_feed, bool): raise ValueError('add_feed must be bool') if api == 'gtfsdataexch': - log( - 'Warning: The GTFSDataExchange is no longer being maintained as ' - 'of Summer 2016. ' - 'Data accessed here may be out of date.', level=lg.WARNING) + log('Warning: The GTFSDataExchange is no longer being maintained as ' + 'of Summer 2016. Data accessed here may be out of date.', + level=lg.WARNING) feed_table = pd.read_table(config.settings.gtfs_api[api], sep=',') - feed_table['date_added'] = pd.to_datetime(feed_table['date_added'], - unit='s') + feed_table['date_added'] = pd.to_datetime( + feed_table['date_added'], unit='s') feed_table['date_last_updated'] = pd.to_datetime( feed_table['date_last_updated'], unit='s') if search_text is None: - log( - 'No search parameters were passed. Returning full list of {} ' - 'GTFS feeds:'.format( - len(feed_table))) + log('No search parameters were passed. Returning full list of {} ' + 'GTFS feeds:'.format(len(feed_table))) return feed_table else: pass @@ -344,20 +319,19 @@ def search(api='gtfsdataexch', search_text=None, search_field=None, for text in search_text: if match == 'contains': search_result = feed_table[ - feed_table[col].str.contains(text, case=False, - na=False)] + feed_table[col].str.contains( + text, case=False, na=False)] if match == 'exact': search_result = feed_table[ - feed_table[col].str.match(text, case=False, - na=False)] - search_result_df = search_result_df.append(search_result) + feed_table[col].str.match( + text, case=False, na=False)] + search_result_df = search_result_df._append(search_result) search_result_df.drop_duplicates(inplace=True) log('Found {} records that matched {} inside {} columns:'.format( len(search_result_df), search_text, search_field)) if len(search_result_df) != 0: - if add_feed: if overwrite_feed: zip_url = search_result_df[ @@ -366,10 +340,8 @@ def search(api='gtfsdataexch', search_text=None, search_field=None, search_result_dict = search_result_df.set_index('name')[ 'dataexchange_url'].to_dict() feeds.gtfs_feeds = search_result_dict - log( - 'Replaced all records in gtfs_feed list with the {} ' - 'found records:'.format( - len(search_result_df))) + log('Replaced all records in gtfs_feed list with the {} ' + 'found records:'.format(len(search_result_df))) else: zip_url = search_result_df[ 'dataexchange_url'] + 'latest.zip' @@ -380,14 +352,13 @@ def search(api='gtfsdataexch', search_text=None, search_field=None, log('Added {} records to gtfs_feed list:'.format( len(search_result_df))) return search_result_dict - else: return search_result_df def download(data_folder=os.path.join(config.settings.data_folder), feed_name=None, feed_url=None, feed_dict=None, - error_pause_duration=5, delete_zips=False): + error_pause_duration=5, delete_zips=False, raise_url_error=True): """ Connect to the URLs passed in function or the URLs stored in the urbanaccess_gtfsfeeds instance and download the GTFS feed zipfile(s) @@ -400,22 +371,35 @@ def download(data_folder=os.path.join(config.settings.data_folder), data_folder : str, optional directory to download GTFS feed data to feed_name : str, optional - name of transit agency or service to use to name downloaded zipfile + name of transit agency or service to use to name downloaded zipfile. + If using feed_name and feed_url, cannot use feed_dict. feed_url : str, optional - corresponding URL to the feed_name to use to download GTFS feed zipfile + corresponding URL to the feed_name to use to download GTFS feed + zipfile. If using feed_name and feed_url, cannot use feed_dict. feed_dict : dict, optional Dictionary specifying the name of the transit service or agency GTFS feed as the key and the GTFS feed URL as the value: - {unique name of GTFS feed or transit service/agency : URL of feed} + {unique name of GTFS feed or transit service/agency : URL of feed}. + If using feed_dict, cannot use feed_name and feed_url. error_pause_duration : int, optional how long to pause in seconds before re-trying requests if error delete_zips : bool, optional - if true the downloaded zipfiles will be removed + if true the directory created by this function called 'gtfsfeed_zips' + that holds the downloaded zipfiles from each URL will be deleted. + This directory is located inside of the root data directory + specified in the 'data_folder' parameter under the the GTFS feed + directory for that zipfile for example: + data_folder\\feed_folder\\gtfsfeed_zips, default is False + raise_url_error : bool, optional + if true error will be raised when a request to a URL fails, URL + failure response will be returned, else if false URL failures will + still be printed to logs but will fail silently to allow process + to continue. Returns ------- - nothing + Nothing """ - + dtype_raise_error_msg = '{} must be a string' if (feed_name is not None and feed_url is None) or ( feed_url is not None and feed_name is None): raise ValueError( @@ -423,37 +407,33 @@ def download(data_folder=os.path.join(config.settings.data_folder), if feed_name is not None and feed_url is not None: if feed_dict is not None: - raise ValueError('feed_dict is not specified') + raise ValueError('only feed_dict or feed_name and ' + 'feed_url can be used at once. ' + 'Both cannot be used.') if not isinstance(feed_name, str) or not isinstance(feed_url, str): raise ValueError('either feed_name and or feed_url are not string') feeds.gtfs_feeds = {feed_name: feed_url} elif feed_dict is not None: - if feed_name is not None or feed_url is not None: - raise ValueError('either feed_name and or feed_url are not None') if not isinstance(feed_dict, dict): raise ValueError('feed_dict is not dict') for key in feed_dict.keys(): if not isinstance(key, str): - raise ValueError('{} must be a string'.format(key)) - for value in feed_dict[key]: - if not isinstance(value, str): - raise ValueError('{} must be a string'.format(value)) + raise ValueError(dtype_raise_error_msg.format(key)) + value = feed_dict[key] + if not isinstance(value, str): + raise ValueError(dtype_raise_error_msg.format(value)) - for key, value in feed_dict.items(): - if value in feeds.gtfs_feeds.values(): - raise ValueError( - 'duplicate values were found when the passed add_dict ' - 'dictionary was added to the existing dictionary. Feed ' - 'URL values must be unique.') + feed_dict_urls = list(feed_dict.values()) + has_dup_urls = len(set(feed_dict_urls)) != len(feed_dict_urls) + if has_dup_urls: + raise ValueError('duplicate values were found in feed_dict. ' + 'Feed URL values must be unique.') feeds.gtfs_feeds = feed_dict elif feed_name is None and feed_url is None and feed_dict is None: if len(feeds.gtfs_feeds) == 0: raise ValueError('No records were found in passed feed_dict') - feeds.gtfs_feeds - else: - raise ValueError('Passed parameters were incorrect or not specified.') download_folder = os.path.join(data_folder, 'gtfsfeed_zips') @@ -472,7 +452,11 @@ def download(data_folder=os.path.join(config.settings.data_folder), # TODO: add file counter and print number to user for feed_name_key, feed_url_value in feeds.gtfs_feeds.items(): start_time2 = time.time() - zipfile_path = ''.join([download_folder, '/', feed_name_key, '.zip']) + zipfile_name = '{}.zip'.format(feed_name_key) + zipfile_path = os.path.join(download_folder, zipfile_name) + + # resolve issues where request results in certificate verify failure + ssl._create_default_https_context = ssl._create_unverified_context # add default user-agent header in request to avoid 403 Errors opener = request.build_opener() @@ -493,6 +477,8 @@ def download(data_folder=os.path.join(config.settings.data_folder), log(msg_download_succeed.format( feed_name_key, time.time() - start_time2, os.path.getsize(zipfile_path))) + # deal with 429 Too Many Requests and 504 Gateway Timeout + # separately elif status_code in [429, 504]: msg = ('URL at {} returned status code {} and no data. ' 'Re-trying request in {:.2f} seconds.') @@ -502,16 +488,18 @@ def download(data_folder=os.path.join(config.settings.data_folder), time.sleep(error_pause_duration) try: file = request.urlopen(feed_url_value) - - _zipfile_type_check(file=file, - feed_url_value=feed_url_value) - + _zipfile_type_check( + file=file, feed_url_value=feed_url_value) with open(zipfile_path, "wb") as local_file: local_file.write(file.read()) except Exception: log(msg_no_connection_w_status.format( feed_url_value, status_code), level=lg.ERROR) + if raise_url_error: + raise Exception( + msg_no_connection_w_status.format( + feed_url_value, status_code)) else: log(msg_no_connection_w_status.format( feed_url_value, status_code), @@ -520,14 +508,17 @@ def download(data_folder=os.path.join(config.settings.data_folder), log(msg_no_connection.format( feed_url_value, traceback.format_exc()), level=lg.ERROR) + if raise_url_error: + raise Exception( + msg_no_connection.format( + feed_url_value, traceback.format_exc())) else: + # for non http links such as FTP try: file = request.urlopen(feed_url_value) _zipfile_type_check(file=file, feed_url_value=feed_url_value) - file_path = ''.join( - [download_folder, '/', feed_name_key, '.zip']) - with open(file_path, "wb") as local_file: + with open(zipfile_path, "wb") as local_file: local_file.write(file.read()) log(msg_download_succeed.format( feed_name_key, time.time() - start_time2, @@ -536,14 +527,96 @@ def download(data_folder=os.path.join(config.settings.data_folder), log(msg_no_connection.format( feed_url_value, traceback.format_exc()), level=lg.ERROR) + if raise_url_error: + raise Exception( + msg_no_connection.format( + feed_url_value, traceback.format_exc())) log('GTFS feed download completed. Took {:,.2f} seconds'.format( time.time() - start_time1)) - _unzip(zip_rootpath=download_folder, delete_zips=delete_zips) + unzip(zip_rootpath=download_folder, delete_zips=delete_zips) + + +def _unzip_util(zipfile_read_path, unzip_file_path, has_subzips): + """ + unzip GTFS feed zipfile in a root directory with resulting text files + in the root folder: gtfsfeed_text. If zipfile contains only zipfiles in + its subdirectory, recursively these zips will be extracted separately + into their own directories inside of root folder: gtfsfeed_text + + Parameters + ---------- + zipfile_read_path : string + full path to directory where zipfile to unzip is located + unzip_file_path : string + full path to directory where to unzip zipfile + has_subzips : bool + if true the parent zipfile contains zipfiles in its subdirectory. If + true subdirectory zipfiles will be extracted as separate feed folders. + + Returns + ------- + sub_zip_filelist : list + list of zipfile names that exist inside of the parent zipfile + directory, if there are none, will return Nothing + """ + with zipfile.ZipFile(zipfile_read_path) as z: + zip_file_list = z.namelist() + # required to deal with zipfiles that have subdirectories and + # that were created on OSX + filelist = [file for file in zip_file_list if + file.endswith(".txt") and not file.startswith("__MACOSX")] + # in cases where the zip contains multiple zips of GTFS feeds + # such as the case with the SEPTA GTFS feed + sub_zip_filelist = [file for file in zip_file_list if + file.endswith(".zip")] + # Note: there must only be zips inside of the sub directory, if its a + # mix of zips and txt files we assume that the parent zip contained + # both its zipped txt contents and a copy of itself as a zip in which + # case the GTFS txt files will be dealt with correctly but we + # will ignore the subdirectory zip + if filelist and sub_zip_filelist: + msg = ('Warning: Zipfile contains {:,} zipfile(s): {} in addition ' + 'to GTFS txt files: {}. Zipfile(s) inside of the parent ' + 'zipfile will not be extracted.') + log(msg.format(len(sub_zip_filelist), sub_zip_filelist, filelist), + level=lg.WARNING) + sub_zip_filelist = [] # null subdirectory zipfile list + if len(filelist) == 0 and len(sub_zip_filelist) > 0: + msg = ('Zipfile contains {:,} zipfiles: {}. These will be ' + 'extracted as separate feeds.') + log(msg.format(len(sub_zip_filelist), sub_zip_filelist)) + extract_subzips = True + else: + extract_subzips = False + + if has_subzips: + if not os.path.exists(unzip_file_path): + os.makedirs(unzip_file_path) + for file in filelist: + file_path = os.path.join( + unzip_file_path, os.path.basename(file)) + with open(file_path, 'wb') as f: + f.write(z.read(file)) + f.close() + else: + if extract_subzips: + filelist = sub_zip_filelist.copy() + if not os.path.exists(unzip_file_path): + os.makedirs(unzip_file_path) + for file in filelist: + file_path = os.path.join( + unzip_file_path, os.path.basename(file)) + with open(file_path, 'wb') as f: + f.write(z.read(file)) + f.close() + z.close() + if not has_subzips: + return sub_zip_filelist -def _unzip(zip_rootpath, delete_zips=True): +def unzip(zip_rootpath, delete_zips=False): """ unzip all GTFS feed zipfiles in a root directory with resulting text files in the root folder: gtfsfeed_text @@ -553,17 +626,17 @@ def _unzip(zip_rootpath, delete_zips=True): zip_rootpath : string root directory to place downloaded GTFS feed zipfiles delete_zips : bool, optional - if true the downloaded zipfiles will be removed + if true the directory specified in zip_rootpath will be deleted which + will delete all the downloaded zipfiles, default is False Returns ------- - nothing + Nothing """ - start_time = time.time() - unzip_rootpath = os.path.join(os.path.dirname(zip_rootpath), - 'gtfsfeed_text') + unzip_rootpath = os.path.join( + os.path.dirname(zip_rootpath), 'gtfsfeed_text') if not os.path.exists(unzip_rootpath): os.makedirs(unzip_rootpath) @@ -576,52 +649,61 @@ def _unzip(zip_rootpath, delete_zips=True): 'directory: {}'.format(zip_rootpath)) for zfile in zipfilelist: + unzipfile_name = zfile.replace('.zip', '') + unzip_file_path = os.path.join(unzip_rootpath, unzipfile_name) + zipfile_read_path = os.path.join(zip_rootpath, zfile) + + sub_zip_filelist = _unzip_util(zipfile_read_path, unzip_file_path, + has_subzips=False) + if len(sub_zip_filelist) != 0: + for sub_zipfile in sub_zip_filelist: + sub_zipfile_path = os.path.join(unzip_file_path, sub_zipfile) + # make unzip dir name unique in case others exist + sub_unzipfile_name = '{}_{}'.format( + unzipfile_name, sub_zipfile.replace('.zip', '')) + sub_unzipfile_path = os.path.join( + os.path.split(unzip_file_path)[0], sub_unzipfile_name) + _unzip_util( + sub_zipfile_path, sub_unzipfile_path, has_subzips=True) + msg = ('{} with nested zipfile: {} successfully extracted ' + 'to: {}') + log(msg.format(zfile, sub_zipfile, sub_unzipfile_path)) + os.remove(sub_zipfile_path) # remove the nested zip + os.rmdir(unzip_file_path) # remove the dir that had the nested zip + else: + log('{} successfully extracted to: {}'.format( + zfile, unzip_file_path)) - with zipfile.ZipFile(os.path.join(zip_rootpath, zfile)) as z: - # required to deal with zipfiles that have subdirectories and - # that were created on OSX - filelist = [file for file in z.namelist() if - file.endswith(".txt") and not file.startswith( - "__MACOSX")] - if not os.path.exists( - os.path.join(unzip_rootpath, zfile.replace('.zip', ''))): - os.makedirs( - os.path.join(unzip_rootpath, zfile.replace('.zip', ''))) - for file in filelist: - with open( - os.path.join(unzip_rootpath, zfile.replace('.zip', ''), - os.path.basename(file)), 'wb') as f: - f.write(z.read(file)) - f.close() - z.close() - log('{} successfully extracted to: {}'.format(zfile, os.path.join( - unzip_rootpath, zfile.replace('.zip', '')))) if delete_zips: - os.remove(zip_rootpath) - log('Deleted {} folder'.format(zip_rootpath)) - log( - 'GTFS feed zipfile extraction completed. Took {:,.2f} seconds for {} ' - 'files'.format( - time.time() - start_time, len(zipfilelist))) + shutil.rmtree(zip_rootpath) + log('Deleted {} folder and all its contents.'.format(zip_rootpath)) + msg = ('GTFS feed zipfile extraction completed. Took {:,.2f} seconds ' + 'for {:,} file(s)') + log(msg.format(time.time() - start_time, len(zipfilelist))) def _zipfile_type_check(file, feed_url_value): """ - zipfile format checker helper + Helper function to check if HTTP response contains a zipfile. + Valid application content are: 'zip' or 'octet-stream' Parameters ---------- - file : addinfourl - loaded zipfile object in memory + file : http.client.HTTPResponse + loaded zipfile HTTPResponse object in memory feed_url_value : str - URL to download GTFS feed zipfile + URL to download GTFS feed zipfile for informative purposes Returns ------- - nothing + Nothing """ - if 'zip' not in file.info().get('Content-Type') is True \ - or 'octet' not in file.info().get('Content-Type') is True: + # TODO: check for a better method to support cases that are zips but + # dont have header info example: sanfordcommunityredevelopmentagency + # GTFS feed URL + content_type = file.info().get('Content-Type') + if 'zip' not in content_type is True \ + or 'octet' not in content_type is True: raise ValueError( 'data requested at {} is not a zipfile. ' - 'Data must be a zipfile'.format(feed_url_value)) + 'Data must be a zipfile.'.format(feed_url_value)) diff --git a/urbanaccess/network.py b/urbanaccess/network.py index 372dcff..a91872a 100644 --- a/urbanaccess/network.py +++ b/urbanaccess/network.py @@ -1,21 +1,12 @@ import time import os -import geopy -from geopy import distance - -from sklearn.neighbors import KDTree import pandas as pd -from urbanaccess.utils import log, df_to_hdf5, hdf5_to_df +from urbanaccess.utils import log, df_to_hdf5, hdf5_to_df, _add_unique_stop_id +from urbanaccess.network_utils import connector_edges from urbanaccess import config -if int(geopy.__version__[0]) < 2: - dist_calc = distance.vincenty -else: - dist_calc = distance.geodesic - - class urbanaccess_network(object): """ An urbanaccess object of pandas.DataFrames representing @@ -53,34 +44,6 @@ def __init__(self, ua_network = urbanaccess_network() -def _nearest_neighbor(df1, df2): - """ - For a DataFrame of xy coordinates find the nearest xy - coordinates in a subsequent DataFrame - - Parameters - ---------- - df1 : pandas.DataFrame - DataFrame of records to return as the nearest record to records in df2 - df2 : pandas.DataFrame - DataFrame of records with xy coordinates for which to find the - nearest record in df1 for - Returns - ------- - df1.index.values[indexes] : pandas.Series - index of records in df1 that are nearest to the coordinates in df2 - """ - try: - df1_matrix = df1.to_numpy() - df2_matrix = df2.to_numpy() - except AttributeError: - df1_matrix = df1.values - df2_matrix = df2.values - kdt = KDTree(df1_matrix) - indexes = kdt.query(df2_matrix, k=1, return_distance=False) - return df1.index.values[indexes] - - def integrate_network(urbanaccess_network, headways=False, urbanaccess_gtfsfeeds_df=None, headway_statistic='mean'): """ @@ -97,9 +60,9 @@ def integrate_network(urbanaccess_network, headways=False, if true, route stop level headways calculated in a previous step will be applied to the OSM to transit connector edge travel time weights as an approximate measure - of average passenger transit stop waiting time. + of average passenger transit stop wait time. urbanaccess_gtfsfeeds_df : object, optional - required if headways is true; the gtfsfeeds_dfs object that holds + required if 'headways' is true; the gtfsfeeds_dfs object that holds the corresponding headways and stops DataFrames headway_statistic : {'mean', 'std', 'min', 'max'}, optional required if headways is true; route stop headway @@ -117,13 +80,15 @@ def integrate_network(urbanaccess_network, headways=False, urbanaccess_network.net_edges : pandas.DataFrame urbanaccess_network.net_nodes : pandas.DataFrame """ - if urbanaccess_network is None: raise ValueError('urbanaccess_network is not specified') - if urbanaccess_network.transit_edges.empty \ - or urbanaccess_network.transit_nodes.empty \ - or urbanaccess_network.osm_edges.empty \ - or urbanaccess_network.osm_nodes.empty: + is_transit_edges_empty = urbanaccess_network.transit_edges.empty + is_transit_nodes_empty = urbanaccess_network.transit_nodes.empty + is_osm_edges_empty = urbanaccess_network.osm_edges.empty + is_osm_nodes_empty = urbanaccess_network.osm_nodes.empty + + if is_transit_edges_empty or is_transit_nodes_empty or \ + is_osm_edges_empty or is_osm_nodes_empty: raise ValueError( 'one of the network objects: transit_edges, transit_nodes, ' 'osm_edges, or osm_nodes were found to be empty.') @@ -140,22 +105,23 @@ def integrate_network(urbanaccess_network, headways=False, raise ValueError('headways must be bool type') if headways: - - if urbanaccess_gtfsfeeds_df is None or \ - urbanaccess_gtfsfeeds_df.headways.empty or \ - urbanaccess_gtfsfeeds_df.stops.empty: + is_headway_df_empty = urbanaccess_gtfsfeeds_df.headways.empty + is_stops_df_empty = urbanaccess_gtfsfeeds_df.stops.empty + is_ua_gtfsfeeds_empty = urbanaccess_gtfsfeeds_df is None + if is_ua_gtfsfeeds_empty or is_headway_df_empty or is_stops_df_empty: raise ValueError( - 'stops and headway DataFrames were not found in the ' + 'stops and or headway DataFrames were not found in the ' 'urbanaccess_gtfsfeeds object. Please create these ' 'DataFrames in order to use headways.') valid_stats = ['mean', 'std', 'min', 'max'] - if headway_statistic not in valid_stats or not isinstance( - headway_statistic, str): + if headway_statistic not in valid_stats or \ + not isinstance(headway_statistic, str): raise ValueError('{} is not a supported statistic or is not a ' 'string'.format(headway_statistic)) transit_edge_cols = urbanaccess_network.transit_edges.columns + # TODO: verify if these checks are still required in various use cases if 'node_id_from' not in transit_edge_cols or 'from' in \ transit_edge_cols: urbanaccess_network.transit_edges.rename( @@ -177,7 +143,7 @@ def integrate_network(urbanaccess_network, headways=False, stops_df=urbanaccess_gtfsfeeds_df.stops, edges_w_routes=urbanaccess_network.transit_edges) - net_connector_edges = _connector_edges( + net_connector_edges = connector_edges( osm_nodes=urbanaccess_network.osm_nodes, transit_nodes=urbanaccess_network.transit_nodes, travel_speed_mph=3) @@ -188,15 +154,15 @@ def integrate_network(urbanaccess_network, headways=False, headway_statistic=headway_statistic) else: - urbanaccess_network.net_connector_edges = _connector_edges( + urbanaccess_network.net_connector_edges = connector_edges( osm_nodes=urbanaccess_network.osm_nodes, transit_nodes=urbanaccess_network.transit_nodes, travel_speed_mph=3) # change cols in transit edges and nodes if headways: - urbanaccess_network.transit_edges.rename(columns={ - 'node_id_route_from': 'from', 'node_id_route_to': 'to'}, + urbanaccess_network.transit_edges.rename( + columns={'node_id_route_from': 'from', 'node_id_route_to': 'to'}, inplace=True) urbanaccess_network.transit_edges.drop(['node_id_from', 'node_id_to'], inplace=True, axis=1) @@ -207,8 +173,8 @@ def integrate_network(urbanaccess_network, headways=False, urbanaccess_network.transit_edges.rename( columns={'node_id_from': 'from', 'node_id_to': 'to'}, inplace=True) urbanaccess_network.transit_nodes.reset_index(inplace=True, drop=False) - urbanaccess_network.transit_nodes.rename(columns={'node_id': 'id'}, - inplace=True) + urbanaccess_network.transit_nodes.rename( + columns={'node_id': 'id'}, inplace=True) # concat all network components urbanaccess_network.net_edges = pd.concat( @@ -217,8 +183,12 @@ def integrate_network(urbanaccess_network, headways=False, urbanaccess_network.net_connector_edges], axis=0) urbanaccess_network.net_nodes = pd.concat( - [urbanaccess_network.transit_nodes, - urbanaccess_network.osm_nodes], axis=0) + [ + urbanaccess_network.transit_nodes, + urbanaccess_network.osm_nodes.assign(id=urbanaccess_network.osm_nodes.index), + ], + axis=0, + ) urbanaccess_network.net_edges, urbanaccess_network.net_nodes = \ _format_pandana_edges_nodes(edge_df=urbanaccess_network.net_edges, @@ -263,28 +233,25 @@ def _add_headway_impedance(ped_to_transit_edges_df, headways_df, osm_to_transit_wheadway : pandas.DataFrame """ - start_time = time.time() - log( - '{} route stop headway will be used for pedestrian to transit edge ' - 'impedance.'.format( - headway_statistic)) + log('{} route stop headway will be used for pedestrian to transit edge ' + 'impedance.'.format(headway_statistic)) - osm_to_transit_wheadway = pd.merge(ped_to_transit_edges_df, headways_df[ - [headway_statistic, 'node_id_route']], - how='left', left_on=['to'], - right_on=['node_id_route'], sort=False, - copy=False) + osm_to_transit_wheadway = pd.merge( + ped_to_transit_edges_df, + headways_df[[headway_statistic, 'node_id_route']], + how='left', left_on=['to'], right_on=['node_id_route'], sort=False, + copy=False) + # divide headway statistic result by 2 assuming uniform distribution + # of passenger arrival at stop osm_to_transit_wheadway['weight_tmp'] = osm_to_transit_wheadway[ - 'weight'] + ( - osm_to_transit_wheadway[ - headway_statistic] / 2.0) + 'weight'] + (osm_to_transit_wheadway[headway_statistic] / 2.0) osm_to_transit_wheadway['weight_tmp'].fillna( osm_to_transit_wheadway['weight'], inplace=True) osm_to_transit_wheadway.drop('weight', axis=1, inplace=True) - osm_to_transit_wheadway.rename(columns={'weight_tmp': 'weight'}, - inplace=True) + osm_to_transit_wheadway.rename( + columns={'weight_tmp': 'weight'}, inplace=True) log('Headway impedance calculation completed. Took {:,.2f} seconds'.format( time.time() - start_time)) @@ -294,7 +261,8 @@ def _add_headway_impedance(ped_to_transit_edges_df, headways_df, def _route_id_to_node(stops_df, edges_w_routes): """ - Assign route ids to the transit nodes table + Assign route ids to the transit nodes table. Intended for use with + headway workflow. Parameters ---------- @@ -311,12 +279,10 @@ def _route_id_to_node(stops_df, edges_w_routes): start_time = time.time() # create unique stop IDs - stops_df['unique_stop_id'] = ( - stops_df['stop_id'].str.cat( - stops_df['unique_agency_id'].astype('str'), sep='_')) + stops_df = _add_unique_stop_id(stops_df) tmp1 = pd.merge(edges_w_routes[['node_id_from', 'node_id_route_from']], - stops_df[['unique_stop_id', 'stop_lat', 'stop_lon']], + stops_df, how='left', left_on='node_id_from', right_on='unique_stop_id', sort=False, copy=False) tmp1.rename(columns={'node_id_route_from': 'node_id_route', @@ -324,7 +290,7 @@ def _route_id_to_node(stops_df, edges_w_routes): 'stop_lat': 'y'}, inplace=True) tmp2 = pd.merge(edges_w_routes[['node_id_to', 'node_id_route_to']], - stops_df[['unique_stop_id', 'stop_lat', 'stop_lon']], + stops_df, how='left', left_on='node_id_to', right_on='unique_stop_id', sort=False, copy=False) @@ -333,12 +299,13 @@ def _route_id_to_node(stops_df, edges_w_routes): 'stop_lat': 'y'}, inplace=True) - transit_nodes_wroutes = pd.concat([tmp1[['node_id_route', 'x', 'y']], - tmp2[['node_id_route', 'x', 'y']]], - axis=0) + transit_nodes_wroutes = pd.concat([tmp1, tmp2], axis=0) transit_nodes_wroutes.drop_duplicates( subset='node_id_route', keep='first', inplace=True) + # drop temp columns + transit_nodes_wroutes.drop( + columns=['node_id_to', 'node_id_to'], inplace=True) # set node index to be unique stop ID transit_nodes_wroutes = transit_nodes_wroutes.set_index('node_id_route') @@ -348,69 +315,7 @@ def _route_id_to_node(stops_df, edges_w_routes): log('routes successfully joined to transit nodes. ' 'Took {:,.2f} seconds'.format(time.time() - start_time)) - return transit_nodes_wroutes - - -def _connector_edges(osm_nodes, transit_nodes, travel_speed_mph=3): - """ - Generate the connector edges between the OSM and transit edges and - weight by travel time - - Parameters - ---------- - osm_nodes : pandas.DataFrame - OSM nodes DataFrame - transit_nodes : pandas.DataFrame - transit nodes DataFrame - travel_speed_mph : int, optional - travel speed to use to calculate travel time across a - distance on an edge. units are in miles per hour (MPH) - for pedestrian travel this is assumed to be 3 MPH - - Returns - ------- - net_connector_edges : pandas.DataFrame - - """ - start_time = time.time() - - transit_nodes['nearest_osm_node'] = _nearest_neighbor( - osm_nodes[['x', 'y']], - transit_nodes[['x', 'y']]) - - net_connector_edges = [] - - for transit_node_id, row in transit_nodes.iterrows(): - # create new edge between the node in df2 (transit) - # and the node in OpenStreetMap (pedestrian) - - osm_node_id = int(row['nearest_osm_node']) - osm_row = osm_nodes.loc[osm_node_id] - - distance = dist_calc((row['y'], row['x']), - (osm_row['y'], osm_row['x'])).miles - time_ped_to_transit = distance / travel_speed_mph * 60 - time_transit_to_ped = distance / travel_speed_mph * 60 - - # save the edge - net_type = 'transit to osm' - net_connector_edges.append((transit_node_id, osm_node_id, - time_transit_to_ped, net_type)) - # make the edge bi-directional - net_type = 'osm to transit' - net_connector_edges.append((osm_node_id, transit_node_id, - time_ped_to_transit, net_type)) - - net_connector_edges = pd.DataFrame(net_connector_edges, - columns=["from", "to", - "weight", "net_type"]) - - log( - 'Connector edges between the OSM and transit network nodes ' - 'successfully completed. Took {:,.2f} seconds'.format( - time.time() - start_time)) - - return net_connector_edges + return transit_nodes_wroutes.dropna(subset=['x', 'y']) def _format_pandana_edges_nodes(edge_df, node_df): @@ -437,28 +342,36 @@ def _format_pandana_edges_nodes(edge_df, node_df): # Pandana requires IDs that are integer: for nodes - make it the index, # for edges make it the from and to columns node_df['id_int'] = range(1, len(node_df) + 1) + node_df['id_int'] = node_df.id_int.astype(int) + # TODO: reconsider the use of inplace or copy incoming dfs + # to memory to avoid changing in memory tables edge_df.rename(columns={'id': 'edge_id'}, inplace=True) - tmp = pd.merge(edge_df, node_df[['id', 'id_int']], left_on='from', - right_on='id', sort=False, copy=False, how='left') - tmp['from_int'] = tmp['id_int'] - tmp.drop(['id_int', 'id'], axis=1, inplace=True) - edge_df_wnumericid = pd.merge(tmp, node_df[['id', 'id_int']], left_on='to', - right_on='id', sort=False, copy=False, - how='left') - edge_df_wnumericid['to_int'] = edge_df_wnumericid['id_int'] - edge_df_wnumericid.drop(['id_int', 'id'], axis=1, inplace=True) + id_to_int = node_df.set_index('id')['id_int'].astype(int) + edge_df = edge_df.merge(id_to_int.rename('from_int').astype(int), left_on='from', + right_index=True, how='left') + + #tmp = pd.merge(edge_df, node_df[['id', 'id_int']], left_on='from', + # right_on='id', sort=False, copy=False, how='left') + #tmp['from_int'] = tmp['id_int'] + #tmp.drop(['id_int', 'id'], axis=1, inplace=True) + edge_df = edge_df.merge(id_to_int.rename('to_int').astype(int), left_on='to', right_index=True, how='left') + + # edge_df_wnumericid = pd.merge(tmp, node_df[['id', 'id_int']], left_on='to', + # right_on='id', sort=False, copy=False, + # how='left') + #edge_df_wnumericid['to_int'] = edge_df_wnumericid['id_int'] + #edge_df_wnumericid.drop(['id_int', 'id'], axis=1, inplace=True) # turn mixed dtype cols into all same format - col_list = edge_df_wnumericid.select_dtypes(include=['object']).columns + col_list = edge_df.select_dtypes(include=["object"]).columns for col in col_list: try: - edge_df_wnumericid[col] = edge_df_wnumericid[col].astype(str) + edge_df[col] = edge_df[col].astype(str) # deal with edge cases where typically the name of a street is not # in an uniform string encoding such as names with accents except UnicodeEncodeError: log('Fixed unicode error in {} column'.format(col)) - edge_df_wnumericid[col] = edge_df_wnumericid[col].str.encode( - 'utf-8') + edge_df[col] = edge_df[col].str.encode("utf-8") node_df.set_index('id_int', drop=True, inplace=True) # turn mixed dtype col into all same format @@ -469,7 +382,7 @@ def _format_pandana_edges_nodes(edge_df, node_df): log('Edge and node tables formatted for Pandana with integer node IDs: ' 'id_int, to_int, and from_int. Took {:,.2f} seconds'.format( time.time() - start_time)) - return edge_df_wnumericid, node_df + return edge_df, node_df.dropna(subset=["x", "y"]) def save_network(urbanaccess_network, filename, @@ -496,11 +409,12 @@ def save_network(urbanaccess_network, filename, Returns ------- - None + Nothing """ log('Writing HDF5 store...') - if urbanaccess_network is None or urbanaccess_network.net_edges.empty or \ - urbanaccess_network.net_nodes.empty: + is_net_edges_empty = urbanaccess_network.net_edges.empty + is_net_nodes_empty = urbanaccess_network.net_nodes.empty + if urbanaccess_network is None or is_net_edges_empty or is_net_nodes_empty: raise ValueError('Either no urbanaccess_network specified or ' 'net_edges or net_nodes are empty.') diff --git a/urbanaccess/network_utils.py b/urbanaccess/network_utils.py new file mode 100644 index 0000000..26fb0c7 --- /dev/null +++ b/urbanaccess/network_utils.py @@ -0,0 +1,100 @@ +import time +import geopy +from geopy import distance + +from sklearn.neighbors import KDTree +import pandas as pd + +from urbanaccess.utils import log + + +if int(geopy.__version__[0]) < 2: + dist_calc = distance.vincenty +else: + dist_calc = distance.geodesic + + +def nearest_neighbor(df1, df2): + """ + For a DataFrame of xy coordinates find the nearest xy + coordinates in a subsequent DataFrame + + Parameters + ---------- + df1 : pandas.DataFrame + DataFrame of records to return as the nearest record to records in df2 + df2 : pandas.DataFrame + DataFrame of records with xy coordinates for which to find the + nearest record in df1 for + Returns + ------- + df1.index.values[indexes] : pandas.Series + index of records in df1 that are nearest to the coordinates in df2 + """ + try: + df1_matrix = df1.to_numpy() + df2_matrix = df2.to_numpy() + except AttributeError: + df1_matrix = df1.values + df2_matrix = df2.values + kdt = KDTree(df1_matrix) + indexes = kdt.query(df2_matrix, k=1, return_distance=False) + return df1.index.values[indexes] + + +def connector_edges(osm_nodes, transit_nodes, travel_speed_mph=3): + """ + Generate the connector edges between the OSM and transit edges and + weight by travel time + + Parameters + ---------- + osm_nodes : pandas.DataFrame + OSM nodes DataFrame + transit_nodes : pandas.DataFrame + transit nodes DataFrame + travel_speed_mph : int, optional + travel speed to use to calculate travel time across a + distance on an edge. units are in miles per hour (MPH) + for pedestrian travel this is assumed to be 3 MPH + + Returns + ------- + net_connector_edges : pandas.DataFrame + + """ + start_time = time.time() + + transit_nodes['nearest_osm_node'] = nearest_neighbor( + osm_nodes[['x', 'y']], + transit_nodes[['x', 'y']]) + + net_connector_edges = [] + for transit_node_id, row in transit_nodes.iterrows(): + # create new edge between the node in df2 (transit) + # and the node in OpenStreetMap (pedestrian) + osm_node_id = int(row['nearest_osm_node']) + osm_row = osm_nodes.loc[osm_node_id] + + distance_mi = dist_calc( + (row['y'], row['x']), (osm_row['y'], osm_row['x'])).miles + time_ped_to_transit = distance_mi / travel_speed_mph * 60 + time_transit_to_ped = distance_mi / travel_speed_mph * 60 + + # save the edge + net_type = 'transit to osm' + net_connector_edges.append((transit_node_id, osm_node_id, + time_transit_to_ped, net_type)) + # make the edge bi-directional + net_type = 'osm to transit' + net_connector_edges.append((osm_node_id, transit_node_id, + time_ped_to_transit, net_type)) + + net_connector_edges = pd.DataFrame( + net_connector_edges, columns=["from", "to", "weight", "net_type"]) + + log('Connector edges between the OSM and transit network nodes ' + 'successfully completed. Took {:,.2f} seconds'.format( + time.time() - start_time)) + + return net_connector_edges diff --git a/urbanaccess/osm/load.py b/urbanaccess/osm/load.py index c2f6da3..30fe4fd 100644 --- a/urbanaccess/osm/load.py +++ b/urbanaccess/osm/load.py @@ -63,10 +63,9 @@ def ua_network_from_bbox(lat_min=None, lng_min=None, lat_max=None, Returns ------- - nodesfinal, edgesfinal : pandas.DataFrame + nodes, edges : pandas.DataFrame """ - start_time = time.time() # returned OSM data allows travel in both directions @@ -83,33 +82,32 @@ def ua_network_from_bbox(lat_min=None, lng_min=None, lat_max=None, # remove low connectivity nodes and return cleaned nodes and edges if remove_lcn: log('Checking for low connectivity nodes...') - pandana_net = Network(nodes['x'], nodes['y'], - edges['from'], edges['to'], edges[['distance']]) - lcn = pandana_net.low_connectivity_nodes(impedance=10000, count=10, - imp_name='distance') - log( - '{:,} out of {:,} nodes ({:.2f} percent of total) were ' - 'identified as having low connectivity and have ' - 'been removed.'.format(len(lcn), len(nodes), - (len(lcn) / len(nodes)) * 100)) - - rm_nodes = set(lcn) - - nodes_to_keep = ~nodes.index.isin(rm_nodes) - edges_to_keep = ~( - edges['from'].isin(rm_nodes) | edges['to'].isin(rm_nodes)) - - nodes = nodes.loc[nodes_to_keep] - edges = edges.loc[edges_to_keep] - - log('Completed OSM data download and graph node and edge table ' - 'creation in {:,.2f} seconds'.format(time.time() - start_time)) - - return nodes, edges - - else: - - log('Completed OSM data download and graph node and edge table ' - 'creation in {:,.2f} seconds'.format(time.time() - start_time)) - - return nodes, edges + # use twoway=True only for low connectivity purposes + pandana_net = Network( + node_x=nodes['x'], node_y=nodes['y'], + edge_from=edges['from'], edge_to=edges['to'], + edge_weights=edges[['distance']], twoway=True) + lcn = pandana_net.low_connectivity_nodes( + impedance=10000, count=10, imp_name='distance') + len_lcn = len(lcn) + len_nodes = len(nodes) + if len_lcn > 0: + rm_nodes = set(lcn) + + nodes_to_keep = ~nodes.index.isin(rm_nodes) + edges_to_keep = ~( + edges['from'].isin(rm_nodes) | edges['to'].isin(rm_nodes)) + + nodes = nodes.loc[nodes_to_keep] + edges = edges.loc[edges_to_keep] + msg = ('{:,} out of {:,} node(s) ({:.2f} percent of total) were ' + 'identified as having low connectivity and have ' + 'been removed.') + log(msg.format(len_lcn, len_nodes, (len_lcn / len_nodes) * 100)) + else: + log('No nodes were identified as having low connectivity.') + + log('Completed OSM data download and graph node and edge table ' + 'creation in {:,.2f} seconds.'.format(time.time() - start_time)) + + return nodes, edges diff --git a/urbanaccess/osm/network.py b/urbanaccess/osm/network.py index 9eae162..3ca14a7 100644 --- a/urbanaccess/osm/network.py +++ b/urbanaccess/osm/network.py @@ -36,12 +36,14 @@ def create_osm_net(osm_edges, osm_nodes, start_time = time.time() if not isinstance(network_type, str) or network_type is None: - raise ValueError('{!s} network_type passed is either not a ' - 'string or is None'.format(network_type)) + raise ValueError("'{!s}' network_type passed is either not a " + "string or is None.".format(network_type)) - # assign impedance to OSM edges - osm_edges['weight'] = (osm_edges[ - 'distance'] / 1609.34) / travel_speed_mph * 60 + # assign impedance to OSM edges where: + # (distance in meters / 1 mile in meters) / + # speed in mph * mph in mile per minute + osm_edges['weight'] = \ + (osm_edges['distance'] / 1609.34) / travel_speed_mph * 60 # assign node and edge net type osm_edges['net_type'] = network_type @@ -50,9 +52,8 @@ def create_osm_net(osm_edges, osm_nodes, ua_network.osm_nodes = osm_nodes ua_network.osm_edges = osm_edges - log( - 'Created OSM network with travel time impedance using a travel speed ' - 'of {} MPH. Took {:,.2f} seconds'.format( + log('Created OSM network with travel time impedance using a travel speed ' + 'of {} MPH. Took {:,.2f} seconds.'.format( travel_speed_mph, time.time() - start_time)) return ua_network diff --git a/urbanaccess/tests/conftest.py b/urbanaccess/tests/conftest.py index cdbd8c5..97f7865 100644 --- a/urbanaccess/tests/conftest.py +++ b/urbanaccess/tests/conftest.py @@ -3,6 +3,9 @@ import pandas as pd import numpy as np +from urbanaccess.tests.test_gtfs_headways import \ + _build_expected_hw_intermediate_stop_times_data + @pytest.fixture def agency_feed_1(): @@ -295,8 +298,8 @@ def calendar_feed_1(): 'friday': [1, 1, 1, 0], 'saturday': [0, 0, 0, 1], 'sunday': [0, 0, 0, 1], - 'start_date': [20161224] * 4, - 'end_date': [20170318] * 4} + 'start_date': ['20161224'] * 4, + 'end_date': ['20170318'] * 4} index = range(4) @@ -318,8 +321,8 @@ def calendar_feed_2(): 'friday': [1, 1, 1, 0], 'saturday': [0, 0, 0, 1], 'sunday': [0, 0, 0, 1], - 'start_date': [20161224] * 4, - 'end_date': [20170318] * 4} + 'start_date': ['20161224'] * 4, + 'end_date': ['20170318'] * 4} index = range(4) @@ -338,8 +341,8 @@ def calendar_feed_4(): 'friday': [1], 'saturday': [0], 'sunday': [0], - 'start_date': [20161224], - 'end_date': [20170318]} + 'start_date': ['20161224'], + 'end_date': ['20170318']} index = range(1) @@ -347,23 +350,6 @@ def calendar_feed_4(): return df -@pytest.fixture -def calendar_empty(): - columns = {'service_id', - 'monday', - 'tuesday', - 'wednesday', - 'thursday', - 'friday', - 'saturday', - 'sunday', - 'start_date', - 'end_date'} - - df = pd.DataFrame(columns=columns) - return df - - @pytest.fixture def calendar_dates_feed_1(): data = { @@ -371,8 +357,8 @@ def calendar_dates_feed_1(): 'weekday-2', 'weekday-3', 'weekend-1'], - 'date': [20161224, 20170318, 20160424, 20161230], - 'exception_type': [1, 2, 1, 1], + 'date': ['20161224', '20170318', '20160424', '20161230'], + 'exception_type': ['1', '2', '1', '1'], 'schedule_type': ['WD', 'WD', 'WD', 'WE']} index = range(4) @@ -388,8 +374,8 @@ def calendar_dates_feed_2(): 'weekday-2', 'weekday-3', 'weekend-1'], - 'date': [20161224, 20170318, 20160424, 20161230], - 'exception_type': [1, 2, 1, 1], + 'date': ['20161224', '20170318', '20160424', '20161230'], + 'exception_type': ['1', '2', '1', '1'], 'schedule_type': ['WD', 'WD', 'WD', 'SA'] } @@ -403,8 +389,8 @@ def calendar_dates_feed_2(): def calendar_dates_feed_4(): data = { 'service_id': ['wk-1'], - 'date': [20161224], - 'exception_type': [1]} + 'date': ['20161224'], + 'exception_type': ['1']} index = range(1) @@ -681,6 +667,221 @@ def stop_times_feed_4(): return df +@pytest.fixture +def hw_stops_df_1_agency(): + # intended for use in headway tests + data = { + 'stop_id': ['1', '2', '3', '4', '5', '6', + '7', '8', '9'], + 'stop_name': ['ave a', 'ave b', 'ave c', 'ave d', 'ave e', 'ave f', + '1st st', '2nd st', '3rd st'], + 'stop_lat': [37.797484, 37.774963, 37.803664, 37.80787, 37.828415, + 37.844601, 37.664174, 37.591208, 37.905628], + 'stop_lon': [-122.265609, -122.224274, -122.271604, -122.269029, + -122.267227, -122.251793, -122.444116, -122.017867, + -122.067423], + 'location_type': [1, 1, 1, 1, 1, 1, + 2, 2, 2], + 'wheelchair_boarding': [1, 0, 0, 0, 0, 0, + 1, 1, 1], + 'unique_agency_id': ['agency_a_city_a'] * 9 + } + index = range(9) + df = pd.DataFrame(data, index) + return df + + +@pytest.fixture +def hw_routes_df_1_agency(): + # intended for use in headway tests + data = { + 'agency_id': ['agency a city a'] * 2, + 'route_id': ['10-101', 'B1'], + 'route_long_name': ['Route 10-101', 'B1 Express'], + 'route_type': [3, 2], + 'unique_agency_id': ['agency_a_city_a'] * 2, + 'unique_feed_id': ['feed_1'] * 2} + index = range(2) + df = pd.DataFrame(data, index) + return df + + +@pytest.fixture +def hw_stop_times_int_df_1_agency(): + # intended for use in headway tests + + # only generate the data that is expected, leaving out: arrival_time, + # departure_time + + # outbound direction + data = { + 'trip_id': ['a1', 'a1', + 'a1', 'a1', + 'a1', 'a1', + 'B1', 'B1', + 'B1'], + 'stop_id': [1, 2, 3, 4, 5, 6, + 7, 8, 9], + 'stop_sequence': [1, 2, 3, 4, 5, 6, + 1, 2, 3], + 'unique_agency_id': ['agency_a_city_a'] * 9, + 'unique_feed_id': ['feed_1'] * 9, + 'route_type': [3, 3, 3, 3, 3, 3, + 2, 2, 2], + 'departure_time_sec': [ + 22500.0, 22620.0, 22860.0, 23100.0, 23580.0, 23940.0, + 22500.0, 22680.0, 22980.0], + # nan, 2, 4, 4, 8, 6 # min + # nan, 3, 5 # min + 'timediff': [ + np.nan, 120, 240, 240, 480, 360, + np.nan, 180, 300] + } + index = range(9) + direction_0 = pd.DataFrame(data, index) + + # inbound direction + data = { + 'trip_id': ['a2', 'a2', + 'a2', 'a2', + 'a2', 'a2', + 'B2', 'B2', + 'B2'], + 'stop_id': [6, 5, 4, 3, 2, 1, + 9, 8, 7], + 'stop_sequence': [1, 2, 3, 4, 5, 6, + 1, 2, 3], + 'unique_agency_id': ['agency_a_city_a'] * 9, + 'unique_feed_id': ['feed_1'] * 9, + 'route_type': [3, 3, 3, 3, 3, 3, + 2, 2, 2], + 'departure_time_sec': [ + 23940.0, 23580.0, 23100.0, 22860.0, 22620.0, 22500.0, + 22980.0, 22680.0, 22500.0], + # nan, 2, 4, 4, 8, 6 # min + # nan, 3, 5 # min + 'timediff': [ + np.nan, 360, 480, 240, 240, 120, + np.nan, 300, 180] + } + index = range(9) + direction_1 = pd.DataFrame(data, index) + + df = pd.concat([direction_1, direction_0], ignore_index=True) + + # make multiple trips that stop at the same stops but at offset times + # so we can replicate headways for each route + merged_dfs = df.copy() + trip_count = 5 + # set some trips to have even headways + even_headway = 1200 + # set some trips to have uneven headways but predicable + uneven_headway = {1: 900, 2: 1200, 3: 500, 4: 600, 5: 1800} + for trip_id in range(trip_count): + df_2 = df.copy() + trip_num = trip_id + 1 + # 20 min headways for (1200 sec) with even spacing + col = 'departure_time_sec' + trip = 'a1' + df_2.loc[df['trip_id'] == trip, col] = \ + df_2.loc[df['trip_id'] == trip, col] + even_headway + trip = 'a2' + df_2.loc[df['trip_id'] == trip, col] = \ + df_2.loc[df['trip_id'] == trip, col] + even_headway + # 15-30 min headways for (900-1200 sec) with uneven spacing and + trip = 'B1' + df_2.loc[df_2['trip_id'] == trip, col] = \ + df_2.loc[df['trip_id'] == trip, col] + uneven_headway[trip_num] + trip = 'B2' + df_2.loc[df_2['trip_id'] == trip, col] = \ + df_2.loc[df['trip_id'] == trip, col] + uneven_headway[trip_num] + df_2['trip_id'] = df_2['trip_id'] + '_{}'.format(trip_num) + + merged_dfs = merged_dfs.append(df_2, ignore_index=True) + + # add the other expected cols + merged_dfs['departure_time_sec_interpolate'] = merged_dfs[ + 'departure_time_sec'].astype('int') + merged_dfs['unique_stop_id'] = ( + merged_dfs['stop_id'].astype('str').str.cat( + merged_dfs['unique_agency_id'].astype('str'), sep='_')) + merged_dfs['unique_trip_id'] = merged_dfs['trip_id'].str.cat( + merged_dfs['unique_agency_id'].astype('str'), sep='_') + + return merged_dfs + + +@pytest.fixture +def hw_trips_df_1_agency(): + # intended for use in headway tests + data = { + 'route_id': ['10-101', '10-101', + 'B1', 'B1'], + 'trip_id': ['a1', 'a2', + 'B1', 'B2'], + 'service_id': ['weekday-1'] * 4, + 'direction_id': [1, 0, + 1, 0], + 'unique_agency_id': ['agency_a_city_a'] * 4, + 'unique_feed_id': ['feed_1'] * 4} + index = range(4) + df = pd.DataFrame(data, index) + + # expand trips to match what is in the test stop_times table + merged_dfs = df.copy() + trip_count = 5 + for trip_id in range(trip_count): + df_2 = df.copy() + trip_num = trip_id + 1 + df_2['trip_id'] = df_2['trip_id'] + '_{}'.format(trip_num) + merged_dfs = merged_dfs.append(df_2, ignore_index=True) + + return merged_dfs + + +@pytest.fixture() +def hw_headways_df_1_agency( + hw_trips_df_1_agency, hw_routes_df_1_agency, + hw_stop_times_int_df_1_agency): + # intended for use in headway tests + + # build expected headway table + int_stop_times = _build_expected_hw_intermediate_stop_times_data( + trips_df=hw_trips_df_1_agency, + routes_df=hw_routes_df_1_agency, + stop_times_df=hw_stop_times_int_df_1_agency) + + int_stop_times['unique_stop_route'] = ( + int_stop_times['unique_stop_id'].str.cat( + int_stop_times['unique_route_id'].astype('str'), sep=',')) + + stop_route_groups = int_stop_times.groupby('unique_stop_route') + + results = {} + col = 'departure_time_sec_interpolate' + for unique_stop_route, stop_route_group in stop_route_groups: + stop_route_group.sort_values([col], ascending=True, inplace=True) + next_veh_time = (stop_route_group[col].iloc[1:].values) + prev_veh_time = (stop_route_group[col].iloc[:-1].values) + stop_route_group_headways = (next_veh_time - prev_veh_time) / 60 + results[unique_stop_route] = (pd.Series(stop_route_group_headways) + .describe()) + + headway_by_routestop_df = pd.DataFrame(results).T + + headway_by_routestop_df = pd.merge( + headway_by_routestop_df, + int_stop_times[ + ['unique_stop_route', 'unique_stop_id', 'unique_route_id']], + how='left', left_index=True, right_on='unique_stop_route', sort=False) + headway_by_routestop_df.drop('unique_stop_route', axis=1, inplace=True) + headway_by_routestop_df['node_id_route'] = ( + headway_by_routestop_df['unique_stop_id'].str.cat( + headway_by_routestop_df['unique_route_id'].astype('str'), sep='_')) + + return headway_by_routestop_df + + @pytest.fixture() def agency_a_feed_on_disk_wo_calendar_dates( tmpdir, @@ -826,3 +1027,24 @@ def agency_a_feed_on_disk_w_calendar_and_calendar_dates_empty_txt( feed_file_name = '{}.txt'.format(feed_file) feed_df.to_csv(os.path.join(feed_path, feed_file_name), index=False) return feed_path + + +@pytest.fixture() +def agency_b_feed_on_disk_w_calendar_and_calendar_dates( + tmpdir, + agency_feed_2, stop_times_feed_2, stops_feed_2, + routes_feed_2, trips_feed_2, calendar_feed_2, calendar_dates_feed_2): + feed_file_dict = {'agency': agency_feed_2, + 'stop_times': stop_times_feed_2, + 'stops': stops_feed_2, + 'routes': routes_feed_2, + 'trips': trips_feed_2, + 'calendar': calendar_feed_2, + 'calendar_dates': calendar_dates_feed_2} + feed_path = os.path.join(tmpdir.strpath, 'agency_b_w_both_calendars') + os.makedirs(feed_path) + print('writing test data to dir: {}'.format(feed_path)) + for feed_file, feed_df in feed_file_dict.items(): + feed_file_name = '{}.txt'.format(feed_file) + feed_df.to_csv(os.path.join(feed_path, feed_file_name), index=False) + return feed_path diff --git a/urbanaccess/tests/test_config.py b/urbanaccess/tests/test_config.py new file mode 100644 index 0000000..8d6de48 --- /dev/null +++ b/urbanaccess/tests/test_config.py @@ -0,0 +1,134 @@ +import pytest +import yaml +import os + +from urbanaccess import config + + +@pytest.fixture +def settings_valid(): + # build current dict of configs + return config.settings.to_dict() + + +@pytest.fixture +def expected_settings_dict(): + return { + 'data_folder': 'data', + 'images_folder': 'images', + 'image_filename': 'urbanaccess_plot', + 'logs_folder': 'logs', + 'log_file': True, + 'log_console': False, + 'log_name': 'urbanaccess', + 'log_filename': 'urbanaccess', + 'txt_encoding': 'utf-8', + 'gtfs_api': { + 'gtfsdataexch': + 'http://www.gtfs-data-exchange.com/api/agencies' + '?format=csv'}} + + +@pytest.fixture +def valid_setting_keys(): + return sorted(['data_folder', 'images_folder', 'image_filename', + 'logs_folder', 'log_file', 'log_console', 'log_name', + 'log_filename', 'txt_encoding', 'gtfs_api']) + + +@pytest.fixture +def settings_from_yaml(tmpdir, settings_valid): + # make 1 change to test yaml loading downstream + settings_valid['data_folder'] = 'test' + yaml_path = os.path.join(tmpdir.strpath, 'urbanaccess_config.yaml') + with open(yaml_path, 'w') as f: + yaml.dump(settings_valid, f, default_flow_style=False) + return tmpdir.strpath + + +@pytest.fixture +def settings_to_yaml(tmpdir, settings_valid): + # make 1 change to test yaml loading downstream + config.settings.data_folder = 'test' + return config.settings + + +def test_config_object(): + assert isinstance(config.settings, config.urbanaccess_config) + assert isinstance(config.settings.to_dict(), dict) + + +def test_config_to_dict(expected_settings_dict): + assert isinstance(config.settings.to_dict(), dict) + assert config.settings.to_dict() == expected_settings_dict + + +def test_format_check(settings_valid): + config._format_check(settings_valid) + + +def test_format_check_invalid( + valid_setting_keys, expected_settings_dict): + key_error_msg = ("Configuration keys: {} do not match " + "required keys: {}.") + with pytest.raises(ValueError) as excinfo: + # remove valid key txt_encoding + invalid_dict = expected_settings_dict.copy() + del invalid_dict['txt_encoding'] + config._format_check(invalid_dict) + expected_error = key_error_msg.format( + list(invalid_dict.keys()), valid_setting_keys) + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + # add invalid key + invalid_dict = expected_settings_dict.copy() + invalid_dict.update({'invalid_key': 'test'}) + config._format_check(invalid_dict) + expected_error = key_error_msg.format( + list(invalid_dict.keys()), valid_setting_keys) + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + invalid_dict = expected_settings_dict.copy() + invalid_dict['log_filename'] = 1 + config._format_check(invalid_dict) + expected_error = "Key: 'log_filename' value must be string." + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + invalid_dict = expected_settings_dict.copy() + invalid_dict['log_console'] = 'invalid' + config._format_check(invalid_dict) + expected_error = "Key: 'log_console' value must be boolean." + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + invalid_dict = expected_settings_dict.copy() + invalid_dict['gtfs_api'] = {'invalid': 'test_url'} + config._format_check(invalid_dict) + expected_error = ("gtfs_api key: 'invalid' does not match valid " + "key(s): ['gtfsdataexch'].") + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + invalid_dict = expected_settings_dict.copy() + invalid_dict['gtfs_api']['gtfsdataexch'] = 1 + config._format_check(invalid_dict) + expected_error = ("gtfs_api key: 'gtfsdataexch' value must be " + "string.") + assert expected_error in str(excinfo.value) + + +def test_from_yaml_config(settings_from_yaml): + result_settings = config.settings.from_yaml( + configdir=settings_from_yaml, yamlname='urbanaccess_config.yaml') + result_settings_dict = result_settings.to_dict() + assert result_settings_dict['data_folder'] == 'test' + + +def test_to_yaml_config(tmpdir, settings_to_yaml): + yamlname = 'urbanaccess_config.yaml' + settings_to_yaml.to_yaml( + configdir=tmpdir.strpath, yamlname=yamlname, overwrite=True) + + yaml_path = os.path.join(tmpdir.strpath, yamlname) + with open(yaml_path, 'r') as f: + yaml_config = yaml.safe_load(f) + assert sorted(yaml_config) == sorted(settings_to_yaml.to_dict()) + assert yaml_config['data_folder'] == 'test' diff --git a/urbanaccess/tests/test_gtfs_headways.py b/urbanaccess/tests/test_gtfs_headways.py new file mode 100644 index 0000000..4727ea6 --- /dev/null +++ b/urbanaccess/tests/test_gtfs_headways.py @@ -0,0 +1,270 @@ +import pytest +import pandas as pd + +import urbanaccess.gtfs.headways as headways +from urbanaccess.gtfs.gtfsfeeds_dataframe import urbanaccess_gtfs_df \ + as gtfsfeeds_df + + +def _build_expected_hw_intermediate_stop_times_data( + trips_df, routes_df, stop_times_df): + trips_df['unique_trip_id'] = trips_df['trip_id'].str.cat( + trips_df['unique_agency_id'].astype('str'), sep='_') + trips_df['unique_route_id'] = trips_df['route_id'].str.cat( + trips_df['unique_agency_id'].astype('str'), sep='_') + + columns = ['unique_route_id', 'service_id', 'unique_trip_id', + 'unique_agency_id', 'direction_id'] + trips_df = trips_df[columns] + + routes_df['unique_route_id'] = routes_df['route_id'].str.cat( + routes_df['unique_agency_id'].astype('str'), sep='_') + columns = ['unique_route_id', 'route_long_name', 'route_type', + 'unique_agency_id'] + routes_df = routes_df[columns] + + tmp1 = pd.merge(trips_df, routes_df, how='left', left_on='unique_route_id', + right_on='unique_route_id', sort=False) + merge_df = pd.merge(stop_times_df, tmp1, how='left', + left_on='unique_trip_id', right_on='unique_trip_id', + sort=False) + cols_to_drop = ['unique_agency_id_y', 'unique_agency_id_x'] + merge_df.drop(cols_to_drop, axis=1, inplace=True) + return merge_df + + +def _check_headway_result( + result, hw_stop_times_int_df_1_agency, expected_route_stop_ids, + expected_route_stop_headway_result_sample_1, + expected_route_stop_headway_result_sample_2): + expected_cols = ['count', 'mean', 'std', 'min', '25%', '50%', '75%', 'max', + 'unique_stop_id', 'unique_route_id', 'node_id_route'] + assert isinstance(result, pd.core.frame.DataFrame) + assert result.empty is False + # each stop_time has a corresponding headway so len should be the same + # but this can be simplified in the future to return only unique data + assert len(result) == len(hw_stop_times_int_df_1_agency) + for col in expected_cols: + assert col in result.columns + assert sorted(result['node_id_route'].unique()) == expected_route_stop_ids + + # check subset of values + subset = result.loc[result['node_id_route'] == + '1_agency_a_city_a_10-101_agency_a_city_a'] + subset.reset_index(inplace=True, drop=True) + subset = subset.round(2) + assert len(subset) == 12 + assert subset[0:1].equals(expected_route_stop_headway_result_sample_1) + + subset = result.loc[result['node_id_route'] == + '7_agency_a_city_a_B1_agency_a_city_a'] + subset.reset_index(inplace=True, drop=True) + subset = subset.round(2) + assert len(subset) == 12 + assert subset[0:1].equals(expected_route_stop_headway_result_sample_2) + + +@pytest.fixture +def ua_gtfsfeeds_df( + hw_routes_df_1_agency, hw_trips_df_1_agency, + hw_stop_times_int_df_1_agency): + gtfsfeeds_df.routes = hw_routes_df_1_agency.copy() + gtfsfeeds_df.trips = hw_trips_df_1_agency.copy() + gtfsfeeds_df.stop_times_int = hw_stop_times_int_df_1_agency.copy() + return gtfsfeeds_df + + +@pytest.fixture +def interm_stop_times_df_1_agency( + hw_trips_df_1_agency, hw_routes_df_1_agency, + hw_stop_times_int_df_1_agency): + int_stop_times = _build_expected_hw_intermediate_stop_times_data( + trips_df=hw_trips_df_1_agency, + routes_df=hw_routes_df_1_agency, + stop_times_df=hw_stop_times_int_df_1_agency) + return int_stop_times + + +@pytest.fixture +def expected_route_stop_headway_result_sample_1(): + data = { + 'count': [11.0], + 'mean': [1.818182], + 'std': [6.030227], + 'min': [0.0], + '25%': [0.0], '50%': [0.0], '75%': [0.0], + 'max': [20.0], + 'unique_stop_id': ['1_agency_a_city_a'], + 'unique_route_id': ['10-101_agency_a_city_a'], + 'node_id_route': ['1_agency_a_city_a_10-101_agency_a_city_a'] + } + index = range(1) + expected_subset = pd.DataFrame(data, index) + expected_subset = expected_subset.round(2) + return expected_subset + + +@pytest.fixture +def expected_route_stop_headway_result_sample_2(): + data = { + 'count': [11.0], + 'mean': [2.73], + 'std': [3.75], + 'min': [0.0], + '25%': [0.0], '50%': [0.0], '75%': [5.0], + 'max': [10.0], + 'unique_stop_id': ['7_agency_a_city_a'], + 'unique_route_id': ['B1_agency_a_city_a'], + 'node_id_route': ['7_agency_a_city_a_B1_agency_a_city_a'] + } + index = range(1) + expected_subset = pd.DataFrame(data, index) + expected_subset = expected_subset.round(2) + return expected_subset + + +@pytest.fixture +def expected_route_stop_ids(): + expected_route_stop_ids = sorted([ + '1_agency_a_city_a_10-101_agency_a_city_a', + '2_agency_a_city_a_10-101_agency_a_city_a', + '3_agency_a_city_a_10-101_agency_a_city_a', + '4_agency_a_city_a_10-101_agency_a_city_a', + '5_agency_a_city_a_10-101_agency_a_city_a', + '6_agency_a_city_a_10-101_agency_a_city_a', + '7_agency_a_city_a_B1_agency_a_city_a', + '8_agency_a_city_a_B1_agency_a_city_a', + '9_agency_a_city_a_B1_agency_a_city_a']) + return expected_route_stop_ids + + +def test_calc_headways_by_route_stop( + interm_stop_times_df_1_agency, + expected_route_stop_headway_result_sample_1, + expected_route_stop_headway_result_sample_2, + expected_route_stop_ids): + result = headways._calc_headways_by_route_stop( + df=interm_stop_times_df_1_agency) + assert isinstance(result, pd.core.frame.DataFrame) + assert result.empty is False + assert len(result) == 9 + + expected_cols = ['count', 'mean', 'std', 'min', '25%', '50%', '75%', 'max'] + for col in expected_cols: + assert col in result.columns + # check route stop ids + assert sorted(result.index.str.replace(',', '_').to_list()) == \ + expected_route_stop_ids + + # check subset of values + subset = result.loc[ + result.index == '1_agency_a_city_a,10-101_agency_a_city_a'] + subset.reset_index(inplace=True, drop=True) + subset = subset.round(2) + assert len(subset) == 1 + assert subset.equals( + expected_route_stop_headway_result_sample_1[expected_cols]) + + subset = result.loc[result.index == '7_agency_a_city_a,B1_agency_a_city_a'] + subset.reset_index(inplace=True, drop=True) + subset = subset.round(2) + assert len(subset) == 1 + assert subset.equals( + expected_route_stop_headway_result_sample_2[expected_cols]) + + +def test_headway_handeler( + hw_routes_df_1_agency, hw_stop_times_int_df_1_agency, + hw_trips_df_1_agency, + expected_route_stop_headway_result_sample_1, + expected_route_stop_headway_result_sample_2, + expected_route_stop_ids): + result = headways._headway_handler( + interpolated_stop_times_df=hw_stop_times_int_df_1_agency, + trips_df=hw_trips_df_1_agency, + routes_df=hw_routes_df_1_agency, + headway_timerange=['06:00:00', '12:00:00']) + + _check_headway_result(result, + hw_stop_times_int_df_1_agency, + expected_route_stop_ids, + expected_route_stop_headway_result_sample_1, + expected_route_stop_headway_result_sample_2) + + +def test_headways(ua_gtfsfeeds_df, + hw_routes_df_1_agency, hw_stop_times_int_df_1_agency, + hw_trips_df_1_agency, + expected_route_stop_headway_result_sample_1, + expected_route_stop_headway_result_sample_2, + expected_route_stop_ids): + ua_gtfsfeeds_df = headways.headways( + gtfsfeeds_df=ua_gtfsfeeds_df, + headway_timerange=['06:00:00', '12:00:00']) + # add expected cols to tables + hw_trips_df_1_agency['unique_trip_id'] = ( + hw_trips_df_1_agency['trip_id'].str.cat( + hw_trips_df_1_agency['unique_agency_id'].astype('str'), sep='_')) + hw_trips_df_1_agency['unique_route_id'] = ( + hw_trips_df_1_agency['route_id'].str.cat( + hw_trips_df_1_agency['unique_agency_id'].astype('str'), sep='_')) + hw_routes_df_1_agency['unique_route_id'] = ( + hw_routes_df_1_agency['route_id'].str.cat( + hw_routes_df_1_agency['unique_agency_id'].astype('str'), sep='_')) + + assert ua_gtfsfeeds_df.trips.equals(hw_trips_df_1_agency) + assert ua_gtfsfeeds_df.stop_times_int.equals(hw_stop_times_int_df_1_agency) + assert ua_gtfsfeeds_df.routes.equals(hw_routes_df_1_agency) + + _check_headway_result(ua_gtfsfeeds_df.headways, + hw_stop_times_int_df_1_agency, + expected_route_stop_ids, + expected_route_stop_headway_result_sample_1, + expected_route_stop_headway_result_sample_2) + + +def test_headways_invalid(ua_gtfsfeeds_df): + with pytest.raises(ValueError) as excinfo: + ua_gtfsfeeds_df.trips = pd.DataFrame() + ua_gtfsfeeds_df = headways.headways( + gtfsfeeds_df=ua_gtfsfeeds_df, + headway_timerange=['06:00:00', '12:00:00']) + expected_error = ( + 'One of the following gtfsfeeds_dfs objects: stop_times_int, ' + 'trips, or routes were found to be empty.') + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + ua_gtfsfeeds_df.trips = pd.DataFrame() + ua_gtfsfeeds_df = headways.headways( + gtfsfeeds_df=None, + headway_timerange=['06:00:00', '12:00:00']) + expected_error = 'gtfsfeeds_df cannot be None.' + assert expected_error in str(excinfo.value) + + +def test_add_unique_stop_route(): + data = { + 'unique_route_id': ['Route_A', 'Route_B'], + 'unique_stop_id': ['Stop_1', 'Stop_2'], + 'unique_stop_route': ['Stop_1,Route_A', 'Stop_2,Route_B'] + } + index = range(2) + df = pd.DataFrame(data, index) + + result = headways._add_unique_stop_route( + df[['unique_route_id', 'unique_stop_id']]) + assert result.equals(df) + + +def test_add_node_id_route(): + data = { + 'unique_route_id': ['Route_A', 'Route_B'], + 'unique_stop_id': ['Stop_1', 'Stop_2'], + 'node_id_route': ['Stop_1_Route_A', 'Stop_2_Route_B'] + } + index = range(2) + df = pd.DataFrame(data, index) + + result = headways._add_node_id_route( + df[['unique_route_id', 'unique_stop_id']]) + assert result.equals(df) diff --git a/urbanaccess/tests/test_gtfs_load.py b/urbanaccess/tests/test_gtfs_load.py index 492dbb4..87a28a3 100644 --- a/urbanaccess/tests/test_gtfs_load.py +++ b/urbanaccess/tests/test_gtfs_load.py @@ -229,3 +229,143 @@ def test_loadgtfsfeed_to_df_wo_agency( # check that df is not empty if key in expected_dfs: assert value.empty is False + + +def test_loadgtfsfeed_to_df_invalid_params( + agency_a_feed_on_disk_wo_req_file): + feed_dir = agency_a_feed_on_disk_wo_req_file + with pytest.raises(ValueError) as excinfo: + loaded_feeds = gtfs_load.gtfsfeed_to_df( + gtfsfeed_path=None, + validation=False, + verbose=True, + bbox=None, + remove_stops_outsidebbox=False, + append_definitions=False) + expected_error = "Directory: 'test\\gtfsfeed_text' does not exist." + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + non_existent_dir = os.path.join(feed_dir, 'test') + loaded_feeds = gtfs_load.gtfsfeed_to_df( + gtfsfeed_path=non_existent_dir, + validation=False, + verbose=True, + bbox=None, + remove_stops_outsidebbox=False, + append_definitions=False) + expected_error = "Directory: '{}' does not exist." + assert expected_error.format(non_existent_dir) in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + loaded_feeds = gtfs_load.gtfsfeed_to_df( + gtfsfeed_path=1, + validation=False, + verbose=True, + bbox=None, + remove_stops_outsidebbox=False, + append_definitions=False) + expected_error = 'gtfsfeed_path must be a string.' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + loaded_feeds = gtfs_load.gtfsfeed_to_df( + gtfsfeed_path=feed_dir, + validation=True, + verbose=True, + bbox=None, + remove_stops_outsidebbox=False, + append_definitions=False) + expected_error = ('Attempted to run validation but bbox, verbose, and or ' + 'remove_stops_outsidebbox were set to None. These ' + 'parameters must be specified for validation.') + assert expected_error in str(excinfo.value) + + +def test_loadgtfsfeed_to_df_w_validation_remove_stops_True_bbox_large( + agency_a_feed_on_disk_wo_calendar_dates, + expected_urbanaccess_gtfs_df_keys): + # test that all stops are retained given bbox size + feed_dir = agency_a_feed_on_disk_wo_calendar_dates + loaded_feeds = gtfs_load.gtfsfeed_to_df( + gtfsfeed_path=feed_dir, + validation=True, + verbose=True, + bbox=(-122.2850583493, 37.7947251937, -122.2644589841, 37.8131711247), + remove_stops_outsidebbox=True, + append_definitions=False) + assert isinstance(loaded_feeds, urbanaccess_gtfs_df) + urbanaccess_gtfs_df_info = vars(loaded_feeds) + expected_dfs = ['stops', 'routes', 'trips', 'stop_times', + 'calendar'] + assert expected_urbanaccess_gtfs_df_keys == sorted(list( + urbanaccess_gtfs_df_info.keys())) + for key, value in urbanaccess_gtfs_df_info.items(): + assert isinstance(value, pd.core.frame.DataFrame) + # check that df is not empty + if key in expected_dfs: + assert value.empty is False + + +def test_loadgtfsfeed_to_df_w_validation_remove_stops_True_bbox_small( + agency_a_feed_on_disk_wo_calendar_dates, + expected_urbanaccess_gtfs_df_keys): + # test that specific stops are removed given bbox size + feed_dir = agency_a_feed_on_disk_wo_calendar_dates + loaded_feeds = gtfs_load.gtfsfeed_to_df( + gtfsfeed_path=feed_dir, + validation=True, + verbose=True, + bbox=(-122.2702525556, 37.7992692881, -122.2625707089, 37.8036435628), + remove_stops_outsidebbox=True, + append_definitions=False) + assert isinstance(loaded_feeds, urbanaccess_gtfs_df) + urbanaccess_gtfs_df_info = vars(loaded_feeds) + expected_dfs_not_null = ['routes', 'trips', 'calendar'] + expected_dfs_null = ['stops', 'stop_times'] + assert expected_urbanaccess_gtfs_df_keys == sorted(list( + urbanaccess_gtfs_df_info.keys())) + for key, value in urbanaccess_gtfs_df_info.items(): + assert isinstance(value, pd.core.frame.DataFrame) + # check that df is not empty + if key in expected_dfs_not_null: + assert value.empty is False + for key, value in urbanaccess_gtfs_df_info.items(): + assert isinstance(value, pd.core.frame.DataFrame) + # check that df is empty to make sure bbox stop removal operation + # was successful + if key in expected_dfs_null: + assert value.empty is True + + +def test_loadgtfsfeed_to_df_w_append_definitions_True( + agency_a_feed_on_disk_wo_calendar_dates, + expected_urbanaccess_gtfs_df_keys): + feed_dir = agency_a_feed_on_disk_wo_calendar_dates + loaded_feeds = gtfs_load.gtfsfeed_to_df( + gtfsfeed_path=feed_dir, + validation=False, + verbose=True, + bbox=None, + remove_stops_outsidebbox=False, + append_definitions=True) + assert isinstance(loaded_feeds, urbanaccess_gtfs_df) + urbanaccess_gtfs_df_info = vars(loaded_feeds) + expected_dfs = ['stops', 'routes', 'trips', 'stop_times', + 'calendar'] + expected_attr_cols_dict = {'stops': ['wheelchair_boarding_desc', + 'location_type_desc'], + 'routes': ['route_type_desc'], + 'stop_times': ['pickup_type_desc', + 'drop_off_type_desc'], + 'trips': ['bikes_allowed_desc', + 'wheelchair_accessible_desc']} + + assert expected_urbanaccess_gtfs_df_keys == sorted(list( + urbanaccess_gtfs_df_info.keys())) + for table, df in urbanaccess_gtfs_df_info.items(): + assert isinstance(df, pd.core.frame.DataFrame) + # check that df is not empty + if table in expected_dfs: + assert df.empty is False + # check that attr col was added to table + if table in expected_attr_cols_dict.keys(): + for col in expected_attr_cols_dict[table]: + assert col in df.columns diff --git a/urbanaccess/tests/test_gtfs_network.py b/urbanaccess/tests/test_gtfs_network.py index 9c4f06d..6f5d97e 100644 --- a/urbanaccess/tests/test_gtfs_network.py +++ b/urbanaccess/tests/test_gtfs_network.py @@ -9,6 +9,7 @@ import urbanaccess.gtfs.load as gtfs_load from urbanaccess.network import urbanaccess_network from urbanaccess.gtfs.gtfsfeeds_dataframe import urbanaccess_gtfs_df +from urbanaccess import config @pytest.fixture @@ -65,6 +66,19 @@ def gtfs_feed_w_calendar_and_calendar_dates( return loaded_feeds +@pytest.fixture +def gtfs_feed_w_calendar_and_calendar_dates_multi_agency( + agency_b_feed_on_disk_w_calendar_and_calendar_dates): + loaded_feeds = gtfs_load.gtfsfeed_to_df( + gtfsfeed_path=agency_b_feed_on_disk_w_calendar_and_calendar_dates, + validation=False, + verbose=True, + bbox=None, + remove_stops_outsidebbox=False, + append_definitions=False) + return loaded_feeds + + @pytest.fixture def selected_int_stop_times_from_feed_wo_calendar_dates( gtfs_feed_wo_calendar_dates): @@ -308,6 +322,76 @@ def expected_transit_edge_from_feed_wo_calendar_dates_process_lvl_2_timeaware(): return df +@pytest.fixture +def expected_final_transit_edge_from_feed_wo_calendar(): + # represents df after it has been post-processed downstream + data = { + 'node_id_from': ['6_agency_a_city_a', '5_agency_a_city_a', + '4_agency_a_city_a', '3_agency_a_city_a'], + 'node_id_to': ['5_agency_a_city_a', '4_agency_a_city_a', + '3_agency_a_city_a', '1_agency_a_city_a'], + 'weight': [5.0, 5.0, 5.0, 10.0], + 'unique_agency_id': ['agency_a_city_a'] * 4, + 'unique_trip_id': ['c2_agency_a_city_a'] * 4, + 'sequence': range(1, 5), + 'unique_route_id': ['12-101_agency_a_city_a'] * 4, + 'id': ['c2_agency_a_city_a_1', 'c2_agency_a_city_a_2', + 'c2_agency_a_city_a_3', 'c2_agency_a_city_a_4'], + 'route_type': [1] * 4, + 'net_type': ['transit'] * 4 + } + index = range(4) + df = pd.DataFrame(data, index) + # raw data are read as int32 + df['sequence'] = df['sequence'].astype('int32') + return df + + +@pytest.fixture +def expected_final_transit_edge_from_feed_w_cal_and_cal_dates( + expected_transit_edge_from_feed_wo_calendar_dates_process_lvl_2): + # represents df after it has been post-processed downstream + data = { + 'unique_route_id': ['10-101_agency_a_city_a'] * 5, + 'net_type': ['transit'] * 5 + } + index = range(5) + df = pd.DataFrame(data, index) + df = pd.concat( + [expected_transit_edge_from_feed_wo_calendar_dates_process_lvl_2, + df], + axis=1) + return df + + +@pytest.fixture +def expected_transit_edge_from_agency_b_feed_w_cal_and_cal_dates(): + # represents df after it has been post-processed downstream + data = { + 'node_id_from': ['60_agency_b_district_1', '61_agency_b_district_1', + '62_agency_b_district_1', '63_agency_b_district_1', + '64_agency_b_district_1'], + 'node_id_to': ['61_agency_b_district_1', '62_agency_b_district_1', + '63_agency_b_district_1', '64_agency_b_district_1', + '65_agency_b_district_1'], + 'weight': [5.0] * 5, + 'unique_agency_id': ['agency_b_district_1'] * 5, + 'unique_trip_id': ['13_agency_b_district_1'] * 5, + 'sequence': range(1, 6), + 'unique_route_id': ['40-4_agency_b_district_1'] * 5, + 'id': ['13_agency_b_district_1_1', '13_agency_b_district_1_2', + '13_agency_b_district_1_3', '13_agency_b_district_1_4', + '13_agency_b_district_1_5'], + 'route_type': [3] * 5, + 'net_type': ['transit'] * 5 + } + index = range(5) + df = pd.DataFrame(data, index) + # raw data are read as int32 + df['sequence'] = df['sequence'].astype('int32') + return df + + @pytest.fixture def expected_transit_edge_from_feed_wo_calendar_dates_process_lvl_1_timeaware( expected_transit_edge_from_feed_wo_calendar_dates_process_lvl_2_timeaware): # noqa @@ -439,6 +523,160 @@ def hdf5_file_on_disk_gtfsfeeds_dfs( return hdf5_save_path +@pytest.fixture +def edges_empty(transit_edges_to_simplify): + df_empty = transit_edges_to_simplify[0:0] + return df_empty + + +@pytest.fixture +def transit_edges_to_simplify(): + data = { + 'node_id_from': [ + '1_agency_a_city_a', '2_agency_a_city_a', '3_agency_a_city_a', + '4_agency_a_city_a', '3_agency_a_city_a', '2_agency_a_city_a', + '5_agency_a_city_a', '6_agency_a_city_a', '7_agency_a_city_a', + '5_agency_a_city_a', '6_agency_a_city_a', '7_agency_a_city_a', + '8_agency_a_city_a', '9_agency_a_city_a', '10_agency_a_city_a', + '8_agency_a_city_a', '9_agency_a_city_a', '10_agency_a_city_a' + ], + 'node_id_to': [ + '2_agency_a_city_a', '3_agency_a_city_a', '4_agency_a_city_a', + '3_agency_a_city_a', '2_agency_a_city_a', '1_agency_a_city_a', + '6_agency_a_city_a', '7_agency_a_city_a', '8_agency_a_city_a', + '6_agency_a_city_a', '7_agency_a_city_a', '8_agency_a_city_a', + '9_agency_a_city_a', '10_agency_a_city_a', '11_agency_a_city_a', + '9_agency_a_city_a', '10_agency_a_city_a', '11_agency_a_city_a'], + 'weight': [2.0, 4.0, 5.0, + 5.0, 4.0, 2.0, + 2.0, 2.0, 3.0, + 2.0, 2.0, 3.0, + 4.0, 3.0, 4.0, + 4.0, 3.0, 4.0], + 'unique_agency_id': ['agency_a_city_a'] * 18, + 'unique_trip_id': [ + '1_agency_a_city_a', '1_agency_a_city_a', '1_agency_a_city_a', + '2_agency_a_city_a', '2_agency_a_city_a', '2_agency_a_city_a', + '3_agency_a_city_a', '3_agency_a_city_a', '3_agency_a_city_a', + '4_agency_a_city_a', '4_agency_a_city_a', '4_agency_a_city_a', + '5_agency_a_city_a', '5_agency_a_city_a', '5_agency_a_city_a', + '6_agency_a_city_a', '6_agency_a_city_a', '6_agency_a_city_a'], + 'sequence': [1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3], + 'unique_route_id': [ + 'R1_agency_a_city_a', 'R1_agency_a_city_a', 'R1_agency_a_city_a', + 'R1_agency_a_city_a', 'R1_agency_a_city_a', 'R1_agency_a_city_a', + 'A1_agency_a_city_a', 'A1_agency_a_city_a', 'A1_agency_a_city_a', + 'A1_agency_a_city_a', 'A1_agency_a_city_a', 'A1_agency_a_city_a', + 'B1_agency_a_city_a', 'B1_agency_a_city_a', 'B1_agency_a_city_a', + 'C1_agency_a_city_a', 'C1_agency_a_city_a', 'C1_agency_a_city_a'], + 'route_type': [1] * 18, + 'net_type': ['transit'] * 18 + } + index = range(18) + df = pd.DataFrame(data, index) + df['id'] = (df['unique_trip_id'].str.cat( + df['sequence'].astype('str'), sep='_')) + return df + + +@pytest.fixture +def transit_nodes_to_simplify(): + data = { + 'node_id': [ + '1_agency_a_city_a', '2_agency_a_city_a', '3_agency_a_city_a', + '4_agency_a_city_a', '5_agency_a_city_a', '6_agency_a_city_a', + '7_agency_a_city_a', '8_agency_a_city_a', '9_agency_a_city_a', + '10_agency_a_city_a', '11_agency_a_city_a'], + 'x': [-122.268962, -122.273627, -122.276709, -122.272114, + -122.266589, -122.268842, -122.271760, -122.265859, + -122.274823, -122.271057, -122.266551], + 'y': [37.807730, 37.800201, 37.795430, 37.793342, + 37.806846, 37.803413, 37.798581, 37.796250, + 37.805727, 37.804455, 37.802497], + 'unique_agency_id': ['agency_a_city_a'] * 11, + 'route_type': [1] * 11, + 'stop_id': range(1, 12), + 'stop_name': [ + 'ave a', 'ave b', 'ave c', 'ave d', 'ave 1', 'ave 2', + 'ave 3', 'ave 4', 'st a', 'st b', 'st c'], + 'net_type': ['transit'] * 11 + } + index = range(11) + df = pd.DataFrame(data, index) + df.set_index('node_id', drop=True, inplace=True) + return df + + +@pytest.fixture +def transit_nodes_to_simplify_extra_nodes(transit_nodes_to_simplify): + data = { + 'node_id': ['12_agency_a_city_a'], + 'x': [-122.266551], + 'y': [37.802497], + 'unique_agency_id': ['agency_a_city_a'], + 'route_type': [1], + 'stop_id': [13], + 'stop_name': ['st d'], + 'net_type': ['transit'] + } + index = range(1) + df = pd.DataFrame(data, index) + df.set_index('node_id', drop=True, inplace=True) + df = pd.concat([transit_nodes_to_simplify, df], axis=1) + + return df + + +@pytest.fixture +def stop_times_empty(selected_int_stop_times_from_feed_wo_calendar_dates): + df_empty = selected_int_stop_times_from_feed_wo_calendar_dates[0:0] + return df_empty + + +def test_create_transit_net_wo_calendar_dates_w_high_freq_trips( + gtfs_feed_wo_calendar_dates, + expected_urbanaccess_network_keys, + expected_final_transit_edge_from_feed_wo_calendar_dates): + expected_result = \ + expected_final_transit_edge_from_feed_wo_calendar_dates.copy() + transit_net = gtfs_network.create_transit_net( + gtfs_feed_wo_calendar_dates, day=None, + timerange=['07:00:00', '10:00:00'], + calendar_dates_lookup=None, + overwrite_existing_stop_times_int=False, + use_existing_stop_times_int=False, + save_processed_gtfs=False, + save_dir=None, + save_filename=None, + timerange_pad=None, + time_aware=False, + date=None, + date_range=None, + use_highest_freq_trips_date=True, + simplify=False) + assert isinstance(transit_net, urbanaccess_network) + urbanaccess_network_info = vars(transit_net) + expected_dfs = ['transit_nodes', 'transit_edges'] + assert expected_urbanaccess_network_keys == sorted(list( + urbanaccess_network_info.keys())) + for key, value in urbanaccess_network_info.items(): + assert isinstance(value, pd.core.frame.DataFrame) + # check that df is not empty + if key in expected_dfs: + assert value.empty is False + + result_edge = transit_net.transit_edges.copy() + # test that output df is identical to expected df + result_edge = result_edge.reindex( + sorted(result_edge.columns), axis=1) + expected_result = expected_result.reindex( + sorted(expected_result.columns), axis=1) + # ensure 'sequence' is int32 for test as other OS sometimes reads this as + # int64 and will cause tests to fail when using equals() + result_edge['sequence'] = result_edge['sequence'].astype('int32') + assert result_edge.equals(expected_result) + + def test_create_transit_net_wo_calendar_dates( gtfs_feed_wo_calendar_dates, expected_urbanaccess_network_keys, @@ -477,6 +715,166 @@ def test_create_transit_net_wo_calendar_dates( assert result_edge.equals(expected_result) +def test_create_transit_net_wo_calendar( + gtfs_feed_wo_calendar, + expected_urbanaccess_network_keys, + expected_final_transit_edge_from_feed_wo_calendar +): + expected_result = \ + expected_final_transit_edge_from_feed_wo_calendar.copy() + transit_net = gtfs_network.create_transit_net( + gtfs_feed_wo_calendar, day=None, + timerange=['00:00:00', '23:59:00'], + calendar_dates_lookup=None, + overwrite_existing_stop_times_int=False, + use_existing_stop_times_int=False, + save_processed_gtfs=False, + save_dir=None, + save_filename=None, + date='2016-04-24') + assert isinstance(transit_net, urbanaccess_network) + urbanaccess_network_info = vars(transit_net) + expected_dfs = ['transit_nodes', 'transit_edges'] + assert expected_urbanaccess_network_keys == sorted(list( + urbanaccess_network_info.keys())) + for key, value in urbanaccess_network_info.items(): + assert isinstance(value, pd.core.frame.DataFrame) + # check that df is not empty + if key in expected_dfs: + assert value.empty is False + + result_edge = transit_net.transit_edges.copy() + # test that output df is identical to expected df + result_edge = result_edge.reindex( + sorted(result_edge.columns), axis=1) + expected_result = expected_result.reindex( + sorted(expected_result.columns), axis=1) + # ensure 'sequence' is int32 for test as other OS sometimes reads this as + # int64 and will cause tests to fail when using equals() + result_edge['sequence'] = result_edge['sequence'].astype('int32') + assert result_edge.equals(expected_result) + + +def test_create_transit_net_w_calendar_and_calendar_dates( + gtfs_feed_w_calendar_and_calendar_dates, + expected_urbanaccess_network_keys, + expected_final_transit_edge_from_feed_w_cal_and_cal_dates): + expected_result = \ + expected_final_transit_edge_from_feed_w_cal_and_cal_dates.copy() + transit_net = gtfs_network.create_transit_net( + gtfs_feed_w_calendar_and_calendar_dates, day=None, + timerange=['07:00:00', '10:00:00'], + calendar_dates_lookup=None, + overwrite_existing_stop_times_int=False, + use_existing_stop_times_int=False, + save_processed_gtfs=False, + save_dir=None, + save_filename=None, + date_range=['2016-12-24', '2017-03-18']) + assert isinstance(transit_net, urbanaccess_network) + urbanaccess_network_info = vars(transit_net) + expected_dfs = ['transit_nodes', 'transit_edges'] + assert expected_urbanaccess_network_keys == sorted(list( + urbanaccess_network_info.keys())) + for key, value in urbanaccess_network_info.items(): + assert isinstance(value, pd.core.frame.DataFrame) + # check that df is not empty + if key in expected_dfs: + assert value.empty is False + + result_edge = transit_net.transit_edges.copy() + # test that output df is identical to expected df + result_edge = result_edge.reindex( + sorted(result_edge.columns), axis=1) + expected_result = expected_result.reindex( + sorted(expected_result.columns), axis=1) + # ensure 'sequence' is int32 for test as other OS sometimes reads this as + # int64 and will cause tests to fail when using equals() + result_edge['sequence'] = result_edge['sequence'].astype('int32') + assert result_edge.equals(expected_result) + + +def test_create_transit_net_w_calendar_and_calendar_dates_w_simplify( + gtfs_feed_w_calendar_and_calendar_dates, + expected_urbanaccess_network_keys, + expected_final_transit_edge_from_feed_w_cal_and_cal_dates): + # TODO: note that gtfs_feed_w_calendar_and_calendar_dates does not + # result in any edges that require simplification. Test can be improved + # by changing the test data to one that can be simplified. + expected_result = \ + expected_final_transit_edge_from_feed_w_cal_and_cal_dates.copy() + transit_net = gtfs_network.create_transit_net( + gtfs_feed_w_calendar_and_calendar_dates, day=None, + timerange=['07:00:00', '10:00:00'], + calendar_dates_lookup=None, + overwrite_existing_stop_times_int=False, + use_existing_stop_times_int=False, + save_processed_gtfs=False, + save_dir=None, + save_filename=None, + date_range=['2016-12-24', '2017-03-18'], + simplify=True) + assert isinstance(transit_net, urbanaccess_network) + urbanaccess_network_info = vars(transit_net) + expected_dfs = ['transit_nodes', 'transit_edges'] + assert expected_urbanaccess_network_keys == sorted(list( + urbanaccess_network_info.keys())) + for key, value in urbanaccess_network_info.items(): + assert isinstance(value, pd.core.frame.DataFrame) + # check that df is not empty + if key in expected_dfs: + assert value.empty is False + + result_edge = transit_net.transit_edges.copy() + # test that output df is identical to expected df + result_edge = result_edge.reindex( + sorted(result_edge.columns), axis=1) + expected_result = expected_result.reindex( + sorted(expected_result.columns), axis=1) + # ensure 'sequence' is int32 for test as other OS sometimes reads this as + # int64 and will cause tests to fail when using equals() + result_edge['sequence'] = result_edge['sequence'].astype('int32') + assert result_edge.equals(expected_result) + + +def test_create_transit_net_w_calendar_and_calendar_dates_multi_agency( + gtfs_feed_w_calendar_and_calendar_dates_multi_agency, + expected_urbanaccess_network_keys, + expected_transit_edge_from_agency_b_feed_w_cal_and_cal_dates): + expected_result = \ + expected_transit_edge_from_agency_b_feed_w_cal_and_cal_dates.copy() + transit_net = gtfs_network.create_transit_net( + gtfs_feed_w_calendar_and_calendar_dates_multi_agency, day='monday', + timerange=['07:00:00', '10:00:00'], + calendar_dates_lookup=None, + overwrite_existing_stop_times_int=False, + use_existing_stop_times_int=False, + save_processed_gtfs=False, + save_dir=None, + save_filename=None) + assert isinstance(transit_net, urbanaccess_network) + urbanaccess_network_info = vars(transit_net) + expected_dfs = ['transit_nodes', 'transit_edges'] + assert expected_urbanaccess_network_keys == sorted(list( + urbanaccess_network_info.keys())) + for key, value in urbanaccess_network_info.items(): + assert isinstance(value, pd.core.frame.DataFrame) + # check that df is not empty + if key in expected_dfs: + assert value.empty is False + + result_edge = transit_net.transit_edges.copy() + # test that output df is identical to expected df + result_edge = result_edge.reindex( + sorted(result_edge.columns), axis=1) + expected_result = expected_result.reindex( + sorted(expected_result.columns), axis=1) + # ensure 'sequence' is int32 for test as other OS sometimes reads this as + # int64 and will cause tests to fail when using equals() + result_edge['sequence'] = result_edge['sequence'].astype('int32') + assert result_edge.equals(expected_result) + + def test_create_transit_net_wo_calendar_dates_timepad( gtfs_feed_wo_calendar_dates, expected_urbanaccess_network_keys, @@ -523,7 +921,8 @@ def test_create_transit_net_wo_calendar_dates_timeaware( expected_urbanaccess_network_keys, expected_final_transit_edge_from_feed_wo_calendar_dates_timeaware): expected_result = \ - expected_final_transit_edge_from_feed_wo_calendar_dates_timeaware.copy() # noqa + expected_final_transit_edge_from_feed_wo_calendar_dates_timeaware\ + .copy() # noqa transit_net = gtfs_network.create_transit_net( gtfs_feed_wo_calendar_dates, day='monday', timerange=['07:00:00', '10:00:00'], @@ -768,6 +1167,27 @@ def test_create_transit_net_invalid_params(gtfs_feed_wo_calendar_dates): expected_error = ('overwrite_existing_stop_times_int and ' 'use_existing_stop_times_int cannot both be True.') assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + transit_net = gtfs_network.create_transit_net( + gtfs_feed_wo_calendar_dates, day='monday', + timerange=['07:00:00', '10:00:00'], + calendar_dates_lookup=None, + overwrite_existing_stop_times_int=False, + use_existing_stop_times_int=False, + save_processed_gtfs=False, + save_dir=None, + save_filename=None, + timerange_pad=None, + time_aware=False, + date=None, + date_range=None, + use_highest_freq_trips_date=True, + simplify=False) + expected_error = ("Only one parameter: 'use_highest_freq_trips_date' " + "or one of 'day', 'date', or 'date_range' can be " + "used at a time or both 'day' and 'date_range' can " + "be used.") + assert expected_error in str(excinfo.value) with pytest.raises(ValueError) as excinfo: transit_net = gtfs_network.create_transit_net( gtfs_feed_wo_calendar_dates, day='monday', @@ -980,240 +1400,6 @@ def test_skip_interpolator(stop_times, calendar): 1, 2, 3, 4, 5] -def test_trip_schedule_selector_wo_cal_dates(gtfs_feed_wo_calendar_dates): - expected_result = gtfs_feed_wo_calendar_dates.trips.copy() - # create expected trips result - expected_result.reset_index(drop=True, inplace=True) - expected_result = expected_result.iloc[0:8] - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed_wo_calendar_dates.trips, - input_calendar_df=gtfs_feed_wo_calendar_dates.calendar, - input_calendar_dates_df=gtfs_feed_wo_calendar_dates.calendar_dates, - day='monday', - calendar_dates_lookup=None) - - assert len(result) == 8 - assert result.equals(expected_result) - - -def test_trip_schedule_selector_wo_cal_dates_wo_direction_id( - gtfs_feed_wo_calendar_dates): - # remove 'direction_id' col for test - trips_df = gtfs_feed_wo_calendar_dates.trips.copy() - trips_df.drop(columns=['direction_id'], inplace=True) - expected_result = gtfs_feed_wo_calendar_dates.trips.copy() - # create expected trips result - expected_result.reset_index(drop=True, inplace=True) - expected_result.drop(columns=['direction_id'], inplace=True) - expected_result = expected_result.iloc[0:8] - - result = gtfs_network._trip_schedule_selector( - input_trips_df=trips_df, - input_calendar_df=gtfs_feed_wo_calendar_dates.calendar, - input_calendar_dates_df=gtfs_feed_wo_calendar_dates.calendar_dates, - day='monday', - calendar_dates_lookup=None) - - assert len(result) == 8 - assert result.equals(expected_result) - - -def test_trip_schedule_selector_w_cal_dates(gtfs_feed_wo_calendar): - expected_result = gtfs_feed_wo_calendar.trips.copy() - # create expected trips result - expected_result = expected_result.iloc[4:10] - expected_result.reset_index(drop=True, inplace=True) - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed_wo_calendar.trips, - input_calendar_df=gtfs_feed_wo_calendar.calendar, - input_calendar_dates_df=gtfs_feed_wo_calendar.calendar_dates, - day='sunday', - calendar_dates_lookup={'schedule_type': 'WE', - 'service_id': ['weekday-3', 'weekday-2']}) - - assert len(result) == 6 - assert result.equals(expected_result) - - -def test_trip_schedule_selector_w_cal_and_cal_dates( - gtfs_feed_w_calendar_and_calendar_dates): - trips_df = gtfs_feed_w_calendar_and_calendar_dates.trips.copy() - cal_df = gtfs_feed_w_calendar_and_calendar_dates.calendar.copy() - cal_dates_df = gtfs_feed_w_calendar_and_calendar_dates.calendar_dates \ - .copy() - expected_result = gtfs_feed_w_calendar_and_calendar_dates.trips.copy() - result = gtfs_network._trip_schedule_selector( - input_trips_df=trips_df, - input_calendar_df=cal_df, - input_calendar_dates_df=cal_dates_df, - day='monday', - calendar_dates_lookup={'schedule_type': 'WE'}) - - assert len(result) == 10 - assert result.equals(expected_result) - - -def test_trip_schedule_selector_w_cal_and_cal_dates_wo_lookup( - gtfs_feed_w_calendar_and_calendar_dates): - trips_df_1 = gtfs_feed_w_calendar_and_calendar_dates.trips.copy() - cal_df = gtfs_feed_w_calendar_and_calendar_dates.calendar.copy() - cal_dates_df_1 = gtfs_feed_w_calendar_and_calendar_dates.calendar_dates \ - .copy() - # create extra records in trips and calendar_dates for a different agency - # that do not exist in the calendar table - trips_df_2 = trips_df_1.copy() - trips_df_2['unique_agency_id'] = trips_df_2['unique_agency_id'] + '_x' - trips_df_2['unique_feed_id'] = trips_df_2['unique_feed_id'] + '_x' - trips_df_2 = trips_df_2.iloc[0:8] - trips_df_x2 = pd.concat( - [trips_df_1, trips_df_2], axis=0, - ignore_index=True) - cal_dates_df_2 = cal_dates_df_1.copy() - cal_dates_df_2['unique_agency_id'] = \ - cal_dates_df_2['unique_agency_id'] + '_x' - cal_dates_df_2['unique_feed_id'] = \ - cal_dates_df_2['unique_feed_id'] + '_x' - cal_dates_df_x2 = pd.concat( - [cal_dates_df_1, cal_dates_df_2], axis=0, - ignore_index=True) - # create expected trips result - expected_result = trips_df_1.copy() - expected_result = expected_result.iloc[0:8] - result = gtfs_network._trip_schedule_selector( - input_trips_df=trips_df_x2, - input_calendar_df=cal_df, - input_calendar_dates_df=cal_dates_df_x2, - day='monday', - calendar_dates_lookup=None) - - assert len(result) == 8 - assert result.equals(expected_result) - - -def test_trip_schedule_selector_wo_cal_dates_invalid_params( - gtfs_feed_wo_calendar_dates): - gtfs_feed = gtfs_feed_wo_calendar_dates - # test with invalid 'day' param - with pytest.raises(ValueError) as excinfo: - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed.trips, - input_calendar_df=gtfs_feed.calendar, - input_calendar_dates_df=gtfs_feed.calendar_dates, - day='monday ', - calendar_dates_lookup=None) - expected_error = ( - "Incorrect day specified. Must be one of lowercase strings: 'monday'," - " 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'.") - assert expected_error in str(excinfo.value) - # test with invalid 'calendar_dates_lookup' param - with pytest.raises(ValueError) as excinfo: - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed.trips, - input_calendar_df=gtfs_feed.calendar, - input_calendar_dates_df=gtfs_feed.calendar_dates, - day='monday', - calendar_dates_lookup=['invalid']) - expected_error = "calendar_dates_lookup parameter must be a dictionary." - assert expected_error in str(excinfo.value) - # test with invalid 'calendar_dates_lookup' param - with pytest.raises(ValueError) as excinfo: - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed.trips, - input_calendar_df=gtfs_feed.calendar, - input_calendar_dates_df=gtfs_feed.calendar_dates, - day='monday', - calendar_dates_lookup={1: 'WD'}) - expected_error = "calendar_dates_lookup key: 1 must be a string." - assert expected_error in str(excinfo.value) - # test with invalid 'calendar_dates_lookup' param - with pytest.raises(ValueError) as excinfo: - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed.trips, - input_calendar_df=gtfs_feed.calendar, - input_calendar_dates_df=gtfs_feed.calendar_dates, - day='monday', - calendar_dates_lookup={'schedule_type': 1}) - expected_error = ("calendar_dates_lookup value: 1 must be a " - "string or a list of strings.") - assert expected_error in str(excinfo.value) - # test with invalid 'calendar_dates_lookup' param - with pytest.raises(ValueError) as excinfo: - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed.trips, - input_calendar_df=gtfs_feed.calendar, - input_calendar_dates_df=gtfs_feed.calendar_dates, - day='monday', - calendar_dates_lookup={'schedule_type': ['WD', 1]}) - expected_error = ("calendar_dates_lookup value: ['WD', 1] " - "must contain strings.") - assert expected_error in str(excinfo.value) - - -def test_trip_schedule_selector_no_services_found( - gtfs_feed_wo_calendar): - with pytest.raises(ValueError) as excinfo: - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed_wo_calendar.trips, - input_calendar_df=gtfs_feed_wo_calendar.calendar, - input_calendar_dates_df=gtfs_feed_wo_calendar.calendar_dates, - day='monday', - calendar_dates_lookup=None) - expected_error = ("No service_id(s) were found with the specified " - "calendar and or calendar_dates search parameters.") - assert expected_error in str(excinfo.value) - - -def test_trip_schedule_selector_w_cal_dates_invalid_params_1( - gtfs_feed_wo_calendar_dates): - # test with empty 'calendar_dates'df - with pytest.raises(ValueError) as excinfo: - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed_wo_calendar_dates.trips, - input_calendar_df=gtfs_feed_wo_calendar_dates.calendar, - input_calendar_dates_df=gtfs_feed_wo_calendar_dates.calendar_dates, - day='monday', - calendar_dates_lookup={'schedule_type': 'WD'}) - expected_error = ("calendar_dates is empty. Unable to use the " - "'calendar_dates_lookup' parameter. Set to None.") - assert expected_error in str(excinfo.value) - - -def test_trip_schedule_selector_w_cal_dates_invalid_params_2( - gtfs_feed_wo_calendar): - # create invalid data in calendar dates file - cal_dates_df = gtfs_feed_wo_calendar.calendar_dates.copy() - series = pd.Series( - data=[1, 1, 2, 2], index=range(4), - name='invalid_dtype') - cal_dates_df['invalid_dtype'] = series - series = pd.Series( - data=[10, 11, 10, 'aa'], index=range(4), - name='day_type') - cal_dates_df['day_type'] = series - - # test with invalid col in 'calendar_dates_lookup' param - with pytest.raises(ValueError) as excinfo: - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed_wo_calendar.trips, - input_calendar_df=gtfs_feed_wo_calendar.calendar, - input_calendar_dates_df=cal_dates_df, - day='monday', - calendar_dates_lookup={'invalid_col': 'WD'}) - expected_error = ("Column: invalid_col not found in calendar_dates " - "DataFrame.") - assert expected_error in str(excinfo.value) - # test with invalid col dtype in 'calendar_dates_lookup' param - with pytest.raises(ValueError) as excinfo: - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed_wo_calendar.trips, - input_calendar_df=gtfs_feed_wo_calendar.calendar, - input_calendar_dates_df=cal_dates_df, - day='monday', - calendar_dates_lookup={'invalid_dtype': '1'}) - expected_error = "Column: invalid_dtype must be object type." - assert expected_error in str(excinfo.value) - - def test_time_selector_wo_timerange_pad( selected_int_stop_times_from_feed_wo_calendar_dates): timerange = ['08:20:00', '08:35:00'] @@ -1390,6 +1576,15 @@ def test_format_transit_net_edge_timeaware_True( assert result.equals(expected_result) +def test_format_transit_net_edge_empty_stop_times(stop_times_empty): + with pytest.raises(ValueError) as excinfo: + result = gtfs_network._format_transit_net_edge(stop_times_empty) + expected_error = ( + "Unable to continue processing transit network. stop_times table " + "has 0 records.") + assert expected_error in str(excinfo.value) + + def test_convert_imp_time_units( transit_edge_from_feed_wo_calendar_dates): # test with minutes @@ -1607,6 +1802,17 @@ def test_edge_impedance_by_route_type_invalid_params( funicular=-0.5) # should return a result even if route type is not found in DataFrame assert result.empty is False + with pytest.raises(ValueError) as excinfo: + config._ROUTES_MODE_TYPE_LOOKUP.update({13: 'test mode'}) + result = gtfs_network.edge_impedance_by_route_type( + edge_route_type_impedance_df, + underground_rail='1', + intercity_rail=-0.5) + expected_error = ("ROUTES_MODE_TYPE_LOOKUP keys do not match keys in var_" + "mode_id_lookup. Keys must match.") + assert expected_error in str(excinfo.value) + # reset config removing the test key + del config._ROUTES_MODE_TYPE_LOOKUP[13] def test_save_processed_gtfs_data( @@ -1702,3 +1908,122 @@ def test_load_processed_gtfs_data( # check that df is empty if key in expected_dfs_empty: assert value.empty + + +def test_simplify_transit_net( + transit_edges_to_simplify, transit_nodes_to_simplify): + remove_edge_ids = ['4_agency_a_city_a_1', '4_agency_a_city_a_2', + '4_agency_a_city_a_3'] + expected_edges = transit_edges_to_simplify.loc[ + ~transit_edges_to_simplify['id'].isin(remove_edge_ids)] + expected_edges.reset_index(inplace=True, drop=True) + result_edges, result_nodes = gtfs_network._simplify_transit_net( + transit_edges_to_simplify, transit_nodes_to_simplify) + + # expect all nodes to persist + assert len(result_nodes) == 11 + assert result_nodes.equals(transit_nodes_to_simplify) + + # expect 5_agency_a_city_a_1 to 3 and 6_agency_a_city_a_1 to 3 to + # persist since they have different attributes in unique_route_id col + # expect 1_agency_a_city_a_1 to 3 and 2_agency_a_city_a_1 to 3 to + # persist since node_id_from and node_id_to are different + # expect only one of 3_agency_a_city_a_1 to 3 or 4_agency_a_city_a_1 to 3 + # to persist but not both since they share the same attributes + assert len(result_edges) == 15 + result_edges.reset_index(inplace=True, drop=True) + assert result_edges.equals(expected_edges) + + +def test_simplify_transit_net_already_simplified( + transit_edges_to_simplify, transit_nodes_to_simplify): + remove_edge_ids = ['4_agency_a_city_a_1', '4_agency_a_city_a_2', + '4_agency_a_city_a_3'] + expected_edges = transit_edges_to_simplify.loc[ + ~transit_edges_to_simplify['id'].isin(remove_edge_ids)] + expected_edges.reset_index(inplace=True, drop=True) + + # use the simplified edge table as input to test + result_edges, result_nodes = gtfs_network._simplify_transit_net( + expected_edges, transit_nodes_to_simplify) + + # expect all nodes to persist + assert len(result_nodes) == 11 + assert result_nodes.equals(transit_nodes_to_simplify) + + # expect 5_agency_a_city_a_1 to 3 and 6_agency_a_city_a_1 to 3 to + # persist since they have different attributes in unique_route_id col + # expect 1_agency_a_city_a_1 to 3 and 2_agency_a_city_a_1 to 3 to + # persist since node_id_from and node_id_to are different + # expect only one of 3_agency_a_city_a_1 to 3 or 4_agency_a_city_a_1 to 3 + # to persist but not both since they share the same attributes + assert len(result_edges) == 15 + result_edges.reset_index(inplace=True, drop=True) + assert result_edges.equals(expected_edges) + + +def test_simplify_transit_net_remove_extra_nodes( + transit_edges_to_simplify, transit_nodes_to_simplify_extra_nodes): + remove_edge_ids = ['4_agency_a_city_a_1', '4_agency_a_city_a_2', + '4_agency_a_city_a_3'] + expected_edges = transit_edges_to_simplify.loc[ + ~transit_edges_to_simplify['id'].isin(remove_edge_ids)] + expected_edges.reset_index(inplace=True, drop=True) + + remove_node_ids = ['12_agency_a_city_a'] + expected_nodes = transit_nodes_to_simplify_extra_nodes.loc[ + ~transit_nodes_to_simplify_extra_nodes.index.isin(remove_node_ids)] + + result_edges, result_nodes = gtfs_network._simplify_transit_net( + transit_edges_to_simplify, transit_nodes_to_simplify_extra_nodes) + + # expect all nodes except for 12_agency_a_city_a to persist + assert len(result_nodes) == 11 + assert result_nodes.equals(expected_nodes) + + # expect 5_agency_a_city_a_1 to 3 and 6_agency_a_city_a_1 to 3 to + # persist since they have different attributes in unique_route_id col + # expect 1_agency_a_city_a_1 to 3 and 2_agency_a_city_a_1 to 3 to + # persist since node_id_from and node_id_to are different + # expect only one of 3_agency_a_city_a_1 to 3 or 4_agency_a_city_a_1 to 3 + # to persist but not both since they share the same attributes + assert len(result_edges) == 15 + result_edges.reset_index(inplace=True, drop=True) + assert result_edges.equals(expected_edges) + + +def test_simplify_transit_net_empty_edges( + edges_empty, transit_nodes_to_simplify): + with pytest.raises(ValueError) as excinfo: + result_edges, result_nodes = gtfs_network._simplify_transit_net( + edges_empty, transit_nodes_to_simplify) + expected_error = ( + 'Unable to simplify transit network. Edges have 0 records to ' + 'simplify.') + assert expected_error in str(excinfo.value) + + +def test_remove_nodes_not_in_edges( + transit_edges_to_simplify, transit_nodes_to_simplify_extra_nodes): + remove_node_ids = ['12_agency_a_city_a'] + expected_nodes = transit_nodes_to_simplify_extra_nodes.loc[ + ~transit_nodes_to_simplify_extra_nodes.index.isin(remove_node_ids)] + + result_nodes = gtfs_network._remove_nodes_not_in_edges( + transit_nodes_to_simplify_extra_nodes, transit_edges_to_simplify, + from_id_col='node_id_from', to_id_col='node_id_to') + + # expect all nodes except for 12_agency_a_city_a to persist + assert len(result_nodes) == 11 + assert result_nodes.equals(expected_nodes) + + +def test_remove_nodes_not_in_edges_none_to_remove( + transit_edges_to_simplify, transit_nodes_to_simplify): + result_nodes = gtfs_network._remove_nodes_not_in_edges( + transit_nodes_to_simplify, transit_edges_to_simplify, + from_id_col='node_id_from', to_id_col='node_id_to') + + # expect all nodes to persist + assert len(result_nodes) == 11 + assert result_nodes.equals(transit_nodes_to_simplify) diff --git a/urbanaccess/tests/test_gtfs_utils_calendar.py b/urbanaccess/tests/test_gtfs_utils_calendar.py new file mode 100644 index 0000000..19c1248 --- /dev/null +++ b/urbanaccess/tests/test_gtfs_utils_calendar.py @@ -0,0 +1,1834 @@ +import pytest +import pandas as pd + +import urbanaccess.gtfs.utils_calendar as gtfs_utils_cal + + +@pytest.fixture +def trips_agency_a(): + data = { + 'service_id': [ + 'summer-weekday-1', 'summer-weekday-1', 'summer-weekday-1', + 'summer-weekday-2', 'summer-weekday-3', 'summer-weekend-1', + 'fall-weekday-1', 'fall-weekday-2', 'fall-weekday-3', + 'fall-weekend-1', + 'fall-weekday-1', 'fall-weekday-2', 'fall-weekday-3', + 'fall-weekend-1', + 'special-game-day-1', 'special-game-day-2', 'special-game-day-3', + 'special-game-day-1', 'special-game-day-2', 'special-game-day-3'], + 'trip_id': ['a1', 'a2', 'a3', 'a4', 'a5', 'a6', + 'b1', 'b2', 'b3', 'b4', 'b1', 'b2', 'b3', 'b4', + 's1', 's2', 's3', 's4', 's5', 's6'], + 'route_id': ['1R'] * 20, + 'direction_id': [0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 1], + 'unique_agency_id': ['agency_a_city_a'] * 20, + } + index = range(20) + df = pd.DataFrame(data, index) + df['unique_service_id'] = (df['service_id'].str.cat( + df['unique_agency_id'].astype('str'), sep='_')) + return df + + +@pytest.fixture +def trips_agency_b(): + data = { + 'service_id': [ + 'fall-weekday-1', 'fall-weekday-2', 'fall-weekday-3', + 'fall-weekend-1', + 'fall-weekday-1', 'fall-weekday-2', 'fall-weekday-3', + 'fall-weekend-1', + 'special-game-day-1', 'special-game-day-2', 'special-game-day-3', + 'special-game-day-1', 'special-game-day-2', 'special-game-day-3'], + 'trip_id': ['b1', 'b2', 'b3', 'b4', 'b1', 'b2', 'b3', 'b4', + 's1', 's2', 's3', 's4', 's5', 's6'], + 'route_id': ['1R'] * 14, + 'direction_id': [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1], + 'unique_agency_id': ['agency_b_city_a'] * 14, + } + index = range(14) + df = pd.DataFrame(data, index) + df['unique_service_id'] = (df['service_id'].str.cat( + df['unique_agency_id'].astype('str'), sep='_')) + return df + + +@pytest.fixture +def calendar_agency_a(): + # 'summer-weekday-2' only runs on wednesdays + # 'fall-weekday-1' overlaps with summer sched for month of august + data = { + 'service_id': + ['summer-weekday-1', 'summer-weekday-2', 'summer-weekday-3', + 'summer-weekend-1', + 'fall-weekday-1', 'fall-weekday-2', 'fall-weekday-3', + 'fall-weekend-1'], + 'monday': [1, 0, 1, 0, 1, 1, 1, 0], + 'tuesday': [1, 0, 1, 0, 1, 1, 1, 0], + 'wednesday': [1, 1, 1, 0, 1, 1, 1, 0], + 'thursday': [1, 0, 1, 0, 1, 1, 1, 0], + 'friday': [1, 0, 1, 0, 1, 1, 1, 0], + 'saturday': [0, 0, 0, 1, 0, 0, 0, 1], + 'sunday': [0, 0, 0, 1, 0, 0, 0, 1], + 'start_date': + ['2016-6-01', '2016-6-01', '2016-6-01', '2016-6-01', + '2016-8-01', '2016-9-01', '2016-9-01', '2016-9-01'], + 'end_date': + ['2016-8-31', '2016-8-31', '2016-8-31', '2016-8-31', + '2016-11-30', '2016-11-30', '2016-11-30', '2016-11-30'], + 'unique_agency_id': ['agency_a_city_a'] * 8, + 'unique_feed_id': ['agency_a_city_a_feed'] * 8 + } + index = range(8) + df = pd.DataFrame(data, index) + df['unique_service_id'] = (df['service_id'].str.cat( + df['unique_agency_id'].astype('str'), sep='_')) + return df + + +@pytest.fixture +def calendar_dates_agency_a(): + # holiday '2016-7-04' falls on a monday but runs on 'summer-weekend-1' + # sched + # holiday '2016-9-05' falls on a monday but runs on 'fall-weekend-1' sched + # game day '2016-8-02' falls on a tuesday and runs extra service + data = { + 'service_id': + ['summer-weekday-1', 'summer-weekday-3', 'summer-weekend-1', + 'fall-weekday-1', 'fall-weekday-2', 'fall-weekday-3', + 'fall-weekend-1', + 'special-game-day-1', 'special-game-day-2', 'special-game-day-3'], + 'date': ['2016-7-04', '2016-7-04', '2016-7-04', + '2016-9-05', '2016-9-05', '2016-9-05', '2016-9-05', + '2016-8-02', '2016-8-02', '2016-8-02'], + 'exception_type': ['2', '2', '1', + '2', '2', '2', '1', + '1', '1', '1'], + 'schedule_type': ['holiday', 'holiday', 'holiday', + 'holiday', 'holiday', 'holiday', 'holiday', + 'game day', 'game day', 'game day'], + 'unique_agency_id': ['agency_a_city_a'] * 10, + 'unique_feed_id': ['agency_a_city_a_feed'] * 10 + } + index = range(10) + df = pd.DataFrame(data, index) + + return df + + +@pytest.fixture +def trips_and_cal_with_multi_freq_dates( + trips_agency_a, calendar_dates_agency_a): + data = { + 'service_id': [ + 'special-game-day-1_a', 'special-game-day-2_a', + 'special-game-day-3_a', 'special-game-day-1_a', + 'special-game-day-2_a', 'special-game-day-3_a'], + 'trip_id': ['s1_a', 's2_a', 's3_a', 's4_a', 's5_a', 's6_a'], + 'route_id': ['1R'] * 6, + 'direction_id': [0, 1, 0, 1, 0, 0], + 'unique_agency_id': ['agency_a_city_a'] * 6 + } + index = range(6) + extra_trips = pd.DataFrame(data, index) + extra_trips['unique_service_id'] = (extra_trips['service_id'].str.cat( + extra_trips['unique_agency_id'].astype('str'), sep='_')) + trips_df = pd.concat([trips_agency_a, extra_trips], ignore_index=True) + + data = { + 'service_id': + ['special-game-day-1_a', 'special-game-day-2_a', + 'special-game-day-3_a'], + 'date': ['2016-12-02', '2016-12-02', '2016-12-02'], + 'exception_type': ['1', '1', '1'], + 'schedule_type': ['game day', 'game day', 'game day'], + 'unique_agency_id': ['agency_a_city_a'] * 3, + 'unique_feed_id': ['agency_a_city_a_feed'] * 3 + } + index = range(3) + extra_cal_dates = pd.DataFrame(data, index) + cal_dates_df = pd.concat([calendar_dates_agency_a, extra_cal_dates], + ignore_index=True) + + return trips_df, cal_dates_df + + +@pytest.fixture +def calendar_agency_a_datetime(calendar_agency_a): + cols = ['start_date', 'end_date'] + for col in cols: + calendar_agency_a[col] = pd.to_datetime( + calendar_agency_a[col], format='mixed') + return calendar_agency_a + + +@pytest.fixture +def calendar_dates_agency_a_datetime(calendar_dates_agency_a): + col = 'date' + calendar_dates_agency_a[col] = pd.to_datetime( + calendar_dates_agency_a[col], format='mixed') + return calendar_dates_agency_a + + +@pytest.fixture +def calendar_dates_w_uni_svc_id(calendar_dates_agency_a): + df = calendar_dates_agency_a.copy() + df['unique_service_id'] = (df['service_id'].str.cat( + df['unique_agency_id'].astype('str'), sep='_')) + return df + + +@pytest.fixture +def calendar_dates_w_uni_svc_id_datetime(calendar_dates_agency_a_datetime): + df = calendar_dates_agency_a_datetime.copy() + df['unique_service_id'] = (df['service_id'].str.cat( + df['unique_agency_id'].astype('str'), sep='_')) + return df + + +@pytest.fixture +def calendar_empty(calendar_agency_a): + df_empty = calendar_agency_a[0:0] + return df_empty + + +@pytest.fixture +def calendar_dates_empty(calendar_dates_agency_a): + df_empty = calendar_dates_agency_a[0:0] + return df_empty + + +@pytest.fixture +def msg_param(): + return 'Selecting service_ids from calendar' + + +@pytest.fixture +def day_param(): + return 'monday' + + +@pytest.fixture +def date_param_cal(): + return '2016-06-13' + + +@pytest.fixture +def date_param_cal_dates(): + return '2016-08-02' + + +@pytest.fixture +def daterange_param(): + return ['2016-06-01', '2016-08-31'] + + +@pytest.fixture +def cal_dates_lookup_param(): + return {'schedule_type': 'holiday', + 'service_id': ['fall-weekday-1', 'fall-weekday-2']} + + +@pytest.fixture +def expected_result_cal_param_day(): + expected_active_srv_ids = sorted( + ['summer-weekday-1_agency_a_city_a', + 'summer-weekday-3_agency_a_city_a', + 'fall-weekday-1_agency_a_city_a', + 'fall-weekday-2_agency_a_city_a', + 'fall-weekday-3_agency_a_city_a']) + return expected_active_srv_ids + + +@pytest.fixture +def expected_result_cal_param_date(): + expected_active_srv_ids = sorted( + ['summer-weekday-1_agency_a_city_a', + 'summer-weekday-3_agency_a_city_a']) + return expected_active_srv_ids + + +@pytest.fixture +def expected_result_cal_param_daterange(): + expected_active_srv_ids = sorted( + ['summer-weekday-1_agency_a_city_a', + 'summer-weekday-2_agency_a_city_a', + 'summer-weekday-3_agency_a_city_a', + 'summer-weekend-1_agency_a_city_a']) + return expected_active_srv_ids + + +@pytest.fixture +def expected_result_cal_dates_param_day(): + expected_active_srv_ids = sorted( + ['summer-weekend-1_agency_a_city_a', + 'fall-weekend-1_agency_a_city_a']) + return expected_active_srv_ids + + +@pytest.fixture +def expected_result_cal_dates_param_date(): + expected_active_srv_ids = sorted( + ['special-game-day-3_agency_a_city_a', + 'special-game-day-2_agency_a_city_a', + 'special-game-day-1_agency_a_city_a']) + return expected_active_srv_ids + + +@pytest.fixture +def expected_result_cal_dates_param_daterange(): + expected_active_srv_ids = sorted( + ['special-game-day-1_agency_a_city_a', + 'special-game-day-3_agency_a_city_a', + 'special-game-day-2_agency_a_city_a', + 'summer-weekend-1_agency_a_city_a']) + return expected_active_srv_ids + + +@pytest.fixture +def expected_result_cal_dates_param_lookup(): + expected_srvc_ids_add = sorted( + ['summer-weekend-1_agency_a_city_a', 'fall-weekend-1_agency_a_city_a']) + expected_srvc_ids_del = sorted( + ['summer-weekday-1_agency_a_city_a', + 'fall-weekday-3_agency_a_city_a', + 'fall-weekday-1_agency_a_city_a', + 'summer-weekday-3_agency_a_city_a', + 'fall-weekday-2_agency_a_city_a']) + return expected_srvc_ids_add, expected_srvc_ids_del + + +def test_select_cal_service_ids_w_day( + calendar_agency_a, expected_result_cal_param_day, day_param): + result = gtfs_utils_cal._select_calendar_service_ids( + calendar_agency_a, + params={'day': day_param, + 'date': None, + 'date_range': None, + 'cal_dates_lookup': None, + 'has_cal': True, + 'has_day_param': True, + 'has_date_param': False, + 'has_date_range_param': False, + 'has_cal_dates': False, + 'has_cal_dates_param': False, + 'has_day_and_date_range_param': False}) + + assert isinstance(result, list) + assert len(result) == 5 + # result should only include service IDs in calendar where monday = 1 + assert sorted(result) == expected_result_cal_param_day + + +def test_select_cal_service_ids_w_date( + calendar_agency_a, date_param_cal, expected_result_cal_param_date): + result = gtfs_utils_cal._select_calendar_service_ids( + calendar_agency_a, + params={'day': None, + 'date': date_param_cal, + 'date_range': None, + 'cal_dates_lookup': None, + 'has_cal': True, + 'has_day_param': False, + 'has_date_param': True, + 'has_date_range_param': False, + 'has_cal_dates': False, + 'has_cal_dates_param': False, + 'has_day_and_date_range_param': False}) + + assert isinstance(result, list) + assert len(result) == 2 + # result should only include service IDs in calendar that are active on + # '2016-06-13' representing a monday sched + assert sorted(result) == expected_result_cal_param_date + + +def test_select_cal_service_ids_w_date_range( + calendar_agency_a, daterange_param, + expected_result_cal_param_daterange): + result = gtfs_utils_cal._select_calendar_service_ids( + calendar_agency_a, + params={'day': None, + 'date': None, + 'date_range': daterange_param, + 'cal_dates_lookup': None, + 'has_cal': True, + 'has_day_param': False, + 'has_date_param': False, + 'has_date_range_param': True, + 'has_cal_dates': False, + 'has_cal_dates_param': False, + 'has_day_and_date_range_param': False}) + + assert isinstance(result, list) + assert len(result) == 4 + # result should only include service IDs in calendar that are active + # between ['2016-06-01', '2016-08-31'] + assert sorted(result) == expected_result_cal_param_daterange + + +def test_select_cal_service_ids_w_day_and_date_range( + calendar_agency_a, day_param, daterange_param): + result = gtfs_utils_cal._select_calendar_service_ids( + calendar_agency_a, + params={'day': day_param, + 'date': None, + 'date_range': daterange_param, + 'cal_dates_lookup': None, + 'has_cal': True, + 'has_day_param': True, + 'has_date_param': False, + 'has_date_range_param': True, + 'has_cal_dates': False, + 'has_cal_dates_param': False, + 'has_day_and_date_range_param': True}) + + expected_active_srv_ids = sorted( + ['summer-weekday-3_agency_a_city_a', + 'summer-weekday-1_agency_a_city_a']) + + assert isinstance(result, list) + assert len(result) == 2 + # result should only include service IDs in calendar that are active + # between ['2016-06-01', '2016-08-31'] and 'monday' + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_day_param_w_cal_only( + calendar_agency_a, calendar_dates_empty, + expected_result_cal_param_day, day_param): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_empty, + day=day_param, + date=None, + date_range=None, + cal_dates_lookup=None) + + assert isinstance(result, list) + assert len(result) == 5 + # result should only include service IDs in calendar where monday = 1 + assert sorted(result) == expected_result_cal_param_day + + +def test_cal_srv_id_selector_day_param_w_cal_dates_only( + calendar_empty, calendar_dates_agency_a, day_param, + expected_result_cal_dates_param_day): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_empty, + calendar_dates_df=calendar_dates_agency_a, + day=day_param, + date=None, + date_range=None, + cal_dates_lookup=None) + + assert isinstance(result, list) + assert len(result) == 2 + # result should only include service IDs in calendar dates whose date is a + # monday and exception_type = 1 + assert sorted(result) == expected_result_cal_dates_param_day + + +def test_cal_srv_id_selector_day_param_w_cal_and_cal_dates( + calendar_agency_a, calendar_dates_agency_a): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_agency_a, + day='tuesday', + date=None, + date_range=None, + cal_dates_lookup=None) + + expected_active_srv_ids = sorted( + ['summer-weekday-1_agency_a_city_a', + 'summer-weekday-3_agency_a_city_a', + 'fall-weekday-1_agency_a_city_a', + 'fall-weekday-2_agency_a_city_a', + 'fall-weekday-3_agency_a_city_a', + 'special-game-day-1_agency_a_city_a', + 'special-game-day-2_agency_a_city_a', + 'special-game-day-3_agency_a_city_a']) + + assert isinstance(result, list) + assert len(result) == 8 + # result should only include service IDs in calendar dates whose date is a + # tuesday regardless of exception_type + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_date_param_w_cal_only( + calendar_agency_a, calendar_dates_empty, date_param_cal): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_empty, + day=None, + date=date_param_cal, + date_range=None, + cal_dates_lookup=None) + + expected_active_srv_ids = sorted( + ['summer-weekday-1_agency_a_city_a', + 'summer-weekday-3_agency_a_city_a']) + + assert isinstance(result, list) + assert len(result) == 2 + # result should only include service IDs in calendar that are active on + # '2016-06-13' representing a monday sched + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_date_param_w_cal_dates_only_case_1( + calendar_empty, calendar_dates_agency_a, date_param_cal_dates, + expected_result_cal_dates_param_date): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_empty, + calendar_dates_df=calendar_dates_agency_a, + day=None, + date=date_param_cal_dates, + date_range=None, + cal_dates_lookup=None) + + assert isinstance(result, list) + assert len(result) == 3 + # result should only include service IDs in calendar dates that are + # active on '2016-08-02' with exception_type = 1 + assert sorted(result) == expected_result_cal_dates_param_date + + +def test_cal_srv_id_selector_date_param_w_cal_dates_only_case_2( + calendar_empty, calendar_dates_agency_a): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_empty, + calendar_dates_df=calendar_dates_agency_a, + day=None, + date='2016-07-04', + date_range=None, + cal_dates_lookup=None) + + expected_active_srv_ids = ['summer-weekend-1_agency_a_city_a'] + + assert isinstance(result, list) + assert len(result) == 1 + # result should only include service IDs in calendar dates that are + # active on '2016-07-04' with exception_type = 1 + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_date_param_w_cal_and_cal_dates_case_1( + calendar_agency_a, calendar_dates_agency_a, date_param_cal_dates): + # case 1: add service IDs via cal dates + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_agency_a, + day=None, + date=date_param_cal_dates, + date_range=None, + cal_dates_lookup=None) + + expected_active_srv_ids = sorted( + ['special-game-day-1_agency_a_city_a', + 'special-game-day-2_agency_a_city_a', + 'special-game-day-3_agency_a_city_a', + 'summer-weekday-1_agency_a_city_a', + 'summer-weekday-3_agency_a_city_a', + 'fall-weekday-1_agency_a_city_a' + ]) + + assert isinstance(result, list) + assert len(result) == 6 + # result should only include service IDs in calendar that are active + # on '2016-08-02' and add service IDs from calendar dates where + # exception = 1 and remove service IDs from calendar dates where + # exception = 2 + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_date_param_w_cal_and_cal_dates_case_2( + calendar_agency_a, calendar_dates_agency_a): + # case 2: remove service IDs via cal dates + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_agency_a, + day=None, + date='2016-07-04', + date_range=None, + cal_dates_lookup=None) + + expected_active_srv_ids = ['summer-weekend-1_agency_a_city_a'] + + assert isinstance(result, list) + assert len(result) == 1 + # result should only include service IDs in calendar that are active + # on '2016-07-04' and add service IDs from calendar dates where + # exception = 1 and remove service IDs from calendar dates where + # exception = 2 + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_date_range_param_w_cal_only( + calendar_agency_a, calendar_dates_empty, daterange_param, + expected_result_cal_param_daterange): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_empty, + day=None, + date=None, + date_range=daterange_param, + cal_dates_lookup=None) + + assert isinstance(result, list) + assert len(result) == 4 + # result should only include service IDs in calendar that are active + # between ['2016-06-01', '2016-08-31'] + assert sorted(result) == expected_result_cal_param_daterange + + +def test_cal_srv_id_selector_date_range_param_w_cal_dates_only( + calendar_empty, calendar_dates_agency_a, daterange_param, + expected_result_cal_dates_param_daterange): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_empty, + calendar_dates_df=calendar_dates_agency_a, + day=None, + date=None, + date_range=daterange_param, + cal_dates_lookup=None) + + assert isinstance(result, list) + assert len(result) == 4 + # result should only include service IDs in calendar dates that are + # active between ['2016-06-01', '2016-08-31'] with exception_type = 1 + assert sorted(result) == expected_result_cal_dates_param_daterange + + +def test_cal_srv_id_selector_date_range_param_w_cal_and_cal_dates( + calendar_agency_a, calendar_dates_agency_a, daterange_param): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_agency_a, + day=None, + date=None, + date_range=daterange_param, + cal_dates_lookup=None) + + expected_active_srv_ids = sorted( + ['special-game-day-1_agency_a_city_a', + 'special-game-day-2_agency_a_city_a', + 'special-game-day-3_agency_a_city_a', + 'summer-weekend-1_agency_a_city_a', + 'summer-weekday-2_agency_a_city_a']) + + assert isinstance(result, list) + assert len(result) == 5 + # result should only include service IDs in calendar that are active + # between ['2016-06-01', '2016-08-31'] and in calendar dates that are + # active between ['2016-06-01', '2016-08-31'] with exception_type = 1 + # and remove service IDs from calendar dates where exception = 2 + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_day_and_date_range_param_w_cal_only( + calendar_agency_a, calendar_dates_empty, day_param, daterange_param): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_empty, + day=day_param, + date=None, + date_range=daterange_param, + cal_dates_lookup=None) + + expected_active_srv_ids = sorted( + ['summer-weekday-3_agency_a_city_a', + 'summer-weekday-1_agency_a_city_a']) + + assert isinstance(result, list) + assert len(result) == 2 + # result should only include service IDs in calendar that are active + # between ['2016-06-01', '2016-08-31'] and on 'monday' + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_day_and_date_range_param_w_cal_dates_only( + calendar_empty, calendar_dates_agency_a, day_param, daterange_param): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_empty, + calendar_dates_df=calendar_dates_agency_a, + day=day_param, + date=None, + date_range=daterange_param, + cal_dates_lookup=None) + + expected_active_srv_ids = ['summer-weekend-1_agency_a_city_a'] + + assert isinstance(result, list) + assert len(result) == 1 + # result should only include service IDs in calendar dates that are + # active between ['2016-06-01', '2016-08-31'] on a 'monday' with + # exception_type = 1 + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_day_date_range_param_w_cal_and_cal_dates( + calendar_agency_a, calendar_dates_agency_a, day_param, + daterange_param): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_agency_a, + day=day_param, + date=None, + date_range=daterange_param, + cal_dates_lookup=None) + + expected_active_srv_ids = sorted(['summer-weekend-1_agency_a_city_a', + 'summer-weekday-3_agency_a_city_a', + 'summer-weekday-1_agency_a_city_a']) + + assert isinstance(result, list) + assert len(result) == 3 + # result should only include service IDs in calendar that are active + # between ['2016-06-01', '2016-08-31'] on a monday and in calendar dates + # that are active between ['2016-06-01', '2016-08-31'] on a monday with + # exception_type = 1 + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_lookup_param_w_cal_only_invalid( + calendar_agency_a, calendar_dates_empty, day_param, + cal_dates_lookup_param): + with pytest.raises(ValueError) as excinfo: + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_empty, + day=day_param, + date=None, + date_range=None, + cal_dates_lookup=cal_dates_lookup_param) + expected_error = ("Calendar_dates is empty. Unable to use the " + "'calendar_dates_lookup' parameter. Set to None.") + assert expected_error in str(excinfo.value) + + +def test_cal_srv_id_selector_lookup_param_w_cal_dates_only( + calendar_empty, calendar_dates_agency_a, day_param, + cal_dates_lookup_param, + expected_result_cal_dates_param_day): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_empty, + calendar_dates_df=calendar_dates_agency_a, + day=day_param, + date=None, + date_range=None, + cal_dates_lookup=cal_dates_lookup_param) + + assert isinstance(result, list) + assert len(result) == 2 + # result should only include service IDs in calendar dates that match + # the values in the query with exception_type = 1 and where date is a + # 'monday' + assert sorted(result) == expected_result_cal_dates_param_day + + +def test_cal_srv_id_selector_lookup_param_w_cal_and_cal_dates( + calendar_agency_a, calendar_dates_agency_a, day_param, + cal_dates_lookup_param): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_agency_a, + day=day_param, + date=None, + date_range=None, + cal_dates_lookup=cal_dates_lookup_param) + + expected_active_srv_ids = sorted( + ['fall-weekend-1_agency_a_city_a', + 'summer-weekend-1_agency_a_city_a']) + + assert isinstance(result, list) + assert len(result) == 2 + # result should only include service IDs in calendar dates that match + # the values in the query with exception_type = 1 and remove service IDs + # from calendar dates where exception = 2 and also service IDs from + # calendar that are active for 'monday' + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_warn(capsys, + calendar_agency_a, calendar_dates_empty): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_empty, + day=None, + date='2016-01-01', + date_range=None, + cal_dates_lookup=None) + + assert isinstance(result, list) + assert len(result) == 0 + + # check that expected print prints + captured = capsys.readouterr() + assert ('Warning: No active service_ids were found matching the specified ' + 'parameters.') in captured.out + + +def test_add_unique_service_id( + calendar_dates_agency_a, calendar_agency_a): + # remove the existing column from the test data + calendar_agency_a.drop(columns=['unique_service_id'], inplace=True) + df_list = [calendar_dates_agency_a, calendar_agency_a] + result = gtfs_utils_cal._add_unique_service_id(df_list) + assert isinstance(result, list) + assert isinstance(result[0], pd.core.frame.DataFrame) + assert isinstance(result[1], pd.core.frame.DataFrame) + assert result[0].empty is False + assert result[1].empty is False + assert 'unique_service_id' in result[0].columns + assert 'unique_service_id' in result[1].columns + assert result[0]['unique_service_id'].iloc[0] == \ + 'summer-weekday-1_agency_a_city_a' + + +def test_cal_date_dt_conversion_valid(calendar_agency_a): + result = gtfs_utils_cal._cal_date_dt_conversion( + df=calendar_agency_a, + date_cols=['start_date', 'end_date']) + assert isinstance(result, pd.core.frame.DataFrame) + assert result.dtypes['start_date'] == '