From ee7a8a0acba231bfb4eb06baab0e1ddc6016a7a5 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 31 Aug 2021 13:47:35 -0700 Subject: [PATCH 01/72] validate if timerange is none --- urbanaccess/gtfs/utils_validation.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/urbanaccess/gtfs/utils_validation.py b/urbanaccess/gtfs/utils_validation.py index e9c7dff..25737ee 100644 --- a/urbanaccess/gtfs/utils_validation.py +++ b/urbanaccess/gtfs/utils_validation.py @@ -199,6 +199,8 @@ def _check_time_range_format(timerange): ------- None """ + if timerange is None: + raise ValueError('timerange cannot be None.') time_error_statement = ( '{} starttime and endtime are not in the correct format. ' 'Format should be a 24 hour clock in the following format: 08:00:00 ' From e8ebd1d667881a8ca21107abe7417a8821bc18a4 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 31 Aug 2021 13:49:20 -0700 Subject: [PATCH 02/72] update calendar, calendar_dates and trips dtypes config to deal with datetime and further processing --- urbanaccess/config.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/urbanaccess/config.py b/urbanaccess/config.py index 67bea78..c5f23c6 100644 --- a/urbanaccess/config.py +++ b/urbanaccess/config.py @@ -305,7 +305,7 @@ def to_yaml(self, configdir='configs', yamlname='urbanaccess_config.yaml', 'remove_whitespace': ['route_id'], 'min_required_cols': ['route_id']}, 'trips': {'req_dtypes': {'trip_id': object, - 'service_id': object, + 'service_id': str, 'route_id': object}, 'opt_dtypes': {'shape_id': object}, 'numeric_converter': None, @@ -321,7 +321,9 @@ def to_yaml(self, configdir='configs', yamlname='urbanaccess_config.yaml', 'remove_whitespace': ['trip_id', 'stop_id'], 'min_required_cols': ['trip_id', 'stop_id', 'departure_time', 'arrival_time']}, - 'calendar': {'req_dtypes': {'service_id': object}, + 'calendar': {'req_dtypes': {'service_id': object, + 'start_date': object, + 'end_date': object}, 'opt_dtypes': None, 'numeric_converter': ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', @@ -331,7 +333,9 @@ def to_yaml(self, configdir='configs', yamlname='urbanaccess_config.yaml', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday', 'start_date', 'end_date']}, - 'calendar_dates': {'req_dtypes': {'service_id': object}, + 'calendar_dates': {'req_dtypes': {'service_id': object, + 'date': object, + 'exception_type': object}, 'opt_dtypes': None, 'numeric_converter': None, 'remove_whitespace': ['service_id'], From 53751c601cd840493b92dfecc97d00597cbf257c Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 31 Aug 2021 13:51:10 -0700 Subject: [PATCH 03/72] create new utils_calendar.py and expand options for service ID selection using cal and or cal dates tables with date, date range, high feq trip date --- urbanaccess/gtfs/utils_calendar.py | 973 +++++++++++++++++++++++++++++ 1 file changed, 973 insertions(+) create mode 100644 urbanaccess/gtfs/utils_calendar.py diff --git a/urbanaccess/gtfs/utils_calendar.py b/urbanaccess/gtfs/utils_calendar.py new file mode 100644 index 0000000..b559228 --- /dev/null +++ b/urbanaccess/gtfs/utils_calendar.py @@ -0,0 +1,973 @@ +from __future__ import division +import pandas as pd +import time +from datetime import datetime as dt +import logging as lg + +from urbanaccess.utils import log + + +def _trip_schedule_selector_validate_params(calendar_dates_df, params): + has_cal = params['has_cal'] + has_cal_dates = params['has_cal_dates'] + + day = params['day'] + date = params['date'] + date_range = params['date_range'] + cal_dates_lookup = params['cal_dates_lookup'] + + has_day_param = params['has_day_param'] + has_date_param = params['has_date_param'] + has_date_range_param = params['has_date_range_param'] + has_cal_dates_param = params['has_cal_dates_param'] + + # only one search parameter can be used at a time + param_list = [has_day_param, has_date_param, has_date_range_param] + param_true_cnt = param_list.count(True) + if param_true_cnt > 1: + raise ValueError( + "Only one parameter: 'day', 'date', or 'date_range' can be used " + "at a time. To proceed, keep one parameter and set the other " + "parameter(s) to None.") + + if has_day_param: + if not isinstance(day, str): + raise ValueError('Day must be a string.') + valid_days = ['monday', 'tuesday', 'wednesday', 'thursday', + 'friday', 'saturday', 'sunday'] + if day not in valid_days: + valid_days_str = str(valid_days).replace('[', '').replace(']', '') + raise ValueError( + 'Day: {} is not a supported day. Must be one of lowercase ' + 'strings: {}.'.format(day, valid_days_str)) + + if has_date_param: + if not isinstance(date, str): + raise ValueError('Date must be a string.') + try: + dt.strptime(date, '%Y-%m-%d') + except ValueError: + raise ValueError("Date: {} is not a supported date format. " + "Expected format: 'YYYY-MM-DD'".format(date)) + + if has_date_range_param: + if len(date_range) != 2: + raise ValueError("Date range {} must have a " + "length of 2.".format(date_range)) + for date_item in date_range: + if not isinstance(date_item, str): + raise ValueError('Dates in date range must be a string.') + try: + dt.strptime(date_item, '%Y-%m-%d') + except ValueError: + raise ValueError("Date: {} in date range: {} is not a " + "supported date format. Expected format: " + "'YYYY-MM-DD'".format(date_item, date_range)) + item_1 = dt.strptime(date_range[0], '%Y-%m-%d') + item_2 = dt.strptime(date_range[1], '%Y-%m-%d') + if item_1 > item_2: + raise ValueError("First date in date range {} must be less than " + "the last date.".format(date_range)) + + if has_cal_dates_param: + if has_cal_dates: + if not isinstance(cal_dates_lookup, dict): + raise ValueError( + 'calendar_dates_lookup parameter must be a dictionary.') + for key in cal_dates_lookup.keys(): + if not isinstance(key, str): + raise ValueError('calendar_dates_lookup key: {} ' + 'must be a string.'.format(key)) + if isinstance(cal_dates_lookup[key], str): + value = [cal_dates_lookup[key]] + else: + if not isinstance(cal_dates_lookup[key], list): + raise ValueError( + 'calendar_dates_lookup value: {} must be a ' + 'string or a list of strings.'.format( + cal_dates_lookup[key])) + else: + value = cal_dates_lookup[key] + + for string in value: + if not isinstance(string, str): + raise ValueError('calendar_dates_lookup value: {} ' + 'must contain strings.'.format(value)) + for col_name_key, string_value in cal_dates_lookup.items(): + if col_name_key not in calendar_dates_df.columns: + raise ValueError('Column: {} not found in calendar_dates ' + 'DataFrame.'.format(col_name_key)) + if col_name_key not in calendar_dates_df.select_dtypes( + include=[object]).columns: + raise ValueError('Column: {} must be object type.'.format( + col_name_key)) + else: + raise ValueError("Calendar_dates is empty. Unable to use the " + "'calendar_dates_lookup' parameter. Set to None.") + + if not has_cal: + log(' Calendar table has no records and will not be used to ' + 'select service_ids.') + if not has_cal_dates: + log(' Calendar dates table has no records and will not be used to ' + 'select service_ids.') + + +def _cal_date_dt_conversion(df, date_cols): + for col in date_cols: + df[col] = pd.to_datetime( + df[col], format='%y%m%d', infer_datetime_format=True) + return df + + +def _select_calendar_service_ids(calendar_df, params): + # collect service IDs that match search parameters in calendar.txt + msg = 'Selecting service_ids from calendar' + + day = params['day'] + date = params['date'] + date_range = params['date_range'] + has_day_param = params['has_day_param'] + has_date_param = params['has_date_param'] + has_date_range_param = params['has_date_range_param'] + + # convert cols to datetime expected format: 'yyyymmdd' e.g.: '20130825' + date_cols = ['start_date', 'end_date'] + calendar_df = _cal_date_dt_conversion(df=calendar_df, date_cols=date_cols) + + if has_day_param: + srvc_ids_day = _select_calendar_service_ids_by_day( + calendar_df, msg, day) + srvc_ids = srvc_ids_day.copy() + + if has_date_range_param: + srvc_ids_date_range = _select_calendar_service_ids_by_date_range( + calendar_df, msg, date_range) + srvc_ids = srvc_ids_date_range.copy() + + if has_date_param: + srvc_ids_date = _select_calendar_service_ids_by_date( + calendar_df, msg, date) + srvc_ids = srvc_ids_date.copy() + + if has_day_param and has_date_range_param: + print_msg = '{} that are active within date_range: {} on day: {}...' + log(print_msg.format(msg, date_range, day)) + intersect_srvc_ids = _intersect_cal_service_ids( + dict_1={'day': srvc_ids_day}, + dict_2={'date range': srvc_ids_date_range}) + srvc_ids = intersect_srvc_ids.copy() + + return srvc_ids + + +def _print_cal_service_ids_len(srvc_ids, table_name): + cnt_msg = ' {:,} service_id(s) were selected from {}.' + warning_msg = ' Warning: No service_ids were selected from {}.' + srvc_ids_cnt = len(srvc_ids) + uni_srvc_ids_cnt = len(set(srvc_ids)) + # TODO: print count of records and unique IDs + if srvc_ids_cnt > 0: + log(cnt_msg.format(uni_srvc_ids_cnt, table_name)) + else: + log(warning_msg.format(table_name), level=lg.WARNING) + + +def _intersect_cal_service_ids(dict_1, dict_2, verbose=True): + srvc_id_1_name, srvc_id_1 = list(dict_1.items())[0] + srvc_id_2_name, srvc_id_2 = list(dict_2.items())[0] + srvc_ids = list(set(srvc_id_1) & set(srvc_id_2)) + if verbose: + srvc_ids_len = len(srvc_ids) + list_1_len = len(srvc_id_1) + list_2_len = len(srvc_id_2) + msg = (' Intersection between service_ids selected with: ' + '{} parameter (count: {:,}) and {} parameter (count: {:,}) ' + 'returned {:,} total service_id(s).'.format( + srvc_id_1_name, list_1_len, srvc_id_2_name, list_2_len, + srvc_ids_len)) + log(msg) + return srvc_ids + + +def _select_calendar_service_ids_by_day( + calendar_df, msg, day=None, verbose=True): + # Global search using: day in calendar.txt + # select service IDs where day specified = 1 where + # 1 = service is active; 0 = service inactive + if verbose: + log(' {} that are active on day: {}...'.format(msg, day)) + subset_cal_df = calendar_df.loc[calendar_df[day] == 1] + srvc_ids = subset_cal_df['unique_service_id'].to_list() + + if verbose: + _print_cal_service_ids_len(srvc_ids, table_name='calendar') + return srvc_ids + + +def _select_calendar_service_ids_by_date_range( + calendar_df, msg, date_range=None): + # Global search using: date_range to subset records in calendar.txt + # using 'start_date' and 'end_date' columns when GTFS feed has + # seasonal service_ids + # select service IDs where 'start_date' and 'end_date' is within the + # date_range + log(' {} that are active within date_range: {}...'.format( + msg, date_range)) + + # select service_ids that are within the date range + service_ids = calendar_df['unique_service_id'] + start_d_mask = service_ids.loc[calendar_df['start_date'].between( + date_range[0], date_range[1], inclusive=True)].to_list() + end_d_mask = service_ids.loc[calendar_df['end_date'].between( + date_range[0], date_range[1], inclusive=True)].to_list() + ids_in_range = service_ids.loc[service_ids.isin(start_d_mask) & + service_ids.isin(end_d_mask)] + subset_cal_df = calendar_df.loc[service_ids.isin(ids_in_range)] + srvc_ids = subset_cal_df['unique_service_id'].to_list() + + if len(srvc_ids) == 0: + date_rngs = _calendar_date_ranges(calendar_df) + print_msg = ( + ' Warning: Date range: {} does not contain any of the ' + 'date ranges available in the calendar. Date ranges available ' + 'are: \n {}'.format(date_range, date_rngs)) + log(print_msg, level=lg.WARNING) + else: + date_rngs = _calendar_date_ranges( + calendar_df.loc[calendar_df['unique_service_id'].isin(srvc_ids)]) + print_msg = ( + ' Date range: {} contains the following date ranges ' + 'available in the calendar. Date ranges available ' + 'are:: \n {}'.format(date_range, date_rngs)) + log(print_msg) + + _print_cal_service_ids_len(srvc_ids, table_name='calendar') + return srvc_ids + + +def _calendar_date_ranges(calendar_df, for_print=True): + date_cols = ['start_date', 'end_date'] + unique_range = calendar_df[date_cols].drop_duplicates(subset=date_cols) + # build list of ranges that exist in table to print to user + if for_print: + for col in date_cols: + # convert to string for prints + unique_range[col] = unique_range[col].dt.strftime('%Y-%m-%d') + unique_range[col] = unique_range[col].astype('str') + date_rngs = list(unique_range['start_date'].str.cat( + unique_range['end_date'], sep=' to ').values) + # build dict of ranges that exist in table to use elsewhere + else: + # keep in datetime format + date_rngs = unique_range.to_dict('records') + return date_rngs + + +def _select_calendar_service_ids_by_date( + calendar_df, msg, date=None, verbose=True): + # Global search using: date to subset records in calendar.txt + # using 'start_date' and 'end_date' columns when GTFS feed has + # seasonal service_ids + # select service IDs where date is within the 'start_date' and + # 'end_date' and then select IDs that are active on the day of + # the week the date represents + + # convert date to day of the week + day = dt.strptime(date, '%Y-%m-%d').strftime('%A').lower() + + if verbose: + log(' {} that are active on date: {} ' + '(using a {} schedule)...'.format(msg, date, day)) + + # select date ranges where the 'date' is within the bounds of + # 'start_date' and 'end_date' + service_ids = calendar_df['unique_service_id'] + date_dt = pd.to_datetime(date) + start_d_mask = service_ids.loc[ + calendar_df['start_date'] <= date_dt].to_list() + end_d_mask = service_ids.loc[ + calendar_df['end_date'] >= date_dt].to_list() + ids_in_range = service_ids.loc[ + service_ids.isin(start_d_mask) & service_ids.isin(end_d_mask)] + subset_cal_df = calendar_df.loc[service_ids.isin(ids_in_range)] + srvc_ids_date = subset_cal_df['unique_service_id'].to_list() + + if len(srvc_ids_date) == 0: + date_rngs = _calendar_date_ranges(calendar_df) + if verbose: + print_msg = ( + ' Warning: Date: {} does not fall within any ' + 'date range(s) available in the calendar. ' + 'Available date range(s) found in the calendar ' + 'are: \n {}') + log(print_msg.format(date, date_rngs), level=lg.WARNING) + else: + date_rngs = _calendar_date_ranges( + calendar_df.loc[ + calendar_df['unique_service_id'].isin(srvc_ids_date)]) + if verbose: + print_msg = ( + ' Date: {} is within the available date range(s) ' + 'in the calendar. Available date range(s) found in the ' + 'calendar are: \n {}') + log(print_msg.format(date, date_rngs)) + + # use the date's day of the week to then select service_ids that are + # active on that day + srvc_ids_day = _select_calendar_service_ids_by_day( + calendar_df, msg, day, verbose=False) + # find the intersection of where service_ids are active on the date and + # the day of week to get the final list of IDs + srvc_ids = _intersect_cal_service_ids( + dict_1={'date': srvc_ids_date}, + dict_2={'day': srvc_ids_day}, + verbose=verbose) + if verbose: + _print_cal_service_ids_len(srvc_ids, table_name='calendar') + return srvc_ids + + +def _parse_cal_dates_exception_type(cal_dates_df, verbose=True): + # 1 - Service has been added for the specified date. + # 2 - Service has been removed for the specified date. + id_col = 'unique_service_id' + exception_col = 'exception_type' + + srvc_ids_add = cal_dates_df[id_col].loc[ + cal_dates_df[exception_col] == '1'].to_list() + srvc_ids_del = cal_dates_df[id_col].loc[ + cal_dates_df[exception_col] == '2'].to_list() + srvc_ids_add = list(set(srvc_ids_add)) + srvc_ids_del = list(set(srvc_ids_del)) + if verbose: + log(' {:,} service_id(s) were selected from calendar_dates ' + 'with exception_type 1 ' + '(service added for specified date).'.format(len(srvc_ids_add))) + log(' {:,} service_id(s) were selected from calendar_dates ' + 'with exception_type 2 ' + '(service removed for specified date).'.format(len(srvc_ids_del))) + return srvc_ids_add, srvc_ids_del + + +def _select_calendar_dates_service_ids_by_day( + calendar_dates_df, msg, day=None): + # select all service ids that are active on this day of week, + # that are inclusive + + # determine day of the week for each date + log(' {} that are active on day: {}...'.format(msg, day)) + # TODO: check if there is an issue with other versions of pandas such that + # requiring pandas > 0.18.1 is needed to run the line below: + calendar_dates_df['_ua_weekday'] = calendar_dates_df[ + 'date'].dt.day_name().str.lower() + subset_cal_dates_df = calendar_dates_df.loc[ + calendar_dates_df['_ua_weekday'] == day] + srvc_ids = subset_cal_dates_df['unique_service_id'].to_list() + _print_cal_service_ids_len(srvc_ids, table_name='calendar dates') + return srvc_ids + + +def _select_calendar_dates_service_ids_by_date( + calendar_dates_df, msg, date=None, verbose=True): + # select all that are on this day, that are inclusive + # remove from those found in calendar that are exclusive + if verbose: + log(' {} that are active on date: {}...'.format(msg, date)) + subset_df = calendar_dates_df.loc[calendar_dates_df['date'] == date] + + srvc_ids_add, srvc_ids_del = _parse_cal_dates_exception_type( + subset_df, verbose) + + if verbose: + if len(srvc_ids_add) == 0: + print_msg = (' Warning: No active service_ids were ' + 'found on date: {}.') + log(print_msg.format(date), level=lg.WARNING) + + return srvc_ids_add, srvc_ids_del + + +def _select_calendar_dates_service_ids_by_date_range( + calendar_dates_df, msg, date_range=None, verbose=True): + # select all service ids that are active within date_range + if verbose: + log(' {} that are active within date_range: {}...'.format( + msg, date_range)) + subset_df = calendar_dates_df.loc[ + calendar_dates_df['date'].between( + date_range[0], date_range[1], inclusive=True)] + + srvc_ids_add, srvc_ids_del = _parse_cal_dates_exception_type(subset_df) + + if verbose: + if len(srvc_ids_add) == 0: + earliest_date = calendar_dates_df[ + 'date'].min().strftime('%Y-%m-%d') + latest_date = calendar_dates_df[ + 'date'].max().strftime('%Y-%m-%d') + print_msg = (' Warning: No active service_ids were ' + 'found within date range: {}. ' + 'Dates only exist between: {} and {}.') + log(print_msg.format(date_range, earliest_date, latest_date), + level=lg.WARNING) + + return srvc_ids_add, srvc_ids_del + + +def _add_exception_type_service_id_lists(srvc_ids_list_dict): + print_dict = {} + for list_type, srvc_id_lists in srvc_ids_list_dict.items(): + srvc_ids_cnt_1 = len(srvc_id_lists[0]) + srvc_id_lists[0].extend(srvc_id_lists[1]) + srvc_ids_wo_dups = set(srvc_id_lists[0]) + result_srvc_ids = list(srvc_ids_wo_dups) + srvc_ids_cnt_2 = len(srvc_id_lists[0]) + result_cnt = abs(srvc_ids_cnt_1 - srvc_ids_cnt_2) + print_dict.update( + {list_type: [result_cnt, srvc_ids_cnt_1, srvc_ids_cnt_2]}) + + if list_type == 'add': + srvc_ids_add = result_srvc_ids + if list_type == 'del': + srvc_ids_del = result_srvc_ids + + msg_print = ( + ' Added {:,} active and {:,} inactive service_id(s) ' + 'found with calendar_dates_lookup parameter to the existing {:,} ' + 'active and {:,} inactive service_id(s) selected in calendar_' + 'dates to total: {:,} active and {:,} inactive service_id(s).') + log(msg_print.format( + print_dict['add'][0], print_dict['del'][0], + print_dict['add'][1], print_dict['del'][1], + print_dict['add'][2], print_dict['del'][2])) + return srvc_ids_add, srvc_ids_del + + +def _select_calendar_dates_service_ids(calendar_dates_df, params): + # collect service IDs that match search parameters in calendar_dates.txt + msg = ' Selecting service_ids from calendar_dates' + + day = params['day'] + date = params['date'] + date_range = params['date_range'] + cal_dates_lookup = params['cal_dates_lookup'] + has_day_param = params['has_day_param'] + has_date_param = params['has_date_param'] + has_date_range_param = params['has_date_range_param'] + has_cal_dates_param = params['has_cal_dates_param'] + + # convert 'date' to datetime expected format: 'yyyymmdd' e.g.: '20130825' + calendar_dates_df = _cal_date_dt_conversion( + df=calendar_dates_df, date_cols=['date']) + + # No service_ids can be removed when using day of the week since its + # a global selection parameter over the entire calendar + srvc_ids_add = [] + srvc_ids_del = [] + + if has_day_param: + srvc_ids_add = _select_calendar_dates_service_ids_by_day( + calendar_dates_df, msg, day) + + if has_date_param: + srvc_ids_add, srvc_ids_del = \ + _select_calendar_dates_service_ids_by_date( + calendar_dates_df, msg, date) + + if has_date_range_param: + srvc_ids_add, srvc_ids_del = \ + _select_calendar_dates_service_ids_by_date_range( + calendar_dates_df, msg, date_range) + + if has_cal_dates_param: + query_srvc_ids_add, query_srvc_ids_del = \ + _select_calendar_dates_str_match( + calendar_dates_df, msg, cal_dates_lookup) + # add the service_ids found from query to any existing + # active service_ids or inactive service_ids + merge_dict = {'add': [srvc_ids_add, query_srvc_ids_add], + 'del': [srvc_ids_del, query_srvc_ids_del]} + _add_exception_type_service_id_lists(merge_dict) + + return srvc_ids_add, srvc_ids_del + + +def _merge_service_ids_cal_dates_w_cal( + srvc_ids, srvc_ids_add, srvc_ids_del, verbose=True): + if verbose: + log(' Reconciling service_id(s) between calendar and ' + 'calendar_dates based on exception_type... ') + srvc_ids_add = list(set(srvc_ids_add)) + srvc_ids_add_cnt = len(srvc_ids_add) + srvc_ids_del = list(set(srvc_ids_del)) + srvc_ids_cnt_before_add = len(set(srvc_ids)) + + # add IDs that had exception_type = 1 in calendar_dates + srvc_ids.extend(srvc_ids_add) + active_srvc_ids = list(set(srvc_ids)) + active_srvc_ids_cnt_1 = len(active_srvc_ids) + + diff_cnt = abs(active_srvc_ids_cnt_1 - srvc_ids_cnt_before_add) + # TODO: clean up this section to make sure that the calculation of the + # subtraction works the same as the value difference shown in the print + # statements + if verbose: + msg_print = (' Adding {:,} service_id(s) to those found in ' + 'calendar. Total active service_id(s): {:,}.') + log(msg_print.format(diff_cnt, active_srvc_ids_cnt_1)) + # remove IDs that had exception_type = 2 in calendar_dates + active_srvc_ids = list(set(active_srvc_ids) - set(srvc_ids_del)) + active_srvc_ids_cnt_2 = len(active_srvc_ids) + if verbose: + msg_print = (' Removing {:,} service_id(s) from those found ' + 'in calendar. Total active service_id(s): {:,}.') + log(msg_print.format( + abs(active_srvc_ids_cnt_1 - active_srvc_ids_cnt_2), + active_srvc_ids_cnt_2)) + return active_srvc_ids + + +def _select_calendar_dates_str_match( + calendar_dates_df, msg, cal_dates_lookup=None): + # collect service IDs that match search parameters in calendar_dates.txt + log('{} that match calendar_date_lookup parameter(s) for ' + 'column(s) and string(s): {}...'.format(msg, cal_dates_lookup)) + srvc_ids = [] + for col_name_key, string_value in cal_dates_lookup.items(): + if not isinstance(string_value, list): + string_value = [string_value] + for text in string_value: + # TODO: support or/and condition + result_df = calendar_dates_df[ + calendar_dates_df[col_name_key].str.match( + text, case=False, na=False)] + result_srvc_ids = result_df['unique_service_id'].to_list() + result_srvc_ids = list(set(result_srvc_ids)) + result_cnt = len(result_srvc_ids) + log(' Found {:,} unique service_id(s) that matched query:' + ' column: {} and string: {}.'.format( + result_cnt, col_name_key, text)) + srvc_ids.extend(result_srvc_ids) + srvc_ids = list(set(srvc_ids)) + + subset_df = calendar_dates_df.loc[ + calendar_dates_df['unique_service_id'].isin(srvc_ids)] + srvc_ids_add, srvc_ids_del = _parse_cal_dates_exception_type(subset_df) + + if len(srvc_ids_add) == 0: + print_msg = (' Warning: No active service_ids were ' + 'found with query: {}.') + log(print_msg.format(cal_dates_lookup), level=lg.WARNING) + + return srvc_ids_add, srvc_ids_del + + +def _print_count_service_ids(df_dict, subset_ids): + # TODO: flip this around to have: + # {agency: {table: {org_count: n, proc_count: n}}}} + # if agency has records in both tables print differently + log(' In summary, active service IDs were derived from:') + unique_col = 'unique_service_id' + results = {'original_cnts': {}, 'processed_cnts': {}} + for table, df in df_dict.items(): + if not df.empty: + # only count unique IDs + unique_ids = df.drop_duplicates(subset=unique_col) + subset_df = unique_ids.loc[unique_ids[unique_col].isin(subset_ids)] + original_cnts = len(unique_ids) + processed_cnts = len(subset_df) + results['original_cnts'].update({table: original_cnts}) + results['processed_cnts'].update({table: processed_cnts}) + # NOTE: for feeds that use both calendar and + # calendar_dates tables, the sum between that agency's calendar + # count and calendar_dates count will not total to the sum of + # active service IDs given that calendar_dates IDs are used to add + # or remove IDs from the calendar table + start_count = results['original_cnts'][table] + end_count = results['processed_cnts'][table] + msg_print = ( + ' {:,} out of {:,} unique service_id(s) from {}.') + log(msg_print.format(end_count, start_count, table)) + + +def _calendar_service_id_selector( + calendar_df, + calendar_dates_df, + day=None, + date=None, + date_range=None, + cal_dates_lookup=None): + """ + Select trips that correspond to a specific schedule in either calendar.txt + and or calendar_dates.txt by finding service_ids that correspond to the + specified search parameters and the trips related to those service_ids + + Parameters + ---------- + calendar_df : pandas.DataFrame + calendar DataFrame + calendar_dates_df : pandas.DataFrame + calendar_dates DataFrame + day : {'monday', 'tuesday', 'wednesday', 'thursday', + 'friday', 'saturday', 'sunday'}, optional + day of the week to extract active service IDs in calendar and or + calendar_dates tables. If GTFS feeds have only calendar, service IDs + that are active on day specified will be extracted. If GTFS feeds + have only calendar_dates, service IDs that are active on dates + that match the day of the week specified will be extracted where + exception_type = 1 (service is added for date). If GTFS feeds have + both calendar and calendar_dates, calendar service IDs that are + active on day specified will be extracted and calendar_dates + service IDs that are active on dates that match the day of the week + specified will be added to those found in the calendar where + exception_type = 1 (service is added for date). + date : str, optional + date to extract active service IDs in calendar and or + calendar_dates tables specified as a string in YYYY-MM-DD format, + e.g. '2013-03-09'. If GTFS feeds have only calendar, service IDs + that are active on the date specified (where the date is within the + date range given by the start_date and end_date columns) and + that date's day of the week will be extracted. If GTFS feeds have + only calendar_dates, service IDs that are active on the date + specified will be extracted where exception_type = 1 + (service is added for date). If GTFS feeds have both calendar + and calendar_dates, calendar service IDs that are active on the date + specified (where the date is within the date range given by the + start_date and end_date columns) and that date's day of the week + will be extracted and calendar_dates service IDs that are active + on the date specified will be added to those found in the + calendar where exception_type = 1 (service is added for date) + and service IDs that are inactive on the date specified will be + removed from those found in the calendar where exception_type = 2 + (service is removed for date). + date_range : list, optional + date range to extract active service IDs in calendar and or + calendar_dates tables specified as a 2 element list of strings + where the first element is the first date and the second element + is the last date in the date range. Dates should be specified + as strings in YYYY-MM-DD format, e.g. ['2013-03-09', '2013-09-01']. + If GTFS feeds have only calendar, service IDs that are active + within the date range specified (where the date range is within + the date range given by the start_date and end_date columns) + will be extracted. If GTFS feeds have only calendar_dates, + service IDs that are active within the date range specified + will be extracted where exception_type = 1 + (service is added for date). If GTFS feeds have both calendar + and calendar_dates, calendar service IDs that are active within + the date range specified (where the date range is within the date + range given by the state_date and end_date columns) will be + extracted and calendar_dates service IDs that are active within + the date range specified will be added to those found in the + calendar where exception_type = 1 (service is added for date). + This parameter is best utilized if its known that the GTFS + feed calendar table has seasonal service IDs. + use_highest_freq_trips_date : boolean, optional + when True, the date that contains the highest number of active trips + will be determined and will be used to select active service IDs. + See the docstring for the 'date' parameter to understand how the + 'date' is then used to select service IDs. If multiple dates are found + to contain the highest number of trips, and they contain the same + trip IDs, one date from those found will be used. If they do not + contain the same trip IDs, an error will be raised to prompt you + to explicitly set one of the dates found as the date in the 'date' + parameter. + cal_dates_lookup : dict, optional + dictionary of the lookup column (key) as a string and corresponding + string (value) as string or list of strings to use to subset trips + using the calendar_dates DataFrame. Parameter should only be used if + there is a high degree of certainly that the GTFS feed in use has + service_ids listed in calendar_dates that would otherwise not be + selected if using any of the other calendar_dates parameters such as: + 'day', 'date' or 'date_range'. Search will be exact and will select + all records that meet each key value pair criteria. + Example: {'schedule_type' : 'WD'} or {'schedule_type' : ['WD', 'SU']} + + Returns + ------- + active_srvc_ids : list + list of strings of active unique service IDs that match the calendar + parameters to use to select active trip IDs + + """ + start_time = time.time() + + log('Processing active service_ids that match the specified ' + 'parameters for trip selection...') + + # check if calendar dfs and related params are empty or not to determine + # what will be used in processing + has_cal = calendar_df.empty is False + has_cal_dates = calendar_dates_df.empty is False + has_day_param = day is not None + has_date_param = date is not None + has_date_range_param = date_range is not None + has_cal_dates_param = cal_dates_lookup is not None + + params = {'day': day, + 'date': date, + 'date_range': date_range, + 'cal_dates_lookup': cal_dates_lookup, + 'has_cal': has_cal, + 'has_day_param': has_day_param, + 'has_date_param': has_date_param, + 'has_date_range_param': has_date_range_param, + 'has_cal_dates': has_cal_dates, + 'has_cal_dates_param': has_cal_dates_param} + + _trip_schedule_selector_validate_params(calendar_dates_df, params) + + # create unique service IDs for dfs in list if they are not empty + df_list = [] + if has_cal: + df_list.extend([calendar_df]) + if has_cal_dates: + df_list.extend([calendar_dates_df]) + df_list = _unique_service_id(df_list) + + if has_cal: + srvc_ids = _select_calendar_service_ids( + calendar_df, params) + + if has_cal_dates: + srvc_ids_add, srvc_ids_del = _select_calendar_dates_service_ids( + calendar_dates_df, params) + + if has_cal and has_cal_dates: + active_srvc_ids = _merge_service_ids_cal_dates_w_cal( + srvc_ids, srvc_ids_add, srvc_ids_del) + elif has_cal: + active_srvc_ids = srvc_ids.copy() + elif has_cal_dates: + active_srvc_ids = srvc_ids_add.copy() + active_srvc_ids_cnt = len(active_srvc_ids) + + if active_srvc_ids_cnt == 0: + raise ValueError( + 'No active service_ids were found matching the specified ' + 'parameters. No trips can be selected. Suggest modifying the ' + 'calendar parameters and or reviewing the GTFS calendar and or ' + 'calendar_dates tables.') + else: + _print_count_service_ids(df_dict={'calendar': calendar_df, + 'calendar_dates': calendar_dates_df}, + subset_ids=active_srvc_ids) + + msg_print = ( + '{:,} active service_id(s) were found that match the specified ' + 'parameters for trip selection. Took {:,.2f} seconds.') + log(msg_print.format(active_srvc_ids_cnt, time.time() - start_time)) + + return active_srvc_ids + + +def _trip_selector(trips_df, service_ids, verbose=True): + """ + Select trips that correspond to a specific schedule in either calendar.txt + and or calendar_dates.txt using the specified active service_ids + + Parameters + ---------- + trips_df : pandas.DataFrame + trips DataFrame + service_ids : list + list of strings that represent active service IDs that will be used + to select trips. Trips that correspond to these service IDs will + be exported and considered active during the specified schedule. + verbose : boolean, optional + if False, turns off logging and print statements in this function + + Returns + ------- + subset_trip_df : pandas.DataFrame + trip DataFrame comprised of trips that are active during the + specified schedule represented in the passed list of active service IDs + + """ + start_time = time.time() + if verbose: + log('--------------------------------') + log('Selecting trip_id(s) with active service_ids that match ' + 'the specified calendar and or calendar date parameters...') + df_list = _unique_service_id([trips_df]) + subset_trip_df = trips_df.loc[ + trips_df['unique_service_id'].isin(service_ids)] + + sort_columns = ['route_id', 'trip_id', 'direction_id'] + if 'direction_id' not in subset_trip_df.columns: + sort_columns.remove('direction_id') + subset_trip_df.sort_values(by=sort_columns, inplace=True) + subset_trip_df.reset_index(drop=True, inplace=True) + subset_trip_df.drop('unique_service_id', axis=1, inplace=True) + + tot_trip_cnt = len(trips_df) + tot_subset_trip_cnt = len(subset_trip_df) + tot_trip_pct = (tot_subset_trip_cnt / tot_trip_cnt) * 100 + + if tot_subset_trip_cnt == 0: + raise ValueError( + 'No trips were found that matched the active service_ids ' + 'identified by the specified calendar parameters. Suggest ' + 'modifying the calendar parameters and or reviewing the GTFS ' + 'trips and calendar and or calendar_dates tables.') + if verbose: + agency_id_col = 'unique_agency_id' + agency_ids = subset_trip_df[agency_id_col].unique() + for agency in agency_ids: + agency_trip_cnt = len(subset_trip_df.loc[ + subset_trip_df[agency_id_col] == agency]) + agency_tot_trip_cnt = len(trips_df.loc[ + trips_df[agency_id_col] == agency]) + agency_trip_pct = (agency_trip_cnt / agency_tot_trip_cnt) * 100 + + if agency_trip_cnt > 0: + msg = ('{:,} of {:,} ({:.2f} percent) of trips for agency: {} ' + 'were selected using service_ids that matched the ' + 'specified calendar parameters.') + log(msg.format(agency_trip_cnt, agency_tot_trip_cnt, + agency_trip_pct, agency)) + else: + msg = ('Warning: For agency: {}, no trips were found that ' + 'matched the active service_ids identified by the ' + 'specified calendar parameters. Suggest modifying the ' + 'calendar parameters and or reviewing the GTFS ' + 'trips and calendar and or calendar_dates tables.') + log(msg.format(agency), level=lg.WARNING) + + if len(agency_ids) != 1: + msg = ('In total: {:,} of {:,} ({:.2f} percent) of trips were ' + 'selected using service_ids that matched the specified ' + 'calendar parameters.') + log(msg.format(tot_subset_trip_cnt, tot_trip_cnt, tot_trip_pct)) + log('Took {:,.2f} seconds.'.format(time.time() - start_time)) + log('--------------------------------') + return subset_trip_df + + +def _highest_freq_trips_date(trips_df, calendar_df, calendar_dates_df): + start_time = time.time() + # get service ids for each date + # count the trips that have those service ids + # the date that has the max is the date to choose + # use select service ids by date with that date + log('Finding date with the most frequent trips...') + has_cal = calendar_df.empty is False + has_cal_dates = calendar_dates_df.empty is False + + df_list = [trips_df] + if has_cal: + df_list.extend([calendar_df]) + if has_cal_dates: + df_list.extend([calendar_dates_df]) + df_list = _unique_service_id(df_list) + + if has_cal: + # convert cols to datetime expected format: 'yyyymmdd' e.g.: '20130825' + date_cols = ['start_date', 'end_date'] + calendar_df = _cal_date_dt_conversion( + df=calendar_df, date_cols=date_cols) + date_ranges = _calendar_date_ranges(calendar_df, for_print=False) + date_list = [] + for date_range in date_ranges: + dates = pd.date_range( + date_range['start_date'], + date_range['end_date'], closed=None, freq='d') + dates_dt = dates.to_pydatetime() + dates_str = [date.strftime('%Y-%m-%d') for date in dates_dt] + date_list.extend(dates_str) + unique_date_list = set(date_list) + + date_srv_id_dict = {} + for date in unique_date_list: + srvc_ids_date = _select_calendar_service_ids_by_date( + calendar_df, msg='', date=date, verbose=False) + date_srv_id_dict.update({date: srvc_ids_date}) + + if has_cal_dates: + calendar_dates_df = _cal_date_dt_conversion( + df=calendar_dates_df, date_cols=['date']) + date_add_rmv_srv_id_dict = {} + for date in unique_date_list: + srvc_ids_add, srvc_ids_del = \ + _select_calendar_dates_service_ids_by_date( + calendar_dates_df, msg='', date=date, verbose=False) + date_add_rmv_srv_id_dict.update({date: { + 'add': srvc_ids_add, 'remove': srvc_ids_del}}) + + if has_cal and has_cal_dates: + for date in unique_date_list: + srvc_ids = date_srv_id_dict[date].copy() + srvc_ids_add = date_add_rmv_srv_id_dict[date]['add'] + srvc_ids_del = date_add_rmv_srv_id_dict[date]['remove'] + active_srvc_ids = _merge_service_ids_cal_dates_w_cal( + srvc_ids, srvc_ids_add, srvc_ids_del, verbose=False) + date_srv_id_dict[date] = active_srvc_ids.copy() + + # select the trips and count + trips_df['unique_trip_id'] = trips_df['trip_id'].str.cat( + trips_df['unique_agency_id'].astype('str'), sep='_') + date_trip_cnt = {} + for date in unique_date_list: + subset_trips_df = _trip_selector( + trips_df, service_ids=date_srv_id_dict[date], verbose=False) + trip_id_cnt = len(subset_trips_df['unique_trip_id'].to_list()) + date_trip_cnt.update({date: trip_id_cnt}) + + # find the date that has the max number of trips + max_date = max(date_trip_cnt, key=lambda k: date_trip_cnt[k]) + # get the max number of trips number + max_trips = max(date_trip_cnt.values()) + # check to see if there are more than 1 dates that have the same max number + # of trips value + dates_w_max_trips = [k for k, v in date_trip_cnt.items() if v == max_trips] + # get the service_ids for dates that share the max number of trips value + reduce_list = [date_srv_id_dict[x] for x in dates_w_max_trips] + # check to see if any of the dates have different service_ids from + # one another and make a list of the IDs that are not shared between dates + service_ids_not_shared = list(set.difference(*map(set, reduce_list))) + + time_msg = 'Took {:,.2f} seconds.'.format(time.time() - start_time) + if len(dates_w_max_trips) == 1: + msg = ( + 'The date with the most frequent trips ({:,} active trips): {} ' + 'will be used to select service ids. {}'.format( + max_date, max_trips, time_msg)) + log(msg) + # if there are more than 1 dates with the same max number of trips value, + # print informative messages and ValueError depending on if service_ids + # differ between the dates + else: + if len(service_ids_not_shared) == 0: + msg = ('The following dates: {} have identical service_ids ' + 'representing the date with the most frequent trips ' + '({:,} active trips). The service ids active on these dates' + ' will be used to select active trips. Selection of ' + 'service ids will be made using date: {}. {}') + log(msg.format( + sorted(dates_w_max_trips), max_trips, max_date, time_msg)) + # if there are different service_ids between the dates that have the + # same max number of trips value, throw error to make user explicitly + # pick one of the dates in the list to use since we cannot arbitrary + # pick one for them + else: + msg = ('The following dates: {} have different service ids ' + 'representing the date with the most frequent trips ' + '({:,} active trips). Unable to use the ' + '"use_highest_freq_trips_date" parameter. ' + 'Pick one date from this list as the "date" parameter ' + 'to proceed.') + raise ValueError(msg.format(sorted(dates_w_max_trips), max_trips)) + + # return the date of the max trip value instead of the date's service IDs + # in order to utilize the informative debugging prints in the + # _calendar_service_id_selector() function to be transparent in how IDs + # were selected and to help catch potential data issues + return max_date + + +def _unique_service_id(df_list): + for index, df in enumerate(df_list): + df['unique_service_id'] = (df['service_id'].str.cat( + df['unique_agency_id'].astype('str'), sep='_')) + df_list[index] = df + return df_list From c130aafe5a23e19b9e605a2d4e91681eac5f3989 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 31 Aug 2021 13:53:45 -0700 Subject: [PATCH 04/72] refactor to use new params and functions in utils_calendar.py for expanded options for service ID selection using cal and or cal dates tables with date, date range, high feq trip date --- urbanaccess/gtfs/network.py | 407 ++++++++++-------------------------- 1 file changed, 115 insertions(+), 292 deletions(-) diff --git a/urbanaccess/gtfs/network.py b/urbanaccess/gtfs/network.py index 35a3d18..f23ce84 100644 --- a/urbanaccess/gtfs/network.py +++ b/urbanaccess/gtfs/network.py @@ -7,6 +7,8 @@ from urbanaccess.utils import log, df_to_hdf5, hdf5_to_df from urbanaccess.gtfs.utils_validation import _check_time_range_format +from urbanaccess.gtfs.utils_calendar import _calendar_service_id_selector, \ + _trip_selector, _highest_freq_trips_date from urbanaccess.network import ua_network from urbanaccess import config from urbanaccess.gtfs.gtfsfeeds_dataframe import gtfsfeeds_dfs, \ @@ -17,8 +19,8 @@ def create_transit_net( gtfsfeeds_dfs, - day, - timerange, + day=None, + timerange=None, calendar_dates_lookup=None, overwrite_existing_stop_times_int=False, use_existing_stop_times_int=False, @@ -26,7 +28,11 @@ def create_transit_net( save_dir=config.settings.data_folder, save_filename=None, timerange_pad=None, - time_aware=False): + time_aware=False, + date=None, + date_range=None, + use_highest_freq_trips_date=False +): """ Create a travel time weight network graph in units of minutes from GTFS data @@ -38,9 +44,18 @@ def create_transit_net( stop_times, calendar, calendar_dates (optional) and stop_times_int (optional) day : {'monday', 'tuesday', 'wednesday', 'thursday', - 'friday', 'saturday', 'sunday'} - day of the week to extract transit schedule from that - corresponds to the day in the GTFS calendar + 'friday', 'saturday', 'sunday'}, optional + day of the week to extract active service IDs in calendar and or + calendar_dates tables. If GTFS feeds have only calendar, service IDs + that are active on day specified will be extracted. If GTFS feeds + have only calendar_dates, service IDs that are active on dates + that match the day of the week specified will be extracted where + exception_type = 1 (service is added for date). If GTFS feeds have + both calendar and calendar_dates, calendar service IDs that are + active on day specified will be extracted and calendar_dates + service IDs that are active on dates that match the day of the week + specified will be added to those found in the calendar where + exception_type = 1 (service is added for date). timerange : list time range to extract transit schedule from in a list with time 1 and time 2 as strings. It is suggested the time range @@ -52,10 +67,12 @@ def create_transit_net( calendar_dates_lookup : dict, optional dictionary of the lookup column (key) as a string and corresponding string (value) as string or list of strings to use to subset trips - using the calendar_dates DataFrame. Search will be exact. If none, - then the calendar_dates DataFrame will not be used to select trips - that are not in the calendar DataFrame. Note search will select all - records that meet each key value pair criteria. + using the calendar_dates DataFrame. Parameter should only be used if + there is a high degree of certainly that the GTFS feed in use has + service_ids listed in calendar_dates that would otherwise not be + selected if using any of the other calendar_dates parameters such as: + 'day', 'date' or 'date_range'. Search will be exact and will select + all records that meet each key value pair criteria. Example: {'schedule_type' : 'WD'} or {'schedule_type' : ['WD', 'SU']} overwrite_existing_stop_times_int : bool, optional if true, and if there is an existing stop_times_int @@ -83,6 +100,56 @@ def create_transit_net( from the stop_times table will be included in the transit edge table where 'departure_time' is the departure time at node_id_from stop and 'arrival_time' is the arrival time at node_id_to stop + date : str, optional + date to extract active service IDs in calendar and or + calendar_dates tables specified as a string in YYYY-MM-DD format, + e.g. '2013-03-09'. If GTFS feeds have only calendar, service IDs + that are active on the date specified (where the date is within the + date range given by the start_date and end_date columns) and + that date's day of the week will be extracted. If GTFS feeds have + only calendar_dates, service IDs that are active on the date + specified will be extracted where exception_type = 1 + (service is added for date). If GTFS feeds have both calendar + and calendar_dates, calendar service IDs that are active on the date + specified (where the date is within the date range given by the + start_date and end_date columns) and that date's day of the week + will be extracted and calendar_dates service IDs that are active + on the date specified will be added to those found in the + calendar where exception_type = 1 (service is added for date) + and service IDs that are inactive on the date specified will be + removed from those found in the calendar where exception_type = 2 + (service is removed for date). + date_range : list, optional + date range to extract active service IDs in calendar and or + calendar_dates tables specified as a 2 element list of strings + where the first element is the first date and the second element + is the last date in the date range. Dates should be specified + as strings in YYYY-MM-DD format, e.g. ['2013-03-09', '2013-09-01']. + If GTFS feeds have only calendar, service IDs that are active + within the date range specified (where the date range is within + the date range given by the start_date and end_date columns) + will be extracted. If GTFS feeds have only calendar_dates, + service IDs that are active within the date range specified + will be extracted where exception_type = 1 + (service is added for date). If GTFS feeds have both calendar + and calendar_dates, calendar service IDs that are active within + the date range specified (where the date range is within the date + range given by the state_date and end_date columns) will be + extracted and calendar_dates service IDs that are active within + the date range specified will be added to those found in the + calendar where exception_type = 1 (service is added for date). + This parameter is best utilized if its known that the GTFS + feed calendar table has seasonal service IDs. + use_highest_freq_trips_date : boolean, optional + when True, the date that contains the highest number of active trips + will be determined and will be used to select active service IDs. + See the docstring for the 'date' parameter to understand how the + 'date' is then used to select service IDs. If multiple dates are found + to contain the highest number of trips, and they contain the same + trip IDs, one date from those found will be used. If they do not + contain the same trip IDs, an error will be raised to prompt you + to explicitly set one of the dates found as the date in the 'date' + parameter. Returns ------- @@ -128,15 +195,44 @@ def create_transit_net( if 'direction_id' not in gtfsfeeds_dfs.trips.columns: columns.remove('direction_id') - # TODO: support use case where only calendar_dates is in use: make 'day' - # optional as None but require either day or calendar_dates_lookup - # to exist but both are not required - calendar_selected_trips_df = _trip_schedule_selector( - input_trips_df=gtfsfeeds_dfs.trips[columns], - input_calendar_df=gtfsfeeds_dfs.calendar, - input_calendar_dates_df=gtfsfeeds_dfs.calendar_dates, - day=day, - calendar_dates_lookup=calendar_dates_lookup) + # since data can be composed of multiple feeds and agencies, + # run calendar service ID operations on each individual agency to + # be transparent on what service IDs are being extracted from + # each agency + active_service_ids = [] + agency_id_col = 'unique_agency_id' + agency_ids = gtfsfeeds_dfs.trips[agency_id_col].unique() + for agency in agency_ids: + log('--------------------------------') + log('Running active service ID selection operation for ' + 'agency: {}'.format(agency)) + agency_trips = gtfsfeeds_dfs.trips.loc[ + gtfsfeeds_dfs.trips[agency_id_col] == agency] + agency_calendar = gtfsfeeds_dfs.calendar.loc[ + gtfsfeeds_dfs.calendar[agency_id_col] == agency] + agency_calendar_dates = gtfsfeeds_dfs.calendar_dates.loc[ + gtfsfeeds_dfs.calendar_dates[agency_id_col] == agency] + + if use_highest_freq_trips_date: + date = _highest_freq_trips_date( + agency_trips, + agency_calendar, + agency_calendar_dates) + + agency_active_service_ids = _calendar_service_id_selector( + calendar_df=agency_calendar, + calendar_dates_df=agency_calendar_dates, + day=day, + date=date, + date_range=date_range, + cal_dates_lookup=calendar_dates_lookup) + # combine each agency's active service IDs + active_service_ids.extend(agency_active_service_ids) + + # use aggregated active_service_ids to get all active trip IDs + calendar_selected_trips_df = _trip_selector( + trips_df=gtfsfeeds_dfs.trips[columns], + service_ids=active_service_ids) # proceed to calc stop_times_int if stop_times_int is already empty, or # overwrite existing is True, or use existing is False @@ -198,279 +294,6 @@ def create_transit_net( return ua_network -def _trip_schedule_selector(input_trips_df, input_calendar_df, - input_calendar_dates_df, day, - calendar_dates_lookup=None): - """ - Select trips that correspond to a specific schedule in either calendar.txt - and or calendar_dates.txt by finding service_ids that correspond to the - specified search parameters and the trips related to those service_ids - - Parameters - ---------- - input_trips_df : pandas.DataFrame - trips DataFrame - input_calendar_df : pandas.DataFrame - calendar DataFrame - input_calendar_dates_df : pandas.DataFrame - calendar_dates DataFrame - day : {'monday', 'tuesday', 'wednesday', 'thursday', - 'friday', 'saturday', 'sunday'} - day of the week to extract transit schedule that corresponds to the - day in the GTFS calendar - calendar_dates_lookup : dict, optional - dictionary of the lookup column (key) as a string and corresponding - string (value) as string or list of strings to use to subset trips - using the calendar_dates DataFrame. Search will be exact. If none, - then the calendar_dates DataFrame will not be used to select trips - that are not in the calendar DataFrame. Note search will select all - records that meet each key value pair criteria. - Example: {'schedule_type' : 'WD'} or {'schedule_type' : ['WD', 'SU']} - - Returns - ------- - calendar_selected_trips_df : pandas.DataFrame - - """ - start_time = time.time() - - valid_days = ['monday', 'tuesday', 'wednesday', 'thursday', - 'friday', 'saturday', 'sunday'] - - if day not in valid_days: - valid_days_str = str(valid_days).replace('[', '').replace(']', '') - raise ValueError('Incorrect day specified. Must be one of lowercase ' - 'strings: {}.'.format(valid_days_str)) - - # check format of calendar_dates_lookup - if calendar_dates_lookup is not None: - if not isinstance(calendar_dates_lookup, dict): - raise ValueError( - 'calendar_dates_lookup parameter must be a dictionary.') - for key in calendar_dates_lookup.keys(): - if not isinstance(key, str): - raise ValueError('calendar_dates_lookup key: {} ' - 'must be a string.'.format(key)) - - if isinstance(calendar_dates_lookup[key], str): - value = [calendar_dates_lookup[key]] - else: - if not isinstance(calendar_dates_lookup[key], list): - raise ValueError( - 'calendar_dates_lookup value: {} must be a string or ' - 'a list of strings.'.format( - calendar_dates_lookup[key])) - else: - value = calendar_dates_lookup[key] - - for string in value: - if not isinstance(string, str): - raise ValueError('calendar_dates_lookup value: {} ' - 'must contain strings.'.format(value)) - - # check if calendar dfs and related params are empty or not to determine - # what will be used in processing - has_calendar = input_calendar_df.empty is False - has_calendar_param = day is not None - has_calendar_dates = input_calendar_dates_df.empty is False - has_calendar_dates_param = calendar_dates_lookup is not None - - if not has_calendar: - log('calendar table has no records and will not be used to ' - 'select trips.') - if has_calendar_param: - log("Warning: calendar is empty. " - "Unable to use the 'day' parameter.", level=lg.WARNING) - if has_calendar_dates: - if not has_calendar_dates_param: - log("calendar_dates table has records however the " - "'calendar_dates_lookup' parameter is None, no trips will be " - "selected using calendar_dates.") - else: - log('calendar_dates table has no records and will not be used to ' - 'select trips.') - if has_calendar_dates_param: - raise ValueError("calendar_dates is empty. Unable to use the " - "'calendar_dates_lookup' parameter. Set to None.") - - # create unique service IDs for dfs in list if they are not empty - df_list = [input_trips_df] - if has_calendar: - df_list.extend([input_calendar_df]) - if has_calendar_dates: - df_list.extend([input_calendar_dates_df]) - for index, df in enumerate(df_list): - df['unique_service_id'] = (df['service_id'].str.cat( - df['unique_agency_id'].astype('str'), sep='_')) - df_list[index] = df - - service_ids_df = pd.DataFrame() - - # collect service IDs that match search parameters in calendar.txt - if has_calendar and has_calendar_param: - # select service IDs where day specified has a 1 = service - # runs on that day - log('Using calendar to extract service_ids to select trips...') - service_ids_df = input_calendar_df[(input_calendar_df[day] == 1)] - service_ids_df = service_ids_df[['unique_service_id']] - num_cal_service_ids_extracted = len(service_ids_df) - log('{:,} service_ids were extracted from calendar.'.format( - num_cal_service_ids_extracted)) - - # generate information needed to tell user the status of their trips in - # terms of service_ids in calendar table - trips_in_calendar = input_trips_df.loc[ - input_trips_df['unique_service_id'].isin( - service_ids_df['unique_service_id'])] - trips_notin_calendar = input_trips_df.loc[ - ~input_trips_df['unique_service_id'].isin( - service_ids_df['unique_service_id'])] - cnt_input_trips_df = len(input_trips_df) - cnt_trips_in_calendar = len(trips_in_calendar) - pct_trips_in_calendar = round(cnt_trips_in_calendar / len( - input_trips_df) * 100, 2) - - feeds_wtrips_in_cal = trips_in_calendar['unique_feed_id'].unique() - print_feed_ids = [' '.join(feed_id.split('_')[:-1]) for feed_id in - feeds_wtrips_in_cal] - feeds_wotrips_in_cal = trips_notin_calendar['unique_feed_id'].unique() - if print_feed_ids: - log('{:,} trip(s) {:.2f} percent of {:,} total trip records were ' - 'found in calendar for GTFS feed(s): {}.'.format( - cnt_trips_in_calendar, pct_trips_in_calendar, - cnt_input_trips_df, print_feed_ids)) - - feed_id_not_in_cal = [x for x in feeds_wotrips_in_cal if - x not in feeds_wtrips_in_cal] - for feed_id in feed_id_not_in_cal: - trip_feed_name = ' '.join(feed_id.split('_')[:-1]) - log('0 trip(s) 0 percent of {:,} total trip records were ' - 'found in calendar for GTFS feed: {}.'.format( - cnt_input_trips_df, trip_feed_name)) - - # warn user that if they have a calendar_dates table and they - # expected more trips to be selected from the calendar table that - # they should consider using the calendar_dates table to supplement - # the selection of trips - if has_calendar_dates and len(trips_notin_calendar) > 0 and \ - has_calendar_dates_param is False: - warning_msg = ( - 'NOTE: If you expected more trips to have been extracted and ' - 'your GTFS feed(s) have a calendar_dates file, consider ' - 'utilizing the calendar_dates_lookup parameter in order to ' - 'add additional trips based on information inside of ' - 'calendar_dates. This should only be done if you know the ' - 'corresponding GTFS feed is using calendar_dates instead of ' - 'calendar to specify service_ids. When in doubt do not use ' - 'the calendar_dates_lookup parameter.') - log(warning_msg, level=lg.WARNING) - - if len(feeds_wtrips_in_cal) != len(feeds_wotrips_in_cal) and \ - calendar_dates_lookup is None: - for feed_id in feeds_wotrips_in_cal: - trip_feed_name = ' '.join(feed_id.split('_')[:-1]) - log('{:,} trip(s) {:.2f} percent of {:,} total trip records ' - 'were not found in calendar for GTFS feed: {}.'.format( - cnt_trips_in_calendar, pct_trips_in_calendar, - cnt_input_trips_df, trip_feed_name)) - if feed_id not in feeds_wtrips_in_cal: - log('Warning: GTFS feed: {} no trips were selected using ' - 'calendar. It is suggested you use the ' - 'calendar_dates_lookup parameter to utilize this ' - 'feed\'s calendar_dates file.'.format(trip_feed_name), - level=lg.WARNING) - else: - num_cal_service_ids_extracted = 0 - cnt_input_trips_df = 0 - - # collect service IDs that match search parameters in calendar_dates.txt - if has_calendar_dates and has_calendar_dates_param: - # look for service_ids inside of calendar_dates if calendar does not - # supply enough service_ids to select trips by - if has_calendar: - if len(trips_notin_calendar) > 0: - log('Using calendar_dates to supplement service_ids extracted ' - 'from calendar to select trips...') - - subset_result_df = pd.DataFrame() - - for col_name_key, string_value in calendar_dates_lookup.items(): - if col_name_key not in input_calendar_dates_df.columns: - raise ValueError('Column: {} not found in calendar_dates ' - 'DataFrame.'.format(col_name_key)) - if col_name_key not in input_calendar_dates_df.select_dtypes( - include=[object]).columns: - raise ValueError('Column: {} must be object type.'.format( - col_name_key)) - - if not isinstance(string_value, list): - string_value = [string_value] - - for text in string_value: - # TODO: modify this in order to allow subset based on GTFS - # feed name or a or/and condition - subset_result = input_calendar_dates_df[ - input_calendar_dates_df[col_name_key].str.match( - text, case=False, na=False)] - cnt_subset_result = len(subset_result) - if cnt_subset_result != 0: - feed_id_list = subset_result['unique_feed_id'].unique() - for index, id in enumerate(feed_id_list): - feed_id_list[index] = ' '.join(id.split('_')[:-1]) - - log('Found {:,} record(s) that matched query: column: {} ' - 'and string: {} for GTFS feed(s): {}.'.format( - cnt_subset_result, col_name_key, text, feed_id_list)) - - subset_result_df = subset_result_df.append(subset_result) - - subset_result_df.drop_duplicates(inplace=True) - subset_result_df = subset_result_df[['unique_service_id']] - - num_caldates_service_ids_extracted = len(subset_result_df) - tot_service_ids_extracted = \ - num_caldates_service_ids_extracted + num_cal_service_ids_extracted - log('An additional {:,} service_id(s) were extracted from ' - 'calendar_dates. Total service_id(s) extracted: {:,}.'.format( - num_caldates_service_ids_extracted, tot_service_ids_extracted)) - service_ids_df = service_ids_df.append(subset_result_df) - service_ids_df.drop_duplicates(inplace=True) - - if service_ids_df.empty: - raise ValueError('No service_id(s) were found with ' - 'the specified calendar and or calendar_dates ' - 'search parameters.') - - # select and create df of trips that match the service IDs for the day of - # the week specified merge calendar df that has service IDs for - # specified day with trips df - calendar_selected_trips_df = input_trips_df.loc[ - input_trips_df['unique_service_id'].isin( - service_ids_df['unique_service_id'])] - - sort_columns = ['route_id', 'trip_id', 'direction_id'] - if 'direction_id' not in calendar_selected_trips_df.columns: - sort_columns.remove('direction_id') - calendar_selected_trips_df.sort_values(by=sort_columns, inplace=True) - calendar_selected_trips_df.reset_index(drop=True, inplace=True) - calendar_selected_trips_df.drop('unique_service_id', axis=1, inplace=True) - - calendar_selected_trips_count = len(calendar_selected_trips_df) - if calendar_dates_lookup is None: - log('{:,} of {:,} total trips were extracted representing calendar ' - 'day: {}. Took {:,.2f} seconds.'.format( - calendar_selected_trips_count, cnt_input_trips_df, day, - time.time() - start_time)) - else: - log('{:,} of {:,} total trips were extracted representing calendar ' - 'day: {} and calendar_dates search parameters: {}. ' - 'Took {:,.2f} seconds.'.format( - calendar_selected_trips_count, cnt_input_trips_df, day, - calendar_dates_lookup, time.time() - start_time)) - - return calendar_selected_trips_df - - def _interpolate_stop_times(stop_times_df, calendar_selected_trips_df): """ Interpolate missing stop times using a linear From dbbb2dea4ce07f4a107450e5e01db5dff4181603 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 31 Aug 2021 13:54:53 -0700 Subject: [PATCH 05/72] minor code readability update --- urbanaccess/gtfs/network.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/urbanaccess/gtfs/network.py b/urbanaccess/gtfs/network.py index f23ce84..6a42667 100644 --- a/urbanaccess/gtfs/network.py +++ b/urbanaccess/gtfs/network.py @@ -236,9 +236,9 @@ def create_transit_net( # proceed to calc stop_times_int if stop_times_int is already empty, or # overwrite existing is True, or use existing is False - if gtfsfeeds_dfs.stop_times_int.empty or \ - overwrite_existing_stop_times_int or use_existing_stop_times_int \ - is False: + existing_stop_times_int = gtfsfeeds_dfs.stop_times_int.empty + if existing_stop_times_int or overwrite_existing_stop_times_int or \ + use_existing_stop_times_int is False: if overwrite_existing_stop_times_int: log(' Overwriting existing stop_times_int DataFrame...') gtfsfeeds_dfs.stop_times_int = _interpolate_stop_times( From ef62f22c5a26c8e625718b7c6ee92a87f1b16448 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Thu, 2 Sep 2021 11:48:54 -0700 Subject: [PATCH 06/72] add transit network simplification param and function --- urbanaccess/gtfs/network.py | 86 ++++++++++++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/urbanaccess/gtfs/network.py b/urbanaccess/gtfs/network.py index 6a42667..212cce4 100644 --- a/urbanaccess/gtfs/network.py +++ b/urbanaccess/gtfs/network.py @@ -31,7 +31,8 @@ def create_transit_net( time_aware=False, date=None, date_range=None, - use_highest_freq_trips_date=False + use_highest_freq_trips_date=False, + simplify=False ): """ Create a travel time weight network graph in units of @@ -150,6 +151,13 @@ def create_transit_net( contain the same trip IDs, an error will be raised to prompt you to explicitly set one of the dates found as the date in the 'date' parameter. + simplify : boolean + when True, the transit network is significantly reduced in size by + collapsing trips that share identical characteristics including + agency, travel time, stop sequence, and route into one unique + representative trip with its corresponding edges. It is suggested to + use simplification when using networks with network analysis tools + such as Pandana for computational efficiency. Default is False. Returns ------- @@ -280,6 +288,14 @@ def create_transit_net( transit_edges = _route_id_to_edge( transit_edge_df=transit_edges, trips_df=gtfsfeeds_dfs.trips) + transit_nodes = _remove_nodes_not_in_edges( + nodes=transit_nodes, edges=transit_edges, + from_id_col='node_id_from', to_id_col='node_id_to') + + if simplify: + transit_edges, transit_nodes = _simplify_transit_net( + transit_edges, transit_nodes) + # assign node and edge net type transit_nodes['net_type'] = 'transit' transit_edges['net_type'] = 'transit' @@ -294,6 +310,68 @@ def create_transit_net( return ua_network +def _simplify_transit_net(edges, nodes): + # TODO: add option for groups that are duplicates, place trip ids + # of the groups that are removed in a new column as list of + # strings for debugging + start_time = time.time() + + log('Running transit network simplification...') + + # columns to use for group to group value comparison to identify + # groups of similar values for simplification + col_list = ['node_id_from', 'node_id_to', 'weight', 'unique_agency_id', + 'sequence', 'route_type', 'unique_route_id'] + + # group records that have identical values for each col in col_list + # tag each record in group as True if the group exists more than once + id_col = 'unique_trip_id' + edges_wdup = edges.groupby(id_col)[col_list].agg(tuple).sum(1).duplicated() + + # remove records where their group was duplicated + simp_edges = edges.loc[edges[id_col].isin(edges_wdup[~edges_wdup].index)] + + # simplify nodes by removing nodes that do not exist in the simplified + # edge table, this catches edges cases but normally there should never be + # node ids in the node table that are not in the edge table + simp_nodes = _remove_nodes_not_in_edges( + nodes=nodes, edges=simp_edges, + from_id_col='node_id_from', to_id_col='node_id_to') + + # calculate various data len and print statistics + trip_remove_cnt = len(edges_wdup.loc[edges_wdup == True]) + trip_org_tot_cnt = len(edges[id_col].unique()) + trip_proc_tot_cnt = len(simp_edges[id_col].unique()) + trip_remove_pct = (trip_remove_cnt / trip_org_tot_cnt) * 100 + edge_org_rec_tot_cnt = len(edges) + edge_proc_rec_tot_cnt = len(simp_edges) + edge_remove_cnt = edge_org_rec_tot_cnt - edge_proc_rec_tot_cnt + edge_remove_pct = (edge_remove_cnt / edge_org_rec_tot_cnt) * 100 + msg = ('Transit edges have been simplified removing {:,} trip(s) ' + '({:.2f} percent) (reduced from {:,} to {:,} trip(s)) ' + 'resulting in the removal of {:,} edge(s) ({:.2f} percent) ' + '(reduced from {:,} to {:,} edges(s)).') + log(msg.format(trip_remove_cnt, trip_remove_pct, + trip_org_tot_cnt, trip_proc_tot_cnt, + edge_remove_cnt, edge_remove_pct, + edge_org_rec_tot_cnt, edge_proc_rec_tot_cnt)) + + node_org_tot_cnt = len(nodes) + node_proc_tot_cnt = len(simp_nodes) + # if nodes have a different count after simplification notify the user, + # however this should never occur + if node_org_tot_cnt != node_proc_tot_cnt: + node_remove_cnt = node_org_tot_cnt - node_proc_tot_cnt + node_remove_pct = (node_remove_cnt / node_org_tot_cnt) * 100 + msg = ('Transit nodes have been simplified removing {:,} nodes(s) ' + '({:.2f} percent) (reduced from {:,} to {:,} nodes(s)).') + log(msg.format(node_remove_cnt, node_remove_pct, node_org_tot_cnt, + node_proc_tot_cnt)) + log('Transit edge simplification complete. ' + 'Took {:,.2f} seconds.'.format(time.time() - start_time)) + return simp_edges, simp_nodes + + def _interpolate_stop_times(stop_times_df, calendar_selected_trips_df): """ Interpolate missing stop times using a linear @@ -1182,3 +1260,9 @@ def _check_if_index_name_in_cols(df): cols = df.columns.values iname = df.index.name return (iname in cols) + + +def _remove_nodes_not_in_edges(nodes, edges, from_id_col, to_id_col): + edge_node_list = set(edges[from_id_col]) | set(edges[to_id_col]) + nodes_df = nodes.loc[nodes.index.isin(edge_node_list)] + return nodes_df From 4c1b488e781e0412fd0e22e43bb72a4a2caec0a4 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Fri, 10 Sep 2021 12:42:39 -0700 Subject: [PATCH 07/72] convert agency_id col to str before running str operator --- urbanaccess/gtfs/utils_format.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/urbanaccess/gtfs/utils_format.py b/urbanaccess/gtfs/utils_format.py index a0e8b4f..48379df 100644 --- a/urbanaccess/gtfs/utils_format.py +++ b/urbanaccess/gtfs/utils_format.py @@ -495,11 +495,13 @@ def _add_unique_agencyid(agency_df, stops_df, routes_df, # replace nans with '' agency_df[cols] = agency_df[cols].fillna('') # determine extent of missing values - all_missing = (agency_df['agency_id'].str.isspace().all() and - agency_df['agency_name'].str.isspace().all()) or \ + agency_id_str = agency_df['agency_id'].astype(str) + agency_name_str = agency_df['agency_name'].astype(str) + all_missing = (agency_id_str.str.isspace().all() and + agency_name_str.str.isspace().all()) or \ (agency_df[cols].values == '').all() - any_missing = (agency_df['agency_id'].str.isspace().any() and - agency_df['agency_name'].str.isspace().any()) or \ + any_missing = (agency_id_str.str.isspace().any() and + agency_name_str.str.isspace().any()) or \ (agency_df[cols].values == '').any() # if 'agency_name' and 'agency_id' both have no records or both contain From ef097c74966976d6c371e0c909f67f07b87ede4c Mon Sep 17 00:00:00 2001 From: sablanchard Date: Fri, 10 Sep 2021 12:57:29 -0700 Subject: [PATCH 08/72] refactor _add_unique_gtfs_id() to match df dict structure of _add_unique_agencyid() and add empty 'unique_agency_id' if cal/cal dates is empty --- urbanaccess/gtfs/utils_format.py | 38 +++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/urbanaccess/gtfs/utils_format.py b/urbanaccess/gtfs/utils_format.py index 48379df..a808ae1 100644 --- a/urbanaccess/gtfs/utils_format.py +++ b/urbanaccess/gtfs/utils_format.py @@ -633,13 +633,15 @@ def _add_unique_agencyid(agency_df, stops_df, routes_df, feed_folder_name)) df_dict[name] = df + # TODO: this dict may be redundant check if can refactor optional_df_dict = { 'calendar': calendar_df, 'calendar_dates': calendar_dates_df} # if optional calendar or calendar_dates_df are empty then return - # the original empty df + # the original empty df with an empty agency ID column for name, df in optional_df_dict.items(): if df.empty: + df['unique_agency_id'] = np.nan df_dict.update({name: df}) # ensure returned items in list are always in the same expected order @@ -684,22 +686,38 @@ def _add_unique_gtfsfeed_id(stops_df, routes_df, trips_df, """ start_time = time.time() - df_list = [stops_df, routes_df, trips_df, stop_times_df, calendar_df] - # if calendar_dates_df is not empty then add it to the processing list - if calendar_dates_df.empty is False: - df_list.extend([calendar_dates_df]) + df_dict = {'stops': stops_df, + 'routes': routes_df, + 'trips': trips_df, + 'stop_times': stop_times_df} + + optional_df_dict = {'calendar': calendar_df, + 'calendar_dates': calendar_dates_df} + # if optional calendar or calendar_dates_df are not empty then add it + # to the processing list + for name, df in optional_df_dict.items(): + if df.empty is False: + df_dict.update({name: df}) # standardize feed_folder name feed_folder = _generate_unique_feed_id(feed_folder) - for index, df in enumerate(df_list): + for name, df in df_dict.items(): # create new unique_feed_id column based on the name of the feed folder df['unique_feed_id'] = '_'.join([feed_folder, str(feed_number)]) - df_list[index] = df + df_dict[name] = df - # if calendar_dates_df is empty then return the original empty df - if calendar_dates_df.empty: - df_list.extend([calendar_dates_df]) + # if optional calendar or calendar_dates_df are empty then return + # the original empty df with an empty agency ID column + for name, df in optional_df_dict.items(): + if df.empty: + df['unique_feed_id'] = np.nan + df_dict.update({name: df}) + + # ensure returned items in list are always in the same expected order + df_list = [df_dict['stops'], df_dict['routes'], df_dict['trips'], + df_dict['stop_times'], df_dict['calendar'], + df_dict['calendar_dates']] log('Unique GTFS feed ID operation complete. ' 'Took {:,.2f} seconds.'.format(time.time() - start_time)) From f27d6e922dbf7f513c0594be86eedd0ca508f132 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Fri, 10 Sep 2021 13:19:30 -0700 Subject: [PATCH 09/72] fix cases where request results in certificate verify failure --- urbanaccess/gtfsfeeds.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/urbanaccess/gtfsfeeds.py b/urbanaccess/gtfsfeeds.py index 2ab110d..7be1290 100644 --- a/urbanaccess/gtfsfeeds.py +++ b/urbanaccess/gtfsfeeds.py @@ -5,6 +5,7 @@ import os import logging as lg import time +import ssl from six.moves.urllib import request from urbanaccess.utils import log @@ -474,6 +475,9 @@ def download(data_folder=os.path.join(config.settings.data_folder), start_time2 = time.time() zipfile_path = ''.join([download_folder, '/', feed_name_key, '.zip']) + # resolve issues where request results in certificate verify failure + ssl._create_default_https_context = ssl._create_unverified_context + # add default user-agent header in request to avoid 403 Errors opener = request.build_opener() opener.addheaders = [('User-agent', '')] From bf0c38f1d7bcf7de9204d2e85c30ad30da685bc9 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Fri, 10 Sep 2021 13:21:55 -0700 Subject: [PATCH 10/72] ''.join() -> os.path.join() --- urbanaccess/gtfsfeeds.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/urbanaccess/gtfsfeeds.py b/urbanaccess/gtfsfeeds.py index 7be1290..2d8cd02 100644 --- a/urbanaccess/gtfsfeeds.py +++ b/urbanaccess/gtfsfeeds.py @@ -473,7 +473,8 @@ def download(data_folder=os.path.join(config.settings.data_folder), # TODO: add file counter and print number to user for feed_name_key, feed_url_value in feeds.gtfs_feeds.items(): start_time2 = time.time() - zipfile_path = ''.join([download_folder, '/', feed_name_key, '.zip']) + zipfile_path = os.path.join(download_folder, '{}.zip'.format( + feed_name_key)) # resolve issues where request results in certificate verify failure ssl._create_default_https_context = ssl._create_unverified_context @@ -529,8 +530,8 @@ def download(data_folder=os.path.join(config.settings.data_folder), file = request.urlopen(feed_url_value) _zipfile_type_check(file=file, feed_url_value=feed_url_value) - file_path = ''.join( - [download_folder, '/', feed_name_key, '.zip']) + file_path = os.path.join(download_folder, '{}.zip'.format( + feed_name_key)) with open(file_path, "wb") as local_file: local_file.write(file.read()) log(msg_download_succeed.format( From 9d5e7e48b1532409befe6fa5ad65e20694f7f2f9 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Fri, 10 Sep 2021 13:23:38 -0700 Subject: [PATCH 11/72] add default param as true to raise error if request fails --- urbanaccess/gtfsfeeds.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/urbanaccess/gtfsfeeds.py b/urbanaccess/gtfsfeeds.py index 2d8cd02..6d50171 100644 --- a/urbanaccess/gtfsfeeds.py +++ b/urbanaccess/gtfsfeeds.py @@ -388,7 +388,7 @@ def search(api='gtfsdataexch', search_text=None, search_field=None, def download(data_folder=os.path.join(config.settings.data_folder), feed_name=None, feed_url=None, feed_dict=None, - error_pause_duration=5, delete_zips=False): + error_pause_duration=5, delete_zips=False, raise_url_error=True): """ Connect to the URLs passed in function or the URLs stored in the urbanaccess_gtfsfeeds instance and download the GTFS feed zipfile(s) @@ -412,6 +412,11 @@ def download(data_folder=os.path.join(config.settings.data_folder), how long to pause in seconds before re-trying requests if error delete_zips : bool, optional if true the downloaded zipfiles will be removed + raise_url_error : bool, optional + if true error will be raised when a request to a URL fails, URL + failure response will be returned, else if false URL failures will + still be printed to logs but will fail silently to allow process + to continue. Returns ------- nothing @@ -517,6 +522,10 @@ def download(data_folder=os.path.join(config.settings.data_folder), log(msg_no_connection_w_status.format( feed_url_value, status_code), level=lg.ERROR) + if raise_url_error: + raise Exception( + msg_no_connection_w_status.format( + feed_url_value, status_code)) else: log(msg_no_connection_w_status.format( feed_url_value, status_code), @@ -525,6 +534,10 @@ def download(data_folder=os.path.join(config.settings.data_folder), log(msg_no_connection.format( feed_url_value, traceback.format_exc()), level=lg.ERROR) + if raise_url_error: + raise Exception( + msg_no_connection.format( + feed_url_value, traceback.format_exc())) else: try: file = request.urlopen(feed_url_value) @@ -541,6 +554,10 @@ def download(data_folder=os.path.join(config.settings.data_folder), log(msg_no_connection.format( feed_url_value, traceback.format_exc()), level=lg.ERROR) + if raise_url_error: + raise Exception( + msg_no_connection.format( + feed_url_value, traceback.format_exc())) log('GTFS feed download completed. Took {:,.2f} seconds'.format( time.time() - start_time1)) From 997cdb2c5c1f88658eee03c95a05a500e426f835 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Fri, 10 Sep 2021 13:25:02 -0700 Subject: [PATCH 12/72] update _zipfile_type_check() docstring, add TODO --- urbanaccess/gtfsfeeds.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/urbanaccess/gtfsfeeds.py b/urbanaccess/gtfsfeeds.py index 6d50171..0011ff3 100644 --- a/urbanaccess/gtfsfeeds.py +++ b/urbanaccess/gtfsfeeds.py @@ -633,7 +633,7 @@ def _zipfile_type_check(file, feed_url_value): Parameters ---------- - file : addinfourl + file : http.client.HTTPResponse loaded zipfile object in memory feed_url_value : str URL to download GTFS feed zipfile @@ -642,6 +642,9 @@ def _zipfile_type_check(file, feed_url_value): ------- nothing """ + # TODO: check for a better method to support cases that are zips but + # dont have header info example: sanfordcommunityredevelopmentagency + # GTFS feed URL if 'zip' not in file.info().get('Content-Type') is True \ or 'octet' not in file.info().get('Content-Type') is True: raise ValueError( From 6a9667bb7c61281903b4573f7b6cf06f18f747d3 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Fri, 10 Sep 2021 13:32:10 -0700 Subject: [PATCH 13/72] handle cases where there are multiple gtfs feed zips inside a single zip, minor formatting, make _zip() _> zip() to expose it to api --- urbanaccess/gtfsfeeds.py | 104 ++++++++++++++++++++++++++++----------- 1 file changed, 75 insertions(+), 29 deletions(-) diff --git a/urbanaccess/gtfsfeeds.py b/urbanaccess/gtfsfeeds.py index 0011ff3..4dbd706 100644 --- a/urbanaccess/gtfsfeeds.py +++ b/urbanaccess/gtfsfeeds.py @@ -512,10 +512,8 @@ def download(data_folder=os.path.join(config.settings.data_folder), time.sleep(error_pause_duration) try: file = request.urlopen(feed_url_value) - - _zipfile_type_check(file=file, - feed_url_value=feed_url_value) - + _zipfile_type_check( + file=file, feed_url_value=feed_url_value) with open(zipfile_path, "wb") as local_file: local_file.write(file.read()) except Exception: @@ -562,10 +560,54 @@ def download(data_folder=os.path.join(config.settings.data_folder), log('GTFS feed download completed. Took {:,.2f} seconds'.format( time.time() - start_time1)) - _unzip(zip_rootpath=download_folder, delete_zips=delete_zips) + unzip(zip_rootpath=download_folder, delete_zips=delete_zips) + + +def _unzip_util(zipfile_read_path, unzip_file_path, has_subzips): + with zipfile.ZipFile(zipfile_read_path) as z: + # required to deal with zipfiles that have subdirectories and + # that were created on OSX + filelist = [file for file in z.namelist() if + file.endswith(".txt") and not file.startswith( + "__MACOSX")] + # in cases where the zip contains multiple zips of GTFS feeds + # such as the case with the SEPTA GTFS feed + sub_zip_filelist = [file for file in z.namelist() if + file.endswith(".zip")] + if len(filelist) == 0 and len(sub_zip_filelist) > 0: + msg = ('Zipfile contains {:,} zipfiles: {}. These will be ' + 'extracted as separate feeds.') + log(msg.format(len(sub_zip_filelist), sub_zip_filelist)) + extract_subzips = True + else: + extract_subzips = False + + if has_subzips: + if not os.path.exists(unzip_file_path): + os.makedirs(unzip_file_path) + for file in filelist: + file_path = os.path.join( + unzip_file_path, os.path.basename(file)) + with open(file_path, 'wb') as f: + f.write(z.read(file)) + f.close() + else: + if extract_subzips: + filelist = sub_zip_filelist.copy() + if not os.path.exists(unzip_file_path): + os.makedirs(unzip_file_path) + for file in filelist: + file_path = os.path.join( + unzip_file_path, os.path.basename(file)) + with open(file_path, 'wb') as f: + f.write(z.read(file)) + f.close() + z.close() + if not has_subzips: + return sub_zip_filelist -def _unzip(zip_rootpath, delete_zips=True): +def unzip(zip_rootpath, delete_zips=True): """ unzip all GTFS feed zipfiles in a root directory with resulting text files in the root folder: gtfsfeed_text @@ -598,33 +640,37 @@ def _unzip(zip_rootpath, delete_zips=True): 'directory: {}'.format(zip_rootpath)) for zfile in zipfilelist: + unzipfile_name = zfile.replace('.zip', '') + unzip_file_path = os.path.join(unzip_rootpath, unzipfile_name) + zipfile_read_path = os.path.join(zip_rootpath, zfile) + + sub_zip_filelist = _unzip_util(zipfile_read_path, unzip_file_path, + has_subzips=False) + if len(sub_zip_filelist) != 0: + for sub_zipfile in sub_zip_filelist: + sub_zipfile_path = os.path.join(unzip_file_path, sub_zipfile) + # make unzip dir name unique in case others exist + sub_unzipfile_name = '{}_{}'.format( + unzipfile_name, sub_zipfile.replace('.zip', '')) + sub_unzipfile_path = os.path.join( + os.path.split(unzip_file_path)[0], sub_unzipfile_name) + _unzip_util(sub_zipfile_path, sub_unzipfile_path, + has_subzips=True) + msg = ('{} with nested zipfile: {} successfully extracted ' + 'to: {}') + log(msg.format(zfile, sub_zipfile, sub_unzipfile_path)) + os.remove(sub_zipfile_path) # remove the nested zip + os.rmdir(unzip_file_path) # remove the dir that had the nested zip + else: + log('{} successfully extracted to: {}'.format( + zfile, unzip_file_path)) - with zipfile.ZipFile(os.path.join(zip_rootpath, zfile)) as z: - # required to deal with zipfiles that have subdirectories and - # that were created on OSX - filelist = [file for file in z.namelist() if - file.endswith(".txt") and not file.startswith( - "__MACOSX")] - if not os.path.exists( - os.path.join(unzip_rootpath, zfile.replace('.zip', ''))): - os.makedirs( - os.path.join(unzip_rootpath, zfile.replace('.zip', ''))) - for file in filelist: - with open( - os.path.join(unzip_rootpath, zfile.replace('.zip', ''), - os.path.basename(file)), 'wb') as f: - f.write(z.read(file)) - f.close() - z.close() - log('{} successfully extracted to: {}'.format(zfile, os.path.join( - unzip_rootpath, zfile.replace('.zip', '')))) if delete_zips: os.remove(zip_rootpath) log('Deleted {} folder'.format(zip_rootpath)) - log( - 'GTFS feed zipfile extraction completed. Took {:,.2f} seconds for {} ' - 'files'.format( - time.time() - start_time, len(zipfilelist))) + msg = ('GTFS feed zipfile extraction completed. Took {:,.2f} seconds ' + 'for {:,} file(s)') + log(msg.format(time.time() - start_time, len(zipfilelist))) def _zipfile_type_check(file, feed_url_value): From d050c0b932bc32c94a2bd36f9f457d2396b1fc6a Mon Sep 17 00:00:00 2001 From: sablanchard Date: Fri, 8 Oct 2021 12:27:51 -0700 Subject: [PATCH 14/72] add TODO and warn level to log print --- urbanaccess/gtfs/utils_calendar.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/urbanaccess/gtfs/utils_calendar.py b/urbanaccess/gtfs/utils_calendar.py index b559228..b2bee8f 100644 --- a/urbanaccess/gtfs/utils_calendar.py +++ b/urbanaccess/gtfs/utils_calendar.py @@ -114,6 +114,8 @@ def _trip_schedule_selector_validate_params(calendar_dates_df, params): def _cal_date_dt_conversion(df, date_cols): + # TODO: add a try except section here to capture calendar format issues + # and return in user friendly message for col in date_cols: df[col] = pd.to_datetime( df[col], format='%y%m%d', infer_datetime_format=True) @@ -743,11 +745,12 @@ def _calendar_service_id_selector( active_srvc_ids_cnt = len(active_srvc_ids) if active_srvc_ids_cnt == 0: - raise ValueError( - 'No active service_ids were found matching the specified ' + msg = ( + 'Warning: No active service_ids were found matching the specified ' 'parameters. No trips can be selected. Suggest modifying the ' 'calendar parameters and or reviewing the GTFS calendar and or ' 'calendar_dates tables.') + log(msg, level=lg.WARNING) else: _print_count_service_ids(df_dict={'calendar': calendar_df, 'calendar_dates': calendar_dates_df}, From 0884f65f20e98ccb2e2611227b6130a5774bc4cb Mon Sep 17 00:00:00 2001 From: sablanchard Date: Fri, 8 Oct 2021 12:42:51 -0700 Subject: [PATCH 15/72] raise error when final edges or stop_times len are 0 --- urbanaccess/gtfs/network.py | 40 ++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/urbanaccess/gtfs/network.py b/urbanaccess/gtfs/network.py index 212cce4..d0cbf2c 100644 --- a/urbanaccess/gtfs/network.py +++ b/urbanaccess/gtfs/network.py @@ -292,6 +292,16 @@ def create_transit_net( nodes=transit_nodes, edges=transit_edges, from_id_col='node_id_from', to_id_col='node_id_to') + if transit_edges.empty: + msg = ( + "Resulting transit network edges have 0 record(s). Most likely " + "the 'timerange' and or calendar selection parameters resulted in " + "a network with no active trips within the schedule window " + "specified. It is suggested to expand the 'timerange' and or " + "alter the calendar selection parameters to enlarge the time " + "window for selecting active trips.") + raise ValueError(msg) + if simplify: transit_edges, transit_nodes = _simplify_transit_net( transit_edges, transit_nodes) @@ -313,11 +323,18 @@ def create_transit_net( def _simplify_transit_net(edges, nodes): # TODO: add option for groups that are duplicates, place trip ids # of the groups that are removed in a new column as list of - # strings for debugging + # strings for debugging or trip counts + # TODO: can simplify even more by reducing along edge shared attr in + # addition to trips, ensure that stop ids are connected start_time = time.time() log('Running transit network simplification...') + if edges.empty: + raise ValueError( + 'Unable to simplify transit network. Edges have 0 records to ' + 'simplify.') + # columns to use for group to group value comparison to identify # groups of similar values for simplification col_list = ['node_id_from', 'node_id_to', 'weight', 'unique_agency_id', @@ -351,10 +368,13 @@ def _simplify_transit_net(edges, nodes): '({:.2f} percent) (reduced from {:,} to {:,} trip(s)) ' 'resulting in the removal of {:,} edge(s) ({:.2f} percent) ' '(reduced from {:,} to {:,} edges(s)).') - log(msg.format(trip_remove_cnt, trip_remove_pct, - trip_org_tot_cnt, trip_proc_tot_cnt, - edge_remove_cnt, edge_remove_pct, - edge_org_rec_tot_cnt, edge_proc_rec_tot_cnt)) + if edge_remove_cnt > 0: + log(msg.format(trip_remove_cnt, trip_remove_pct, + trip_org_tot_cnt, trip_proc_tot_cnt, + edge_remove_cnt, edge_remove_pct, + edge_org_rec_tot_cnt, edge_proc_rec_tot_cnt)) + else: + log('Transit edges cannot be further simplified.') node_org_tot_cnt = len(nodes) node_proc_tot_cnt = len(simp_nodes) @@ -736,6 +756,16 @@ def _format_transit_net_edge(stop_times_df, time_aware=False): log('Starting transformation process for {:,} ' 'total trips...'.format(len(stop_times_df['unique_trip_id'].unique()))) + if stop_times_df.empty: + raise ValueError( + "Unable to continue processing transit network. stop_times " + "table has 0 records. Most likely the 'timerange' and or " + "calendar selection parameters resulted in " + "a network with no active trips within the schedule window " + "specified. It is suggested to expand the 'timerange' and or " + "alter the calendar selection parameters to enlarge the time " + "window for selecting active trips.") + # subset to only columns needed for processing cols_of_interest = ['unique_trip_id', 'stop_id', 'unique_stop_id', 'timediff', 'stop_sequence', 'unique_agency_id', From 26428d03d644b0f7d0247d82eff415c60380ff13 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 26 Oct 2021 12:00:12 -0700 Subject: [PATCH 16/72] during _read_gtfs_file() convert dtype after whitespace fix --- urbanaccess/gtfs/utils_format.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/urbanaccess/gtfs/utils_format.py b/urbanaccess/gtfs/utils_format.py index a808ae1..cfc4c61 100644 --- a/urbanaccess/gtfs/utils_format.py +++ b/urbanaccess/gtfs/utils_format.py @@ -92,6 +92,15 @@ def _read_gtfs_file(textfile_path, textfile): df = _remove_whitespace( df=df, textfile=textfile, col_list=remove_whitespace) + # convert dtypes here in case col name was incorrect due to + # whitespace when read in pd.read_csv() + if req_dtypes is not None: + for col_name, dtype in req_dtypes.items(): + if df[col_name].dtype != dtype: + if dtype == object: + dtype = str # force to string instead of object + df[col_name] = df[col_name].astype(dtype) + if numeric_converter is not None: for col in numeric_converter: df[col] = pd.to_numeric(df[col]) From 994a4b18f05e3d7409d365ba84f6bc1c2be79aa6 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 26 Oct 2021 13:01:12 -0700 Subject: [PATCH 17/72] fix typos, add docstrings --- urbanaccess/gtfs/utils_calendar.py | 476 ++++++++++++++++++++++++++--- urbanaccess/gtfs/utils_format.py | 2 +- 2 files changed, 430 insertions(+), 48 deletions(-) diff --git a/urbanaccess/gtfs/utils_calendar.py b/urbanaccess/gtfs/utils_calendar.py index b2bee8f..37249ee 100644 --- a/urbanaccess/gtfs/utils_calendar.py +++ b/urbanaccess/gtfs/utils_calendar.py @@ -8,6 +8,22 @@ def _trip_schedule_selector_validate_params(calendar_dates_df, params): + """ + Validate parameters passed to the _calendar_service_id_selector() + function. + + Parameters + ---------- + calendar_dates_df : pandas.DataFrame + Calendar dates DataFrame + params : dict + Parameters to validate as a dict + + Returns + ------- + Nothing + + """ has_cal = params['has_cal'] has_cal_dates = params['has_cal_dates'] @@ -38,7 +54,7 @@ def _trip_schedule_selector_validate_params(calendar_dates_df, params): if day not in valid_days: valid_days_str = str(valid_days).replace('[', '').replace(']', '') raise ValueError( - 'Day: {} is not a supported day. Must be one of lowercase ' + 'Day: {} is not a supported day. Must be one of lowercase ' 'strings: {}.'.format(day, valid_days_str)) if has_date_param: @@ -48,7 +64,7 @@ def _trip_schedule_selector_validate_params(calendar_dates_df, params): dt.strptime(date, '%Y-%m-%d') except ValueError: raise ValueError("Date: {} is not a supported date format. " - "Expected format: 'YYYY-MM-DD'".format(date)) + "Expected format: 'YYYY-MM-DD'.".format(date)) if has_date_range_param: if len(date_range) != 2: @@ -62,7 +78,7 @@ def _trip_schedule_selector_validate_params(calendar_dates_df, params): except ValueError: raise ValueError("Date: {} in date range: {} is not a " "supported date format. Expected format: " - "'YYYY-MM-DD'".format(date_item, date_range)) + "'YYYY-MM-DD'.".format(date_item, date_range)) item_1 = dt.strptime(date_range[0], '%Y-%m-%d') item_2 = dt.strptime(date_range[1], '%Y-%m-%d') if item_1 > item_2: @@ -99,8 +115,8 @@ def _trip_schedule_selector_validate_params(calendar_dates_df, params): 'DataFrame.'.format(col_name_key)) if col_name_key not in calendar_dates_df.select_dtypes( include=[object]).columns: - raise ValueError('Column: {} must be object type.'.format( - col_name_key)) + raise ValueError('Calendar_dates column: {} must be ' + 'object type.'.format(col_name_key)) else: raise ValueError("Calendar_dates is empty. Unable to use the " "'calendar_dates_lookup' parameter. Set to None.") @@ -114,8 +130,26 @@ def _trip_schedule_selector_validate_params(calendar_dates_df, params): def _cal_date_dt_conversion(df, date_cols): - # TODO: add a try except section here to capture calendar format issues - # and return in user friendly message + """ + Convert columns with dates as strings to datetime64[ns] format. Expected + incoming dates in format:'%y%m%d' (YYYY-MM-DD) + + Parameters + ---------- + df : pandas.DataFrame + DataFrame that contains date columns to convert to DateTime + format. Typically this will be the calendar DataFrame + date_cols : list + List of strings denoting date columns with string values to convert + to DateTime format. For example: ['start_date', 'end_date'] + + Returns + ------- + df : pandas.DataFrame + Returns the same passed df DataFrame with the columns specified in + date_cols converted to datetime64[ns] format. + + """ for col in date_cols: df[col] = pd.to_datetime( df[col], format='%y%m%d', infer_datetime_format=True) @@ -123,6 +157,25 @@ def _cal_date_dt_conversion(df, date_cols): def _select_calendar_service_ids(calendar_df, params): + """ + Wrapper function to select service IDs from the calendar.txt that are + active according to the parameters specified + + Parameters + ---------- + calendar_df : pandas.DataFrame + Calendar DataFrame + params : dict + Parameters denoting how service IDs should be selected from table + and supporting information to guide the selection process + + Returns + ------- + srvc_ids : list + returns a list of service IDs from the calendar.txt that are + active according to the parameters specified + + """ # collect service IDs that match search parameters in calendar.txt msg = 'Selecting service_ids from calendar' @@ -164,6 +217,22 @@ def _select_calendar_service_ids(calendar_df, params): def _print_cal_service_ids_len(srvc_ids, table_name): + """ + Helper function to print counts of service IDs + + Parameters + ---------- + srvc_ids : list + list of selected service IDs to generate counts for print message + table_name : str + name of table where service IDs were selected from to include + in print message + + Returns + ------- + Nothing + + """ cnt_msg = ' {:,} service_id(s) were selected from {}.' warning_msg = ' Warning: No service_ids were selected from {}.' srvc_ids_cnt = len(srvc_ids) @@ -176,6 +245,27 @@ def _print_cal_service_ids_len(srvc_ids, table_name): def _intersect_cal_service_ids(dict_1, dict_2, verbose=True): + """ + Return the intersection of two lists of service IDs that are set as + the values in two dicts + + Parameters + ---------- + dict_1 : dict + dict 1 where the key is the origin of the service IDs for informative + purposes and the value is a list of service IDs. + Example: {'date': srvc_ids_date} + dict_2 : dict + dict 2 where the key is the origin of the service IDs for informative + purposes and the value is a list of service IDs. + Example: {'day': srvc_ids_date} + verbose : boolean, optional + if False, turns off logging and print statements in this function + Returns + ------- + srvc_ids : list + list of intersected and unique selected service IDs + """ srvc_id_1_name, srvc_id_1 = list(dict_1.items())[0] srvc_id_2_name, srvc_id_2 = list(dict_2.items())[0] srvc_ids = list(set(srvc_id_1) & set(srvc_id_2)) @@ -194,9 +284,26 @@ def _intersect_cal_service_ids(dict_1, dict_2, verbose=True): def _select_calendar_service_ids_by_day( calendar_df, msg, day=None, verbose=True): - # Global search using: day in calendar.txt - # select service IDs where day specified = 1 where - # 1 = service is active; 0 = service inactive + """ + Global search using day in calendar.txt to select service IDs where + day specified = 1 where 1 = service is active; 0 = service inactive + + Parameters + ---------- + calendar_df : pandas.DataFrame + Calendar DataFrame + msg : str + string to prepend to print statement for informative purposes + day : {'monday', 'tuesday', 'wednesday', 'thursday', + 'friday', 'saturday', 'sunday'} + day of the week to extract active service IDs in calendar. + verbose : boolean, optional + if False, turns off logging and print statements in this function + Returns + ------- + srvc_ids : list + list of active service IDs + """ if verbose: log(' {} that are active on day: {}...'.format(msg, day)) subset_cal_df = calendar_df.loc[calendar_df[day] == 1] @@ -241,7 +348,7 @@ def _select_calendar_service_ids_by_date_range( print_msg = ( ' Date range: {} contains the following date ranges ' 'available in the calendar. Date ranges available ' - 'are:: \n {}'.format(date_range, date_rngs)) + 'are: \n {}'.format(date_range, date_rngs)) log(print_msg) _print_cal_service_ids_len(srvc_ids, table_name='calendar') @@ -249,6 +356,35 @@ def _select_calendar_service_ids_by_date_range( def _calendar_date_ranges(calendar_df, for_print=True): + """ + Helper function to return date ranges inside of the calendar.txt table + using the columns 'start_date' and 'end_date'. Returns date ranges + in string format to be used in print statements and in pd.Timestamp + format to be used in downstream functions. + + Parameters + ---------- + calendar_df : pandas.DataFrame + Calendar DataFrame + for_print : boolean, optional + If True, returns date ranges in string format to be used in print + statements where each element takes format: + '{start timestamp as string} to {start timestamp as string}'. + If False, returns date ranges in pd.Timestamp format as a + list of dicts where keys are the 'start_date' and + 'end_date' for each range: {'start_date': pd.Timestamp, + 'end_date': pd.Timestamp}. + Returns + ------- + date_rngs : list + If for_print is True, returns date ranges in string format to be used + in print statements where each element takes format: + '{start timestamp as string} to {end timestamp as string}'. + If False, returns date ranges in pd.Timestamp format as a + list of dicts where keys are the 'start_date' and + 'end_date' for each range: {'start_date': pd.Timestamp, + 'end_date': pd.Timestamp}. + """ date_cols = ['start_date', 'end_date'] unique_range = calendar_df[date_cols].drop_duplicates(subset=date_cols) # build list of ranges that exist in table to print to user @@ -268,13 +404,32 @@ def _calendar_date_ranges(calendar_df, for_print=True): def _select_calendar_service_ids_by_date( calendar_df, msg, date=None, verbose=True): - # Global search using: date to subset records in calendar.txt - # using 'start_date' and 'end_date' columns when GTFS feed has - # seasonal service_ids - # select service IDs where date is within the 'start_date' and - # 'end_date' and then select IDs that are active on the day of - # the week the date represents + """ + Global search using date to subset records in calendar.txt + using 'start_date' and 'end_date' columns when GTFS feed has + seasonal service_ids, select service IDs where date is within the + 'start_date' and 'end_date' and then select IDs that are active on + the day of the week the date represents + Parameters + ---------- + calendar_df : pandas.DataFrame + Calendar DataFrame + msg : str + string to prepend to print statement for informative purposes + date : str + date to extract active service IDs in calendar table specified as + a string in YYYY-MM-DD format, e.g. '2013-03-09'. service IDs + that are active on the date specified (where the date is within the + date range given by the start_date and end_date columns) and + that date's day of the week will be extracted. + verbose : boolean, optional + if False, turns off logging and print statements in this function + Returns + ------- + srvc_ids : list + list of active service IDs + """ # convert date to day of the week day = dt.strptime(date, '%Y-%m-%d').strftime('%A').lower() @@ -331,6 +486,23 @@ def _select_calendar_service_ids_by_date( def _parse_cal_dates_exception_type(cal_dates_df, verbose=True): + """ + Helper function to parse selected calendar_dates.txt service IDs + from unique_service_id column by exception_type + + Parameters + ---------- + cal_dates_df : pandas.DataFrame + Calendar dates DataFrame + verbose : boolean, optional + if False, turns off logging and print statements in this function + Returns + ------- + srvc_ids_add : list + list of selected service IDs to add + srvc_ids_del : list + list of selected service IDs to remove + """ # 1 - Service has been added for the specified date. # 2 - Service has been removed for the specified date. id_col = 'unique_service_id' @@ -354,6 +526,30 @@ def _parse_cal_dates_exception_type(cal_dates_df, verbose=True): def _select_calendar_dates_service_ids_by_day( calendar_dates_df, msg, day=None): + """ + Select service IDs in calendar_dates.txt that are active on dates + that match the day of the week specified will be where + exception_type = 1 (service is added for date). + + Parameters + ---------- + calendar_dates_df : pandas.DataFrame + Calendar dates DataFrame + msg : str + string to prepend to print statement for informative purposes + day : {'monday', 'tuesday', 'wednesday', 'thursday', + 'friday', 'saturday', 'sunday'} + day of the week to extract service IDs in calendar_dates table. + Service IDs that are active on dates that match the day of the week + specified will be extracted where exception_type = 1 (service is added + for date). + + Returns + ------- + srvc_ids : list + list of active service IDs + """ + # select all service ids that are active on this day of week, # that are inclusive @@ -372,6 +568,33 @@ def _select_calendar_dates_service_ids_by_day( def _select_calendar_dates_service_ids_by_date( calendar_dates_df, msg, date=None, verbose=True): + """ + Search using date to subset records in calendar_dates.txt + using 'date' column to select service IDs regardless of exception_type + + Parameters + ---------- + calendar_dates_df : pandas.DataFrame + Calendar dates DataFrame + msg : str + string to prepend to print statement for informative purposes + date : str + date to extract service IDs in calendar dates table specified as + a string in YYYY-MM-DD format, e.g. '2013-03-09'. calendar_dates + service IDs that are active within the date range specified will + be selected where exception_type = 1 + (service is added for date) and inactive within + the date range specified will be selected where exception_type = 2 + (service is removed for date). + verbose : boolean, optional + if False, turns off logging and print statements in this function + Returns + ------- + srvc_ids_add : list + list of selected service IDs to add + srvc_ids_del : list + list of selected service IDs to remove + """ # select all that are on this day, that are inclusive # remove from those found in calendar that are exclusive if verbose: @@ -392,7 +615,37 @@ def _select_calendar_dates_service_ids_by_date( def _select_calendar_dates_service_ids_by_date_range( calendar_dates_df, msg, date_range=None, verbose=True): - # select all service ids that are active within date_range + """ + Search using date_range to subset records in calendar_dates.txt + using the 'date' column to select service IDs within the date range + regardless of exception_type + + Parameters + ---------- + calendar_dates_df : pandas.DataFrame + Calendar dates DataFrame + msg : str + string to prepend to print statement for informative purposes + date_range : list + date range to extract active service IDs in calendar_dates table + specified as a 2 element list of strings where the first element + is the first date and the second element is the last date in the date + range. Dates should be specified as strings in YYYY-MM-DD format, + e.g. ['2013-03-09', '2013-09-01']. + calendar_dates service IDs that are active within + the date range specified will be selected where exception_type = 1 + (service is added for date) and inactive within + the date range specified will be selected where exception_type = 2 + (service is removed for date). + verbose : boolean, optional + if False, turns off logging and print statements in this function + Returns + ------- + srvc_ids_add : list + list of selected service IDs to add + srvc_ids_del : list + list of selected service IDs to remove + """ if verbose: log(' {} that are active within date_range: {}...'.format( msg, date_range)) @@ -418,6 +671,26 @@ def _select_calendar_dates_service_ids_by_date_range( def _add_exception_type_service_id_lists(srvc_ids_list_dict): + """ + Helper function to combine selected calendar_dates.txt service IDs + together by exception_type that have been selected with different + parameters + + Parameters + ---------- + srvc_ids_list_dict : dict + dict with keys 'add' (exception_type = 1) and 'del' + (exception_type = 2) with values as a list of lists of service IDs + for example: + {'add': [['service ID 1, 'service ID 2], ['service ID 3']], + {'del': [['service ID 4, 'service ID 5], ['service ID 6']]} + Returns + ------- + srvc_ids_add : list + list of combined selected service IDs to add + srvc_ids_del : list + list of combined selected service IDs to remove + """ print_dict = {} for list_type, srvc_id_lists in srvc_ids_list_dict.items(): srvc_ids_cnt_1 = len(srvc_id_lists[0]) @@ -447,6 +720,26 @@ def _add_exception_type_service_id_lists(srvc_ids_list_dict): def _select_calendar_dates_service_ids(calendar_dates_df, params): + """ + Wrapper function to select service IDs from the calendar_dates.txt + that are active according to the parameters specified + + Parameters + ---------- + calendar_dates_df : pandas.DataFrame + Calendar dates DataFrame + params : dict + Parameters denoting how service IDs should be selected from table + and supporting information to guide the selection process + + Returns + ------- + srvc_ids_add : list + list of selected service IDs to add + srvc_ids_del : list + list of selected service IDs to remove + + """ # collect service IDs that match search parameters in calendar_dates.txt msg = ' Selecting service_ids from calendar_dates' @@ -497,6 +790,28 @@ def _select_calendar_dates_service_ids(calendar_dates_df, params): def _merge_service_ids_cal_dates_w_cal( srvc_ids, srvc_ids_add, srvc_ids_del, verbose=True): + """ + Helper function to combine selected calendar.txt and calendar_dates.txt + service IDs together where service IDs from calendar dates table with + exception type 1 are added to those found in calendar table and those + with exception type 2 are removed from those found in calendar table + + Parameters + ---------- + srvc_ids : list + list of selected service IDs from calendar table + srvc_ids_add : list + list of selected service IDs to add from calendar dates table + srvc_ids_del : list + list of selected service IDs to remove from calendar dates table + verbose : boolean, optional + if False, turns off logging and print statements in this function + + Returns + ------- + active_srvc_ids : list + list of combined selected service IDs + """ if verbose: log(' Reconciling service_id(s) between calendar and ' 'calendar_dates based on exception_type... ') @@ -531,7 +846,34 @@ def _merge_service_ids_cal_dates_w_cal( def _select_calendar_dates_str_match( - calendar_dates_df, msg, cal_dates_lookup=None): + calendar_dates_df, msg, cal_dates_lookup): + """ + Selects service IDs from calendar_dates.txt that match the specified + query composed of a dict of column name and string value selection key + value pair criteria. Search will be exact and will select all records + that meet each key value pair criteria. + + Parameters + ---------- + calendar_dates_df : pandas.DataFrame + Calendar dates DataFrame + msg : str + message to prepend to print statement in function for informative + purposes + cal_dates_lookup : dict + dictionary of the lookup column (key) as a string and corresponding + string (value) as string or list of strings to use to subset service + IDs using the calendar_dates DataFrame. Search will be exact and will + select all records that meet each key value pair criteria. + Example: {'schedule_type' : 'WD'} or {'schedule_type' : ['WD', 'SU']} + + Returns + ------- + srvc_ids_add : list + list of selected service IDs to add + srvc_ids_del : list + list of selected service IDs to remove + """ # collect service IDs that match search parameters in calendar_dates.txt log('{} that match calendar_date_lookup parameter(s) for ' 'column(s) and string(s): {}...'.format(msg, cal_dates_lookup)) @@ -566,6 +908,23 @@ def _select_calendar_dates_str_match( def _print_count_service_ids(df_dict, subset_ids): + """ + Helper function to provide information to user on the results of the + active service ID selection process by printing counts of service IDs + in tables prior and post-processing. + + Parameters + ---------- + df_dict : dict + dict where key is a string denoting the name of the dataframe to use + for prints and the value is a pandas.DataFrame containing the + original DataFrame + subset_ids : list + list of processed active service IDs + Returns + ------- + Nothing + """ # TODO: flip this around to have: # {agency: {table: {org_count: n, proc_count: n}}}} # if agency has records in both tables print differently @@ -664,16 +1023,6 @@ def _calendar_service_id_selector( calendar where exception_type = 1 (service is added for date). This parameter is best utilized if its known that the GTFS feed calendar table has seasonal service IDs. - use_highest_freq_trips_date : boolean, optional - when True, the date that contains the highest number of active trips - will be determined and will be used to select active service IDs. - See the docstring for the 'date' parameter to understand how the - 'date' is then used to select service IDs. If multiple dates are found - to contain the highest number of trips, and they contain the same - trip IDs, one date from those found will be used. If they do not - contain the same trip IDs, an error will be raised to prompt you - to explicitly set one of the dates found as the date in the 'date' - parameter. cal_dates_lookup : dict, optional dictionary of the lookup column (key) as a string and corresponding string (value) as string or list of strings to use to subset trips @@ -681,8 +1030,8 @@ def _calendar_service_id_selector( there is a high degree of certainly that the GTFS feed in use has service_ids listed in calendar_dates that would otherwise not be selected if using any of the other calendar_dates parameters such as: - 'day', 'date' or 'date_range'. Search will be exact and will select - all records that meet each key value pair criteria. + 'day', 'date' or 'date_range'. Search will be exact and will select + all records that meet each key value pair criteria. Example: {'schedule_type' : 'WD'} or {'schedule_type' : ['WD', 'SU']} Returns @@ -848,11 +1197,30 @@ def _trip_selector(trips_df, service_ids, verbose=True): def _highest_freq_trips_date(trips_df, calendar_df, calendar_dates_df): + """ + Counts the number of trips active on each calendar day of active service + and returns the date with the maximum number of trips. + + Parameters + ---------- + trips_df : pandas.DataFrame + trips DataFrame + calendar_df : pandas.DataFrame + calendar DataFrame + calendar_dates_df : pandas.DataFrame + calendar dates DataFrame + + Returns + ------- + max_date : str + date as a string where the maximum number of trips occur in a schedule + + """ start_time = time.time() # get service ids for each date # count the trips that have those service ids # the date that has the max is the date to choose - # use select service ids by date with that date + # use to select service ids by date with the date found here log('Finding date with the most frequent trips...') has_cal = calendar_df.empty is False has_cal_dates = calendar_dates_df.empty is False @@ -931,21 +1299,20 @@ def _highest_freq_trips_date(trips_df, calendar_df, calendar_dates_df): time_msg = 'Took {:,.2f} seconds.'.format(time.time() - start_time) if len(dates_w_max_trips) == 1: - msg = ( - 'The date with the most frequent trips ({:,} active trips): {} ' - 'will be used to select service ids. {}'.format( - max_date, max_trips, time_msg)) + msg = ("The date with the most frequent trips ({:,} active trips): {} " + "will be used to select service ids. {}".format( + max_trips, max_date, time_msg)) log(msg) # if there are more than 1 dates with the same max number of trips value, # print informative messages and ValueError depending on if service_ids # differ between the dates else: if len(service_ids_not_shared) == 0: - msg = ('The following dates: {} have identical service_ids ' - 'representing the date with the most frequent trips ' - '({:,} active trips). The service ids active on these dates' - ' will be used to select active trips. Selection of ' - 'service ids will be made using date: {}. {}') + msg = ("The following dates: {} have identical service_ids " + "representing the date with the most frequent trips " + "({:,} active trips). The service ids active on these dates" + " will be used to select active trips. Selection of " + "service ids will be made using date: {}. {}") log(msg.format( sorted(dates_w_max_trips), max_trips, max_date, time_msg)) # if there are different service_ids between the dates that have the @@ -953,12 +1320,12 @@ def _highest_freq_trips_date(trips_df, calendar_df, calendar_dates_df): # pick one of the dates in the list to use since we cannot arbitrary # pick one for them else: - msg = ('The following dates: {} have different service ids ' - 'representing the date with the most frequent trips ' - '({:,} active trips). Unable to use the ' - '"use_highest_freq_trips_date" parameter. ' - 'Pick one date from this list as the "date" parameter ' - 'to proceed.') + msg = ("The following dates: {} have different service ids " + "representing the date with the most frequent trips " + "({:,} active trips). Unable to use the " + "'use_highest_freq_trips_date' parameter. " + "Pick one date from this list as the 'date' parameter " + "to proceed.") raise ValueError(msg.format(sorted(dates_w_max_trips), max_trips)) # return the date of the max trip value instead of the date's service IDs @@ -969,6 +1336,21 @@ def _highest_freq_trips_date(trips_df, calendar_df, calendar_dates_df): def _unique_service_id(df_list): + """ + Create 'unique_service_id' column and values for a list of + pandas.DataFrames + + Parameters + ---------- + df_list : list + list of pandas.DataFrames to generate 'unique_service_id' column for + + Returns + ------- + df_list : list + list of pandas.DataFrames with 'unique_service_id' column added + + """ for index, df in enumerate(df_list): df['unique_service_id'] = (df['service_id'].str.cat( df['unique_agency_id'].astype('str'), sep='_')) diff --git a/urbanaccess/gtfs/utils_format.py b/urbanaccess/gtfs/utils_format.py index cfc4c61..ef6c7ad 100644 --- a/urbanaccess/gtfs/utils_format.py +++ b/urbanaccess/gtfs/utils_format.py @@ -413,7 +413,7 @@ def _add_unique_agencyid(agency_df, stops_df, routes_df, tables; 4) If GTFS feed has an agency.txt file and it has more than one agency (it must have an 'agency_id' and 'agency_name' column and values), however if there is also a mismatch between the 'agency_id' - in aganecy.txt and routes.txt, then assume records tied to the mismatched + in agency.txt and routes.txt, then assume records tied to the mismatched 'agency_id'(s) are from multiple agencies and label the unique agency ID as such by concatenating 'multiple_operators_' and the GTFS feed directory folder name. From fc872632549c863b9d3d9390853f434844cb9026 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 26 Oct 2021 13:02:12 -0700 Subject: [PATCH 18/72] add check for use_highest_freq_trips_date param in use with other cal params --- urbanaccess/gtfs/network.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/urbanaccess/gtfs/network.py b/urbanaccess/gtfs/network.py index 212cce4..6441e58 100644 --- a/urbanaccess/gtfs/network.py +++ b/urbanaccess/gtfs/network.py @@ -193,6 +193,11 @@ def create_transit_net( if overwrite_existing_stop_times_int and use_existing_stop_times_int: raise ValueError('overwrite_existing_stop_times_int and ' 'use_existing_stop_times_int cannot both be True.') + if use_highest_freq_trips_date and any([day, date, date_range]): + raise ValueError("Only one parameter: 'use_highest_freq_trips_date' " + "or one of 'day', 'date', or 'date_range' can be " + "used at a time or both 'day' and 'date_range' can " + "be used.") columns = ['route_id', 'direction_id', From 3c9b5f3f096f524e2b0faf9285ad577659d165d5 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 26 Oct 2021 13:05:32 -0700 Subject: [PATCH 19/72] update checks for more than one cal param and add has_day_and_date_range_param --- urbanaccess/gtfs/utils_calendar.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/urbanaccess/gtfs/utils_calendar.py b/urbanaccess/gtfs/utils_calendar.py index 37249ee..51776ef 100644 --- a/urbanaccess/gtfs/utils_calendar.py +++ b/urbanaccess/gtfs/utils_calendar.py @@ -36,15 +36,17 @@ def _trip_schedule_selector_validate_params(calendar_dates_df, params): has_date_param = params['has_date_param'] has_date_range_param = params['has_date_range_param'] has_cal_dates_param = params['has_cal_dates_param'] + has_day_and_date_range_param = params['has_day_and_date_range_param'] # only one search parameter can be used at a time - param_list = [has_day_param, has_date_param, has_date_range_param] - param_true_cnt = param_list.count(True) - if param_true_cnt > 1: + all_param_list = [has_day_param, has_date_param, has_date_range_param] + all_param_true_cnt = all_param_list.count(True) + + # Note: combination of day and date_range is valid + if all_param_true_cnt > 1 and not has_day_and_date_range_param: raise ValueError( "Only one parameter: 'day', 'date', or 'date_range' can be used " - "at a time. To proceed, keep one parameter and set the other " - "parameter(s) to None.") + "at a time or both 'day' and 'date_range' can be used.") if has_day_param: if not isinstance(day, str): @@ -185,6 +187,7 @@ def _select_calendar_service_ids(calendar_df, params): has_day_param = params['has_day_param'] has_date_param = params['has_date_param'] has_date_range_param = params['has_date_range_param'] + has_day_and_date_range_param = params['has_day_and_date_range_param'] # convert cols to datetime expected format: 'yyyymmdd' e.g.: '20130825' date_cols = ['start_date', 'end_date'] @@ -205,7 +208,9 @@ def _select_calendar_service_ids(calendar_df, params): calendar_df, msg, date) srvc_ids = srvc_ids_date.copy() - if has_day_param and has_date_range_param: + if has_day_and_date_range_param: + # get the intersection between service IDs found via 'day' and + # 'date_range' params print_msg = '{} that are active within date_range: {} on day: {}...' log(print_msg.format(msg, date_range, day)) intersect_srvc_ids = _intersect_cal_service_ids( @@ -751,6 +756,7 @@ def _select_calendar_dates_service_ids(calendar_dates_df, params): has_date_param = params['has_date_param'] has_date_range_param = params['has_date_range_param'] has_cal_dates_param = params['has_cal_dates_param'] + has_day_and_date_range_param = params['has_day_and_date_range_param'] # convert 'date' to datetime expected format: 'yyyymmdd' e.g.: '20130825' calendar_dates_df = _cal_date_dt_conversion( @@ -1054,6 +1060,8 @@ def _calendar_service_id_selector( has_date_param = date is not None has_date_range_param = date_range is not None has_cal_dates_param = cal_dates_lookup is not None + has_day_and_date_range_param = \ + has_day_param and has_date_range_param is True params = {'day': day, 'date': date, @@ -1064,7 +1072,8 @@ def _calendar_service_id_selector( 'has_date_param': has_date_param, 'has_date_range_param': has_date_range_param, 'has_cal_dates': has_cal_dates, - 'has_cal_dates_param': has_cal_dates_param} + 'has_cal_dates_param': has_cal_dates_param, + 'has_day_and_date_range_param': has_day_and_date_range_param} _trip_schedule_selector_validate_params(calendar_dates_df, params) From 8e78554f57aa69a3dd6c3d60202fc16cddb79bd6 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 26 Oct 2021 13:06:05 -0700 Subject: [PATCH 20/72] add check for datetime str format --- urbanaccess/gtfs/utils_calendar.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/urbanaccess/gtfs/utils_calendar.py b/urbanaccess/gtfs/utils_calendar.py index 51776ef..4ed3498 100644 --- a/urbanaccess/gtfs/utils_calendar.py +++ b/urbanaccess/gtfs/utils_calendar.py @@ -153,8 +153,13 @@ def _cal_date_dt_conversion(df, date_cols): """ for col in date_cols: - df[col] = pd.to_datetime( - df[col], format='%y%m%d', infer_datetime_format=True) + try: + df[col] = pd.to_datetime( + df[col], format='%y%m%d', infer_datetime_format=True) + except ValueError: + raise ValueError("Column: {} has values that are not in a " + "supported date format. Expected format: " + "'YYYY-MM-DD'.".format(col)) return df From e76e9ff80ff37363a583fe1c635edd7f23bfeb71 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 26 Oct 2021 13:06:48 -0700 Subject: [PATCH 21/72] add empty list for when no srv ids are found --- urbanaccess/gtfs/utils_calendar.py | 1 + 1 file changed, 1 insertion(+) diff --git a/urbanaccess/gtfs/utils_calendar.py b/urbanaccess/gtfs/utils_calendar.py index 4ed3498..1fdb653 100644 --- a/urbanaccess/gtfs/utils_calendar.py +++ b/urbanaccess/gtfs/utils_calendar.py @@ -185,6 +185,7 @@ def _select_calendar_service_ids(calendar_df, params): """ # collect service IDs that match search parameters in calendar.txt msg = 'Selecting service_ids from calendar' + srvc_ids = [] day = params['day'] date = params['date'] From bd95727a5a9946ff64b986ea1c1f75bd5cace545 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 26 Oct 2021 13:07:41 -0700 Subject: [PATCH 22/72] minor formatting, added docstring --- urbanaccess/gtfs/utils_calendar.py | 42 +++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/urbanaccess/gtfs/utils_calendar.py b/urbanaccess/gtfs/utils_calendar.py index 1fdb653..9b9ef6c 100644 --- a/urbanaccess/gtfs/utils_calendar.py +++ b/urbanaccess/gtfs/utils_calendar.py @@ -287,8 +287,8 @@ def _intersect_cal_service_ids(dict_1, dict_2, verbose=True): msg = (' Intersection between service_ids selected with: ' '{} parameter (count: {:,}) and {} parameter (count: {:,}) ' 'returned {:,} total service_id(s).'.format( - srvc_id_1_name, list_1_len, srvc_id_2_name, list_2_len, - srvc_ids_len)) + srvc_id_1_name, list_1_len, srvc_id_2_name, list_2_len, + srvc_ids_len)) log(msg) return srvc_ids @@ -327,11 +327,33 @@ def _select_calendar_service_ids_by_day( def _select_calendar_service_ids_by_date_range( calendar_df, msg, date_range=None): - # Global search using: date_range to subset records in calendar.txt - # using 'start_date' and 'end_date' columns when GTFS feed has - # seasonal service_ids - # select service IDs where 'start_date' and 'end_date' is within the - # date_range + """ + Global search using date_range to subset records in calendar.txt + using 'start_date' and 'end_date' columns when GTFS feed has + seasonal service_ids select service IDs where + 'start_date' and 'end_date' are within the date_range + + Parameters + ---------- + calendar_df : pandas.DataFrame + Calendar DataFrame + msg : str + string to prepend to print statement for informative purposes + date_range : list + date range to extract active service IDs in calendar table + specified as a 2 element list of strings where the first element + is the first date and the second element is the last date in the + date range. Dates should be specified as strings in YYYY-MM-DD format, + e.g. ['2013-03-09', '2013-09-01']. Service IDs that are active + within the date range specified (where the date range is within + the date range given by the start_date and end_date columns) + will be extracted. + + Returns + ------- + srvc_ids : list + list of active service IDs + """ log(' {} that are active within date_range: {}...'.format( msg, date_range)) @@ -341,8 +363,8 @@ def _select_calendar_service_ids_by_date_range( date_range[0], date_range[1], inclusive=True)].to_list() end_d_mask = service_ids.loc[calendar_df['end_date'].between( date_range[0], date_range[1], inclusive=True)].to_list() - ids_in_range = service_ids.loc[service_ids.isin(start_d_mask) & - service_ids.isin(end_d_mask)] + ids_in_range = service_ids.loc[ + service_ids.isin(start_d_mask) & service_ids.isin(end_d_mask)] subset_cal_df = calendar_df.loc[service_ids.isin(ids_in_range)] srvc_ids = subset_cal_df['unique_service_id'].to_list() @@ -574,6 +596,7 @@ def _select_calendar_dates_service_ids_by_day( calendar_dates_df['_ua_weekday'] == day] srvc_ids = subset_cal_dates_df['unique_service_id'].to_list() _print_cal_service_ids_len(srvc_ids, table_name='calendar dates') + return srvc_ids @@ -828,7 +851,6 @@ def _merge_service_ids_cal_dates_w_cal( log(' Reconciling service_id(s) between calendar and ' 'calendar_dates based on exception_type... ') srvc_ids_add = list(set(srvc_ids_add)) - srvc_ids_add_cnt = len(srvc_ids_add) srvc_ids_del = list(set(srvc_ids_del)) srvc_ids_cnt_before_add = len(set(srvc_ids)) From 4f69f7b73bc9bcc887277e09fe4e44fbb3e11952 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 26 Oct 2021 13:09:23 -0700 Subject: [PATCH 23/72] update logic to support has_day_and_date_range_param --- urbanaccess/gtfs/utils_calendar.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/urbanaccess/gtfs/utils_calendar.py b/urbanaccess/gtfs/utils_calendar.py index 9b9ef6c..1230dd7 100644 --- a/urbanaccess/gtfs/utils_calendar.py +++ b/urbanaccess/gtfs/utils_calendar.py @@ -799,6 +799,8 @@ def _select_calendar_dates_service_ids(calendar_dates_df, params): if has_day_param: srvc_ids_add = _select_calendar_dates_service_ids_by_day( calendar_dates_df, msg, day) + if has_day_and_date_range_param: + srvc_ids_add_day = srvc_ids_add.copy() if has_date_param: srvc_ids_add, srvc_ids_del = \ @@ -809,6 +811,22 @@ def _select_calendar_dates_service_ids(calendar_dates_df, params): srvc_ids_add, srvc_ids_del = \ _select_calendar_dates_service_ids_by_date_range( calendar_dates_df, msg, date_range) + if has_day_and_date_range_param: + srvc_ids_add_date_range = srvc_ids_add.copy() + # since we are also using 'day' as a param we cannot remove + # service_ids so set it to empty list + srvc_ids_del = [] + + if has_day_and_date_range_param: + # get the intersection between service IDs found via 'day' and + # 'date_range' params; since 'day' is a global param we + # cannot use any exception_type = 2 records to remove records + print_msg = '{} that are active within date_range: {} on day: {}...' + log(print_msg.format(msg, date_range, day)) + intersect_srvc_ids = _intersect_cal_service_ids( + dict_1={'day': srvc_ids_add_day}, + dict_2={'date range': srvc_ids_add_date_range}) + srvc_ids_add = intersect_srvc_ids.copy() if has_cal_dates_param: query_srvc_ids_add, query_srvc_ids_del = \ @@ -818,7 +836,8 @@ def _select_calendar_dates_service_ids(calendar_dates_df, params): # active service_ids or inactive service_ids merge_dict = {'add': [srvc_ids_add, query_srvc_ids_add], 'del': [srvc_ids_del, query_srvc_ids_del]} - _add_exception_type_service_id_lists(merge_dict) + srvc_ids_add, srvc_ids_del = _add_exception_type_service_id_lists( + merge_dict) return srvc_ids_add, srvc_ids_del From 366f67eb105759a42c3fd714f6b5b62e3f3621fe Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 26 Oct 2021 13:10:34 -0700 Subject: [PATCH 24/72] only select exception type 1 in _select_calendar_dates_service_ids_by_day() --- urbanaccess/gtfs/utils_calendar.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/urbanaccess/gtfs/utils_calendar.py b/urbanaccess/gtfs/utils_calendar.py index 1230dd7..ee4225b 100644 --- a/urbanaccess/gtfs/utils_calendar.py +++ b/urbanaccess/gtfs/utils_calendar.py @@ -592,9 +592,9 @@ def _select_calendar_dates_service_ids_by_day( # requiring pandas > 0.18.1 is needed to run the line below: calendar_dates_df['_ua_weekday'] = calendar_dates_df[ 'date'].dt.day_name().str.lower() - subset_cal_dates_df = calendar_dates_df.loc[ - calendar_dates_df['_ua_weekday'] == day] - srvc_ids = subset_cal_dates_df['unique_service_id'].to_list() + subset_df = calendar_dates_df.loc[calendar_dates_df['_ua_weekday'] == day] + srvc_ids = subset_df['unique_service_id'].loc[ + subset_df['exception_type'] == '1'].to_list() _print_cal_service_ids_len(srvc_ids, table_name='calendar dates') return srvc_ids From 64c480c314a485b775a110ccb43ad63119c64b11 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 26 Oct 2021 13:11:24 -0700 Subject: [PATCH 25/72] fix df var used in trip selection --- urbanaccess/gtfs/utils_calendar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/urbanaccess/gtfs/utils_calendar.py b/urbanaccess/gtfs/utils_calendar.py index ee4225b..80dc5eb 100644 --- a/urbanaccess/gtfs/utils_calendar.py +++ b/urbanaccess/gtfs/utils_calendar.py @@ -1220,7 +1220,7 @@ def _trip_selector(trips_df, service_ids, verbose=True): 'trips and calendar and or calendar_dates tables.') if verbose: agency_id_col = 'unique_agency_id' - agency_ids = subset_trip_df[agency_id_col].unique() + agency_ids = trips_df[agency_id_col].unique() for agency in agency_ids: agency_trip_cnt = len(subset_trip_df.loc[ subset_trip_df[agency_id_col] == agency]) From 9d613d521e5ad9576cd4f23810cf79b2973784a2 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 26 Oct 2021 13:12:50 -0700 Subject: [PATCH 26/72] ensure max date selected will always be the same when the data is the same by sorting --- urbanaccess/gtfs/utils_calendar.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/urbanaccess/gtfs/utils_calendar.py b/urbanaccess/gtfs/utils_calendar.py index 80dc5eb..4bf0d38 100644 --- a/urbanaccess/gtfs/utils_calendar.py +++ b/urbanaccess/gtfs/utils_calendar.py @@ -1341,12 +1341,15 @@ def _highest_freq_trips_date(trips_df, calendar_df, calendar_dates_df): date_trip_cnt.update({date: trip_id_cnt}) # find the date that has the max number of trips - max_date = max(date_trip_cnt, key=lambda k: date_trip_cnt[k]) + # sort so the max is always the same between runs of the same data + date_trip_cnt_sorted = dict(sorted(date_trip_cnt.items())) + max_date = max(date_trip_cnt_sorted, key=lambda k: date_trip_cnt_sorted[k]) # get the max number of trips number - max_trips = max(date_trip_cnt.values()) + max_trips = max(date_trip_cnt_sorted.values()) # check to see if there are more than 1 dates that have the same max number # of trips value - dates_w_max_trips = [k for k, v in date_trip_cnt.items() if v == max_trips] + dates_w_max_trips = [k for k, v in + date_trip_cnt_sorted.items() if v == max_trips] # get the service_ids for dates that share the max number of trips value reduce_list = [date_srv_id_dict[x] for x in dates_w_max_trips] # check to see if any of the dates have different service_ids from From 954df7ca67a29b9ec9b4d7d57359106d7589e896 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 26 Oct 2021 13:13:54 -0700 Subject: [PATCH 27/72] update highest_freq_trips_date() logic to accept cal dates srv ids correctly --- urbanaccess/gtfs/utils_calendar.py | 35 +++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/urbanaccess/gtfs/utils_calendar.py b/urbanaccess/gtfs/utils_calendar.py index 4bf0d38..e8cccad 100644 --- a/urbanaccess/gtfs/utils_calendar.py +++ b/urbanaccess/gtfs/utils_calendar.py @@ -1302,10 +1302,10 @@ def _highest_freq_trips_date(trips_df, calendar_df, calendar_dates_df): dates_dt = dates.to_pydatetime() dates_str = [date.strftime('%Y-%m-%d') for date in dates_dt] date_list.extend(dates_str) - unique_date_list = set(date_list) + unique_date_list_cal = list(set(date_list)) date_srv_id_dict = {} - for date in unique_date_list: + for date in unique_date_list_cal: srvc_ids_date = _select_calendar_service_ids_by_date( calendar_df, msg='', date=date, verbose=False) date_srv_id_dict.update({date: srvc_ids_date}) @@ -1313,8 +1313,11 @@ def _highest_freq_trips_date(trips_df, calendar_df, calendar_dates_df): if has_cal_dates: calendar_dates_df = _cal_date_dt_conversion( df=calendar_dates_df, date_cols=['date']) + unique_date_list_cal_dates = list( + calendar_dates_df['date'].dt.strftime('%Y-%m-%d').unique()) + date_add_rmv_srv_id_dict = {} - for date in unique_date_list: + for date in unique_date_list_cal_dates: srvc_ids_add, srvc_ids_del = \ _select_calendar_dates_service_ids_by_date( calendar_dates_df, msg='', date=date, verbose=False) @@ -1322,13 +1325,33 @@ def _highest_freq_trips_date(trips_df, calendar_df, calendar_dates_df): 'add': srvc_ids_add, 'remove': srvc_ids_del}}) if has_cal and has_cal_dates: + # combine both list of dates into one list of unique dates + unique_date_list_cal.extend(unique_date_list_cal_dates) + unique_date_list = set(unique_date_list_cal) + for date in unique_date_list: - srvc_ids = date_srv_id_dict[date].copy() - srvc_ids_add = date_add_rmv_srv_id_dict[date]['add'] - srvc_ids_del = date_add_rmv_srv_id_dict[date]['remove'] + if date in date_srv_id_dict.keys(): + srvc_ids = date_srv_id_dict[date].copy() + else: + srvc_ids = [] + if date in date_add_rmv_srv_id_dict.keys(): + srvc_ids_add = date_add_rmv_srv_id_dict[date]['add'] + srvc_ids_del = date_add_rmv_srv_id_dict[date]['remove'] + else: + srvc_ids_add = [] + srvc_ids_del = [] active_srvc_ids = _merge_service_ids_cal_dates_w_cal( srvc_ids, srvc_ids_add, srvc_ids_del, verbose=False) date_srv_id_dict[date] = active_srvc_ids.copy() + elif has_cal and not has_cal_dates: + unique_date_list = unique_date_list_cal.copy() + elif has_cal_dates and not has_cal: + unique_date_list = unique_date_list_cal_dates.copy() + date_srv_id_dict = {} + for date in unique_date_list: + # only consider service IDs to add (exception type == 1) + date_srv_id_dict.update( + {date: date_add_rmv_srv_id_dict[date]['add']}) # select the trips and count trips_df['unique_trip_id'] = trips_df['trip_id'].str.cat( From 9895c1f342b8fb0ec845c299bf710520d561de76 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 26 Oct 2021 13:28:45 -0700 Subject: [PATCH 28/72] update existing unit tests and add unit tests for utils_calendar.py --- urbanaccess/tests/conftest.py | 62 +- urbanaccess/tests/test_gtfs_network.py | 502 ++--- urbanaccess/tests/test_gtfs_utils_calendar.py | 1836 +++++++++++++++++ urbanaccess/tests/test_gtfs_utils_format.py | 8 +- 4 files changed, 2140 insertions(+), 268 deletions(-) create mode 100644 urbanaccess/tests/test_gtfs_utils_calendar.py diff --git a/urbanaccess/tests/conftest.py b/urbanaccess/tests/conftest.py index cdbd8c5..3d48944 100644 --- a/urbanaccess/tests/conftest.py +++ b/urbanaccess/tests/conftest.py @@ -295,8 +295,8 @@ def calendar_feed_1(): 'friday': [1, 1, 1, 0], 'saturday': [0, 0, 0, 1], 'sunday': [0, 0, 0, 1], - 'start_date': [20161224] * 4, - 'end_date': [20170318] * 4} + 'start_date': ['20161224'] * 4, + 'end_date': ['20170318'] * 4} index = range(4) @@ -318,8 +318,8 @@ def calendar_feed_2(): 'friday': [1, 1, 1, 0], 'saturday': [0, 0, 0, 1], 'sunday': [0, 0, 0, 1], - 'start_date': [20161224] * 4, - 'end_date': [20170318] * 4} + 'start_date': ['20161224'] * 4, + 'end_date': ['20170318'] * 4} index = range(4) @@ -338,8 +338,8 @@ def calendar_feed_4(): 'friday': [1], 'saturday': [0], 'sunday': [0], - 'start_date': [20161224], - 'end_date': [20170318]} + 'start_date': ['20161224'], + 'end_date': ['20170318']} index = range(1) @@ -347,23 +347,6 @@ def calendar_feed_4(): return df -@pytest.fixture -def calendar_empty(): - columns = {'service_id', - 'monday', - 'tuesday', - 'wednesday', - 'thursday', - 'friday', - 'saturday', - 'sunday', - 'start_date', - 'end_date'} - - df = pd.DataFrame(columns=columns) - return df - - @pytest.fixture def calendar_dates_feed_1(): data = { @@ -371,8 +354,8 @@ def calendar_dates_feed_1(): 'weekday-2', 'weekday-3', 'weekend-1'], - 'date': [20161224, 20170318, 20160424, 20161230], - 'exception_type': [1, 2, 1, 1], + 'date': ['20161224', '20170318', '20160424', '20161230'], + 'exception_type': ['1', '2', '1', '1'], 'schedule_type': ['WD', 'WD', 'WD', 'WE']} index = range(4) @@ -388,8 +371,8 @@ def calendar_dates_feed_2(): 'weekday-2', 'weekday-3', 'weekend-1'], - 'date': [20161224, 20170318, 20160424, 20161230], - 'exception_type': [1, 2, 1, 1], + 'date': ['20161224', '20170318', '20160424', '20161230'], + 'exception_type': ['1', '2', '1', '1'], 'schedule_type': ['WD', 'WD', 'WD', 'SA'] } @@ -403,8 +386,8 @@ def calendar_dates_feed_2(): def calendar_dates_feed_4(): data = { 'service_id': ['wk-1'], - 'date': [20161224], - 'exception_type': [1]} + 'date': ['20161224'], + 'exception_type': ['1']} index = range(1) @@ -826,3 +809,24 @@ def agency_a_feed_on_disk_w_calendar_and_calendar_dates_empty_txt( feed_file_name = '{}.txt'.format(feed_file) feed_df.to_csv(os.path.join(feed_path, feed_file_name), index=False) return feed_path + + +@pytest.fixture() +def agency_b_feed_on_disk_w_calendar_and_calendar_dates( + tmpdir, + agency_feed_2, stop_times_feed_2, stops_feed_2, + routes_feed_2, trips_feed_2, calendar_feed_2, calendar_dates_feed_2): + feed_file_dict = {'agency': agency_feed_2, + 'stop_times': stop_times_feed_2, + 'stops': stops_feed_2, + 'routes': routes_feed_2, + 'trips': trips_feed_2, + 'calendar': calendar_feed_2, + 'calendar_dates': calendar_dates_feed_2} + feed_path = os.path.join(tmpdir.strpath, 'agency_b_w_both_calendars') + os.makedirs(feed_path) + print('writing test data to dir: {}'.format(feed_path)) + for feed_file, feed_df in feed_file_dict.items(): + feed_file_name = '{}.txt'.format(feed_file) + feed_df.to_csv(os.path.join(feed_path, feed_file_name), index=False) + return feed_path diff --git a/urbanaccess/tests/test_gtfs_network.py b/urbanaccess/tests/test_gtfs_network.py index 9c4f06d..b6241eb 100644 --- a/urbanaccess/tests/test_gtfs_network.py +++ b/urbanaccess/tests/test_gtfs_network.py @@ -65,6 +65,19 @@ def gtfs_feed_w_calendar_and_calendar_dates( return loaded_feeds +@pytest.fixture +def gtfs_feed_w_calendar_and_calendar_dates_multi_agency( + agency_b_feed_on_disk_w_calendar_and_calendar_dates): + loaded_feeds = gtfs_load.gtfsfeed_to_df( + gtfsfeed_path=agency_b_feed_on_disk_w_calendar_and_calendar_dates, + validation=False, + verbose=True, + bbox=None, + remove_stops_outsidebbox=False, + append_definitions=False) + return loaded_feeds + + @pytest.fixture def selected_int_stop_times_from_feed_wo_calendar_dates( gtfs_feed_wo_calendar_dates): @@ -308,6 +321,76 @@ def expected_transit_edge_from_feed_wo_calendar_dates_process_lvl_2_timeaware(): return df +@pytest.fixture +def expected_final_transit_edge_from_feed_wo_calendar(): + # represents df after it has been post-processed downstream + data = { + 'node_id_from': ['6_agency_a_city_a', '5_agency_a_city_a', + '4_agency_a_city_a', '3_agency_a_city_a'], + 'node_id_to': ['5_agency_a_city_a', '4_agency_a_city_a', + '3_agency_a_city_a', '1_agency_a_city_a'], + 'weight': [5.0, 5.0, 5.0, 10.0], + 'unique_agency_id': ['agency_a_city_a'] * 4, + 'unique_trip_id': ['c2_agency_a_city_a'] * 4, + 'sequence': range(1, 5), + 'unique_route_id': ['12-101_agency_a_city_a'] * 4, + 'id': ['c2_agency_a_city_a_1', 'c2_agency_a_city_a_2', + 'c2_agency_a_city_a_3', 'c2_agency_a_city_a_4'], + 'route_type': [1] * 4, + 'net_type': ['transit'] * 4 + } + index = range(4) + df = pd.DataFrame(data, index) + # raw data are read as int32 + df['sequence'] = df['sequence'].astype('int32') + return df + + +@pytest.fixture +def expected_final_transit_edge_from_feed_w_cal_and_cal_dates( + expected_transit_edge_from_feed_wo_calendar_dates_process_lvl_2): + # represents df after it has been post-processed downstream + data = { + 'unique_route_id': ['10-101_agency_a_city_a'] * 5, + 'net_type': ['transit'] * 5 + } + index = range(5) + df = pd.DataFrame(data, index) + df = pd.concat( + [expected_transit_edge_from_feed_wo_calendar_dates_process_lvl_2, + df], + axis=1) + return df + + +@pytest.fixture +def expected_transit_edge_from_agency_b_feed_w_cal_and_cal_dates(): + # represents df after it has been post-processed downstream + data = { + 'node_id_from': ['60_agency_b_district_1', '61_agency_b_district_1', + '62_agency_b_district_1', '63_agency_b_district_1', + '64_agency_b_district_1'], + 'node_id_to': ['61_agency_b_district_1', '62_agency_b_district_1', + '63_agency_b_district_1', '64_agency_b_district_1', + '65_agency_b_district_1'], + 'weight': [5.0] * 5, + 'unique_agency_id': ['agency_b_district_1'] * 5, + 'unique_trip_id': ['13_agency_b_district_1'] * 5, + 'sequence': range(1, 6), + 'unique_route_id': ['40-4_agency_b_district_1'] * 5, + 'id': ['13_agency_b_district_1_1', '13_agency_b_district_1_2', + '13_agency_b_district_1_3', '13_agency_b_district_1_4', + '13_agency_b_district_1_5'], + 'route_type': [3] * 5, + 'net_type': ['transit'] * 5 + } + index = range(5) + df = pd.DataFrame(data, index) + # raw data are read as int32 + df['sequence'] = df['sequence'].astype('int32') + return df + + @pytest.fixture def expected_transit_edge_from_feed_wo_calendar_dates_process_lvl_1_timeaware( expected_transit_edge_from_feed_wo_calendar_dates_process_lvl_2_timeaware): # noqa @@ -439,6 +522,50 @@ def hdf5_file_on_disk_gtfsfeeds_dfs( return hdf5_save_path +def test_create_transit_net_wo_calendar_dates_w_high_freq_trips( + gtfs_feed_wo_calendar_dates, + expected_urbanaccess_network_keys, + expected_final_transit_edge_from_feed_wo_calendar_dates): + expected_result = \ + expected_final_transit_edge_from_feed_wo_calendar_dates.copy() + transit_net = gtfs_network.create_transit_net( + gtfs_feed_wo_calendar_dates, day=None, + timerange=['07:00:00', '10:00:00'], + calendar_dates_lookup=None, + overwrite_existing_stop_times_int=False, + use_existing_stop_times_int=False, + save_processed_gtfs=False, + save_dir=None, + save_filename=None, + timerange_pad=None, + time_aware=False, + date=None, + date_range=None, + use_highest_freq_trips_date=True, + simplify=False) + assert isinstance(transit_net, urbanaccess_network) + urbanaccess_network_info = vars(transit_net) + expected_dfs = ['transit_nodes', 'transit_edges'] + assert expected_urbanaccess_network_keys == sorted(list( + urbanaccess_network_info.keys())) + for key, value in urbanaccess_network_info.items(): + assert isinstance(value, pd.core.frame.DataFrame) + # check that df is not empty + if key in expected_dfs: + assert value.empty is False + + result_edge = transit_net.transit_edges.copy() + # test that output df is identical to expected df + result_edge = result_edge.reindex( + sorted(result_edge.columns), axis=1) + expected_result = expected_result.reindex( + sorted(expected_result.columns), axis=1) + # ensure 'sequence' is int32 for test as other OS sometimes reads this as + # int64 and will cause tests to fail when using equals() + result_edge['sequence'] = result_edge['sequence'].astype('int32') + assert result_edge.equals(expected_result) + + def test_create_transit_net_wo_calendar_dates( gtfs_feed_wo_calendar_dates, expected_urbanaccess_network_keys, @@ -477,6 +604,123 @@ def test_create_transit_net_wo_calendar_dates( assert result_edge.equals(expected_result) +def test_create_transit_net_wo_calendar( + gtfs_feed_wo_calendar, + expected_urbanaccess_network_keys, + expected_final_transit_edge_from_feed_wo_calendar +): + expected_result = \ + expected_final_transit_edge_from_feed_wo_calendar.copy() + transit_net = gtfs_network.create_transit_net( + gtfs_feed_wo_calendar, day=None, + timerange=['00:00:00', '23:59:00'], + calendar_dates_lookup=None, + overwrite_existing_stop_times_int=False, + use_existing_stop_times_int=False, + save_processed_gtfs=False, + save_dir=None, + save_filename=None, + date='2016-04-24') + assert isinstance(transit_net, urbanaccess_network) + urbanaccess_network_info = vars(transit_net) + expected_dfs = ['transit_nodes', 'transit_edges'] + assert expected_urbanaccess_network_keys == sorted(list( + urbanaccess_network_info.keys())) + for key, value in urbanaccess_network_info.items(): + assert isinstance(value, pd.core.frame.DataFrame) + # check that df is not empty + if key in expected_dfs: + assert value.empty is False + + result_edge = transit_net.transit_edges.copy() + # test that output df is identical to expected df + result_edge = result_edge.reindex( + sorted(result_edge.columns), axis=1) + expected_result = expected_result.reindex( + sorted(expected_result.columns), axis=1) + # ensure 'sequence' is int32 for test as other OS sometimes reads this as + # int64 and will cause tests to fail when using equals() + result_edge['sequence'] = result_edge['sequence'].astype('int32') + assert result_edge.equals(expected_result) + + +def test_create_transit_net_w_calendar_and_calendar_dates( + gtfs_feed_w_calendar_and_calendar_dates, + expected_urbanaccess_network_keys, + expected_final_transit_edge_from_feed_w_cal_and_cal_dates): + expected_result = \ + expected_final_transit_edge_from_feed_w_cal_and_cal_dates.copy() + transit_net = gtfs_network.create_transit_net( + gtfs_feed_w_calendar_and_calendar_dates, day=None, + timerange=['07:00:00', '10:00:00'], + calendar_dates_lookup=None, + overwrite_existing_stop_times_int=False, + use_existing_stop_times_int=False, + save_processed_gtfs=False, + save_dir=None, + save_filename=None, + date_range=['2016-12-24', '2017-03-18']) + assert isinstance(transit_net, urbanaccess_network) + urbanaccess_network_info = vars(transit_net) + expected_dfs = ['transit_nodes', 'transit_edges'] + assert expected_urbanaccess_network_keys == sorted(list( + urbanaccess_network_info.keys())) + for key, value in urbanaccess_network_info.items(): + assert isinstance(value, pd.core.frame.DataFrame) + # check that df is not empty + if key in expected_dfs: + assert value.empty is False + + result_edge = transit_net.transit_edges.copy() + # test that output df is identical to expected df + result_edge = result_edge.reindex( + sorted(result_edge.columns), axis=1) + expected_result = expected_result.reindex( + sorted(expected_result.columns), axis=1) + # ensure 'sequence' is int32 for test as other OS sometimes reads this as + # int64 and will cause tests to fail when using equals() + result_edge['sequence'] = result_edge['sequence'].astype('int32') + assert result_edge.equals(expected_result) + + +def test_create_transit_net_w_calendar_and_calendar_dates_multi_agency( + gtfs_feed_w_calendar_and_calendar_dates_multi_agency, + expected_urbanaccess_network_keys, + expected_transit_edge_from_agency_b_feed_w_cal_and_cal_dates): + expected_result = \ + expected_transit_edge_from_agency_b_feed_w_cal_and_cal_dates.copy() + transit_net = gtfs_network.create_transit_net( + gtfs_feed_w_calendar_and_calendar_dates_multi_agency, day='monday', + timerange=['07:00:00', '10:00:00'], + calendar_dates_lookup=None, + overwrite_existing_stop_times_int=False, + use_existing_stop_times_int=False, + save_processed_gtfs=False, + save_dir=None, + save_filename=None) + assert isinstance(transit_net, urbanaccess_network) + urbanaccess_network_info = vars(transit_net) + expected_dfs = ['transit_nodes', 'transit_edges'] + assert expected_urbanaccess_network_keys == sorted(list( + urbanaccess_network_info.keys())) + for key, value in urbanaccess_network_info.items(): + assert isinstance(value, pd.core.frame.DataFrame) + # check that df is not empty + if key in expected_dfs: + assert value.empty is False + + result_edge = transit_net.transit_edges.copy() + # test that output df is identical to expected df + result_edge = result_edge.reindex( + sorted(result_edge.columns), axis=1) + expected_result = expected_result.reindex( + sorted(expected_result.columns), axis=1) + # ensure 'sequence' is int32 for test as other OS sometimes reads this as + # int64 and will cause tests to fail when using equals() + result_edge['sequence'] = result_edge['sequence'].astype('int32') + assert result_edge.equals(expected_result) + + def test_create_transit_net_wo_calendar_dates_timepad( gtfs_feed_wo_calendar_dates, expected_urbanaccess_network_keys, @@ -523,7 +767,8 @@ def test_create_transit_net_wo_calendar_dates_timeaware( expected_urbanaccess_network_keys, expected_final_transit_edge_from_feed_wo_calendar_dates_timeaware): expected_result = \ - expected_final_transit_edge_from_feed_wo_calendar_dates_timeaware.copy() # noqa + expected_final_transit_edge_from_feed_wo_calendar_dates_timeaware\ + .copy() # noqa transit_net = gtfs_network.create_transit_net( gtfs_feed_wo_calendar_dates, day='monday', timerange=['07:00:00', '10:00:00'], @@ -768,6 +1013,27 @@ def test_create_transit_net_invalid_params(gtfs_feed_wo_calendar_dates): expected_error = ('overwrite_existing_stop_times_int and ' 'use_existing_stop_times_int cannot both be True.') assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + transit_net = gtfs_network.create_transit_net( + gtfs_feed_wo_calendar_dates, day='monday', + timerange=['07:00:00', '10:00:00'], + calendar_dates_lookup=None, + overwrite_existing_stop_times_int=False, + use_existing_stop_times_int=False, + save_processed_gtfs=False, + save_dir=None, + save_filename=None, + timerange_pad=None, + time_aware=False, + date=None, + date_range=None, + use_highest_freq_trips_date=True, + simplify=False) + expected_error = ("Only one parameter: 'use_highest_freq_trips_date' " + "or one of 'day', 'date', or 'date_range' can be " + "used at a time or both 'day' and 'date_range' can " + "be used.") + assert expected_error in str(excinfo.value) with pytest.raises(ValueError) as excinfo: transit_net = gtfs_network.create_transit_net( gtfs_feed_wo_calendar_dates, day='monday', @@ -980,240 +1246,6 @@ def test_skip_interpolator(stop_times, calendar): 1, 2, 3, 4, 5] -def test_trip_schedule_selector_wo_cal_dates(gtfs_feed_wo_calendar_dates): - expected_result = gtfs_feed_wo_calendar_dates.trips.copy() - # create expected trips result - expected_result.reset_index(drop=True, inplace=True) - expected_result = expected_result.iloc[0:8] - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed_wo_calendar_dates.trips, - input_calendar_df=gtfs_feed_wo_calendar_dates.calendar, - input_calendar_dates_df=gtfs_feed_wo_calendar_dates.calendar_dates, - day='monday', - calendar_dates_lookup=None) - - assert len(result) == 8 - assert result.equals(expected_result) - - -def test_trip_schedule_selector_wo_cal_dates_wo_direction_id( - gtfs_feed_wo_calendar_dates): - # remove 'direction_id' col for test - trips_df = gtfs_feed_wo_calendar_dates.trips.copy() - trips_df.drop(columns=['direction_id'], inplace=True) - expected_result = gtfs_feed_wo_calendar_dates.trips.copy() - # create expected trips result - expected_result.reset_index(drop=True, inplace=True) - expected_result.drop(columns=['direction_id'], inplace=True) - expected_result = expected_result.iloc[0:8] - - result = gtfs_network._trip_schedule_selector( - input_trips_df=trips_df, - input_calendar_df=gtfs_feed_wo_calendar_dates.calendar, - input_calendar_dates_df=gtfs_feed_wo_calendar_dates.calendar_dates, - day='monday', - calendar_dates_lookup=None) - - assert len(result) == 8 - assert result.equals(expected_result) - - -def test_trip_schedule_selector_w_cal_dates(gtfs_feed_wo_calendar): - expected_result = gtfs_feed_wo_calendar.trips.copy() - # create expected trips result - expected_result = expected_result.iloc[4:10] - expected_result.reset_index(drop=True, inplace=True) - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed_wo_calendar.trips, - input_calendar_df=gtfs_feed_wo_calendar.calendar, - input_calendar_dates_df=gtfs_feed_wo_calendar.calendar_dates, - day='sunday', - calendar_dates_lookup={'schedule_type': 'WE', - 'service_id': ['weekday-3', 'weekday-2']}) - - assert len(result) == 6 - assert result.equals(expected_result) - - -def test_trip_schedule_selector_w_cal_and_cal_dates( - gtfs_feed_w_calendar_and_calendar_dates): - trips_df = gtfs_feed_w_calendar_and_calendar_dates.trips.copy() - cal_df = gtfs_feed_w_calendar_and_calendar_dates.calendar.copy() - cal_dates_df = gtfs_feed_w_calendar_and_calendar_dates.calendar_dates \ - .copy() - expected_result = gtfs_feed_w_calendar_and_calendar_dates.trips.copy() - result = gtfs_network._trip_schedule_selector( - input_trips_df=trips_df, - input_calendar_df=cal_df, - input_calendar_dates_df=cal_dates_df, - day='monday', - calendar_dates_lookup={'schedule_type': 'WE'}) - - assert len(result) == 10 - assert result.equals(expected_result) - - -def test_trip_schedule_selector_w_cal_and_cal_dates_wo_lookup( - gtfs_feed_w_calendar_and_calendar_dates): - trips_df_1 = gtfs_feed_w_calendar_and_calendar_dates.trips.copy() - cal_df = gtfs_feed_w_calendar_and_calendar_dates.calendar.copy() - cal_dates_df_1 = gtfs_feed_w_calendar_and_calendar_dates.calendar_dates \ - .copy() - # create extra records in trips and calendar_dates for a different agency - # that do not exist in the calendar table - trips_df_2 = trips_df_1.copy() - trips_df_2['unique_agency_id'] = trips_df_2['unique_agency_id'] + '_x' - trips_df_2['unique_feed_id'] = trips_df_2['unique_feed_id'] + '_x' - trips_df_2 = trips_df_2.iloc[0:8] - trips_df_x2 = pd.concat( - [trips_df_1, trips_df_2], axis=0, - ignore_index=True) - cal_dates_df_2 = cal_dates_df_1.copy() - cal_dates_df_2['unique_agency_id'] = \ - cal_dates_df_2['unique_agency_id'] + '_x' - cal_dates_df_2['unique_feed_id'] = \ - cal_dates_df_2['unique_feed_id'] + '_x' - cal_dates_df_x2 = pd.concat( - [cal_dates_df_1, cal_dates_df_2], axis=0, - ignore_index=True) - # create expected trips result - expected_result = trips_df_1.copy() - expected_result = expected_result.iloc[0:8] - result = gtfs_network._trip_schedule_selector( - input_trips_df=trips_df_x2, - input_calendar_df=cal_df, - input_calendar_dates_df=cal_dates_df_x2, - day='monday', - calendar_dates_lookup=None) - - assert len(result) == 8 - assert result.equals(expected_result) - - -def test_trip_schedule_selector_wo_cal_dates_invalid_params( - gtfs_feed_wo_calendar_dates): - gtfs_feed = gtfs_feed_wo_calendar_dates - # test with invalid 'day' param - with pytest.raises(ValueError) as excinfo: - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed.trips, - input_calendar_df=gtfs_feed.calendar, - input_calendar_dates_df=gtfs_feed.calendar_dates, - day='monday ', - calendar_dates_lookup=None) - expected_error = ( - "Incorrect day specified. Must be one of lowercase strings: 'monday'," - " 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'.") - assert expected_error in str(excinfo.value) - # test with invalid 'calendar_dates_lookup' param - with pytest.raises(ValueError) as excinfo: - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed.trips, - input_calendar_df=gtfs_feed.calendar, - input_calendar_dates_df=gtfs_feed.calendar_dates, - day='monday', - calendar_dates_lookup=['invalid']) - expected_error = "calendar_dates_lookup parameter must be a dictionary." - assert expected_error in str(excinfo.value) - # test with invalid 'calendar_dates_lookup' param - with pytest.raises(ValueError) as excinfo: - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed.trips, - input_calendar_df=gtfs_feed.calendar, - input_calendar_dates_df=gtfs_feed.calendar_dates, - day='monday', - calendar_dates_lookup={1: 'WD'}) - expected_error = "calendar_dates_lookup key: 1 must be a string." - assert expected_error in str(excinfo.value) - # test with invalid 'calendar_dates_lookup' param - with pytest.raises(ValueError) as excinfo: - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed.trips, - input_calendar_df=gtfs_feed.calendar, - input_calendar_dates_df=gtfs_feed.calendar_dates, - day='monday', - calendar_dates_lookup={'schedule_type': 1}) - expected_error = ("calendar_dates_lookup value: 1 must be a " - "string or a list of strings.") - assert expected_error in str(excinfo.value) - # test with invalid 'calendar_dates_lookup' param - with pytest.raises(ValueError) as excinfo: - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed.trips, - input_calendar_df=gtfs_feed.calendar, - input_calendar_dates_df=gtfs_feed.calendar_dates, - day='monday', - calendar_dates_lookup={'schedule_type': ['WD', 1]}) - expected_error = ("calendar_dates_lookup value: ['WD', 1] " - "must contain strings.") - assert expected_error in str(excinfo.value) - - -def test_trip_schedule_selector_no_services_found( - gtfs_feed_wo_calendar): - with pytest.raises(ValueError) as excinfo: - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed_wo_calendar.trips, - input_calendar_df=gtfs_feed_wo_calendar.calendar, - input_calendar_dates_df=gtfs_feed_wo_calendar.calendar_dates, - day='monday', - calendar_dates_lookup=None) - expected_error = ("No service_id(s) were found with the specified " - "calendar and or calendar_dates search parameters.") - assert expected_error in str(excinfo.value) - - -def test_trip_schedule_selector_w_cal_dates_invalid_params_1( - gtfs_feed_wo_calendar_dates): - # test with empty 'calendar_dates'df - with pytest.raises(ValueError) as excinfo: - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed_wo_calendar_dates.trips, - input_calendar_df=gtfs_feed_wo_calendar_dates.calendar, - input_calendar_dates_df=gtfs_feed_wo_calendar_dates.calendar_dates, - day='monday', - calendar_dates_lookup={'schedule_type': 'WD'}) - expected_error = ("calendar_dates is empty. Unable to use the " - "'calendar_dates_lookup' parameter. Set to None.") - assert expected_error in str(excinfo.value) - - -def test_trip_schedule_selector_w_cal_dates_invalid_params_2( - gtfs_feed_wo_calendar): - # create invalid data in calendar dates file - cal_dates_df = gtfs_feed_wo_calendar.calendar_dates.copy() - series = pd.Series( - data=[1, 1, 2, 2], index=range(4), - name='invalid_dtype') - cal_dates_df['invalid_dtype'] = series - series = pd.Series( - data=[10, 11, 10, 'aa'], index=range(4), - name='day_type') - cal_dates_df['day_type'] = series - - # test with invalid col in 'calendar_dates_lookup' param - with pytest.raises(ValueError) as excinfo: - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed_wo_calendar.trips, - input_calendar_df=gtfs_feed_wo_calendar.calendar, - input_calendar_dates_df=cal_dates_df, - day='monday', - calendar_dates_lookup={'invalid_col': 'WD'}) - expected_error = ("Column: invalid_col not found in calendar_dates " - "DataFrame.") - assert expected_error in str(excinfo.value) - # test with invalid col dtype in 'calendar_dates_lookup' param - with pytest.raises(ValueError) as excinfo: - result = gtfs_network._trip_schedule_selector( - input_trips_df=gtfs_feed_wo_calendar.trips, - input_calendar_df=gtfs_feed_wo_calendar.calendar, - input_calendar_dates_df=cal_dates_df, - day='monday', - calendar_dates_lookup={'invalid_dtype': '1'}) - expected_error = "Column: invalid_dtype must be object type." - assert expected_error in str(excinfo.value) - - def test_time_selector_wo_timerange_pad( selected_int_stop_times_from_feed_wo_calendar_dates): timerange = ['08:20:00', '08:35:00'] diff --git a/urbanaccess/tests/test_gtfs_utils_calendar.py b/urbanaccess/tests/test_gtfs_utils_calendar.py new file mode 100644 index 0000000..aaf2d69 --- /dev/null +++ b/urbanaccess/tests/test_gtfs_utils_calendar.py @@ -0,0 +1,1836 @@ +import pytest +import pandas as pd + +import urbanaccess.gtfs.utils_calendar as gtfs_utils_cal + + +@pytest.fixture +def trips_agency_a(): + data = { + 'service_id': [ + 'summer-weekday-1', 'summer-weekday-1', 'summer-weekday-1', + 'summer-weekday-2', 'summer-weekday-3', 'summer-weekend-1', + 'fall-weekday-1', 'fall-weekday-2', 'fall-weekday-3', + 'fall-weekend-1', + 'fall-weekday-1', 'fall-weekday-2', 'fall-weekday-3', + 'fall-weekend-1', + 'special-game-day-1', 'special-game-day-2', 'special-game-day-3', + 'special-game-day-1', 'special-game-day-2', 'special-game-day-3'], + 'trip_id': ['a1', 'a2', 'a3', 'a4', 'a5', 'a6', + 'b1', 'b2', 'b3', 'b4', 'b1', 'b2', 'b3', 'b4', + 's1', 's2', 's3', 's4', 's5', 's6'], + 'route_id': ['1R'] * 20, + 'direction_id': [0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 1], + 'unique_agency_id': ['agency_a_city_a'] * 20, + } + index = range(20) + df = pd.DataFrame(data, index) + df['unique_service_id'] = (df['service_id'].str.cat( + df['unique_agency_id'].astype('str'), sep='_')) + return df + + +@pytest.fixture +def trips_agency_b(): + data = { + 'service_id': [ + 'fall-weekday-1', 'fall-weekday-2', 'fall-weekday-3', + 'fall-weekend-1', + 'fall-weekday-1', 'fall-weekday-2', 'fall-weekday-3', + 'fall-weekend-1', + 'special-game-day-1', 'special-game-day-2', 'special-game-day-3', + 'special-game-day-1', 'special-game-day-2', 'special-game-day-3'], + 'trip_id': ['b1', 'b2', 'b3', 'b4', 'b1', 'b2', 'b3', 'b4', + 's1', 's2', 's3', 's4', 's5', 's6'], + 'route_id': ['1R'] * 14, + 'direction_id': [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1], + 'unique_agency_id': ['agency_b_city_a'] * 14, + } + index = range(14) + df = pd.DataFrame(data, index) + df['unique_service_id'] = (df['service_id'].str.cat( + df['unique_agency_id'].astype('str'), sep='_')) + return df + + +@pytest.fixture +def calendar_agency_a(): + # 'summer-weekday-2' only runs on wednesdays + # 'fall-weekday-1' overlaps with summer sched for month of august + data = { + 'service_id': + ['summer-weekday-1', 'summer-weekday-2', 'summer-weekday-3', + 'summer-weekend-1', + 'fall-weekday-1', 'fall-weekday-2', 'fall-weekday-3', + 'fall-weekend-1'], + 'monday': [1, 0, 1, 0, 1, 1, 1, 0], + 'tuesday': [1, 0, 1, 0, 1, 1, 1, 0], + 'wednesday': [1, 1, 1, 0, 1, 1, 1, 0], + 'thursday': [1, 0, 1, 0, 1, 1, 1, 0], + 'friday': [1, 0, 1, 0, 1, 1, 1, 0], + 'saturday': [0, 0, 0, 1, 0, 0, 0, 1], + 'sunday': [0, 0, 0, 1, 0, 0, 0, 1], + 'start_date': + ['2016-6-01', '2016-6-01', '2016-6-01', '2016-6-01', + '2016-8-01', '2016-9-01', '2016-9-01', '2016-9-01'], + 'end_date': + ['2016-8-31', '2016-8-31', '2016-8-31', '2016-8-31', + '2016-11-30', '2016-11-30', '2016-11-30', '2016-11-30'], + 'unique_agency_id': ['agency_a_city_a'] * 8, + 'unique_feed_id': ['agency_a_city_a_feed'] * 8 + } + index = range(8) + df = pd.DataFrame(data, index) + df['unique_service_id'] = (df['service_id'].str.cat( + df['unique_agency_id'].astype('str'), sep='_')) + return df + + +@pytest.fixture +def calendar_dates_agency_a(): + # holiday '2016-7-04' falls on a monday but runs on 'summer-weekend-1' + # sched + # holiday '2016-9-05' falls on a monday but runs on 'fall-weekend-1' sched + # game day '2016-8-02' falls on a tuesday and runs extra service + data = { + 'service_id': + ['summer-weekday-1', 'summer-weekday-3', 'summer-weekend-1', + 'fall-weekday-1', 'fall-weekday-2', 'fall-weekday-3', + 'fall-weekend-1', + 'special-game-day-1', 'special-game-day-2', 'special-game-day-3'], + 'date': ['2016-7-04', '2016-7-04', '2016-7-04', + '2016-9-05', '2016-9-05', '2016-9-05', '2016-9-05', + '2016-8-02', '2016-8-02', '2016-8-02'], + 'exception_type': ['2', '2', '1', + '2', '2', '2', '1', + '1', '1', '1'], + 'schedule_type': ['holiday', 'holiday', 'holiday', + 'holiday', 'holiday', 'holiday', 'holiday', + 'game day', 'game day', 'game day'], + 'unique_agency_id': ['agency_a_city_a'] * 10, + 'unique_feed_id': ['agency_a_city_a_feed'] * 10 + } + index = range(10) + df = pd.DataFrame(data, index) + + return df + + +@pytest.fixture +def trips_and_cal_with_multi_freq_dates( + trips_agency_a, calendar_dates_agency_a): + data = { + 'service_id': [ + 'special-game-day-1_a', 'special-game-day-2_a', + 'special-game-day-3_a', 'special-game-day-1_a', + 'special-game-day-2_a', 'special-game-day-3_a'], + 'trip_id': ['s1_a', 's2_a', 's3_a', 's4_a', 's5_a', 's6_a'], + 'route_id': ['1R'] * 6, + 'direction_id': [0, 1, 0, 1, 0, 0], + 'unique_agency_id': ['agency_a_city_a'] * 6 + } + index = range(6) + extra_trips = pd.DataFrame(data, index) + extra_trips['unique_service_id'] = (extra_trips['service_id'].str.cat( + extra_trips['unique_agency_id'].astype('str'), sep='_')) + trips_df = pd.concat([trips_agency_a, extra_trips], ignore_index=True) + + data = { + 'service_id': + ['special-game-day-1_a', 'special-game-day-2_a', + 'special-game-day-3_a'], + 'date': ['2016-12-02', '2016-12-02', '2016-12-02'], + 'exception_type': ['1', '1', '1'], + 'schedule_type': ['game day', 'game day', 'game day'], + 'unique_agency_id': ['agency_a_city_a'] * 3, + 'unique_feed_id': ['agency_a_city_a_feed'] * 3 + } + index = range(3) + extra_cal_dates = pd.DataFrame(data, index) + cal_dates_df = pd.concat([calendar_dates_agency_a, extra_cal_dates], + ignore_index=True) + + return trips_df, cal_dates_df + + +@pytest.fixture +def calendar_agency_a_datetime(calendar_agency_a): + cols = ['start_date', 'end_date'] + for col in cols: + calendar_agency_a[col] = pd.to_datetime( + calendar_agency_a[col], format='%y%m%d', + infer_datetime_format=True) + return calendar_agency_a + + +@pytest.fixture +def calendar_dates_agency_a_datetime(calendar_dates_agency_a): + col = 'date' + calendar_dates_agency_a[col] = pd.to_datetime( + calendar_dates_agency_a[col], format='%y%m%d', + infer_datetime_format=True) + return calendar_dates_agency_a + + +@pytest.fixture +def calendar_dates_w_uni_svc_id(calendar_dates_agency_a): + df = calendar_dates_agency_a.copy() + df['unique_service_id'] = (df['service_id'].str.cat( + df['unique_agency_id'].astype('str'), sep='_')) + return df + + +@pytest.fixture +def calendar_dates_w_uni_svc_id_datetime(calendar_dates_agency_a_datetime): + df = calendar_dates_agency_a_datetime.copy() + df['unique_service_id'] = (df['service_id'].str.cat( + df['unique_agency_id'].astype('str'), sep='_')) + return df + + +@pytest.fixture +def calendar_empty(calendar_agency_a): + df_empty = calendar_agency_a[0:0] + return df_empty + + +@pytest.fixture +def calendar_dates_empty(calendar_dates_agency_a): + df_empty = calendar_dates_agency_a[0:0] + return df_empty + + +@pytest.fixture +def msg_param(): + return 'Selecting service_ids from calendar' + + +@pytest.fixture +def day_param(): + return 'monday' + + +@pytest.fixture +def date_param_cal(): + return '2016-06-13' + + +@pytest.fixture +def date_param_cal_dates(): + return '2016-08-02' + + +@pytest.fixture +def daterange_param(): + return ['2016-06-01', '2016-08-31'] + + +@pytest.fixture +def cal_dates_lookup_param(): + return {'schedule_type': 'holiday', + 'service_id': ['fall-weekday-1', 'fall-weekday-2']} + + +@pytest.fixture +def expected_result_cal_param_day(): + expected_active_srv_ids = sorted( + ['summer-weekday-1_agency_a_city_a', + 'summer-weekday-3_agency_a_city_a', + 'fall-weekday-1_agency_a_city_a', + 'fall-weekday-2_agency_a_city_a', + 'fall-weekday-3_agency_a_city_a']) + return expected_active_srv_ids + + +@pytest.fixture +def expected_result_cal_param_date(): + expected_active_srv_ids = sorted( + ['summer-weekday-1_agency_a_city_a', + 'summer-weekday-3_agency_a_city_a']) + return expected_active_srv_ids + + +@pytest.fixture +def expected_result_cal_param_daterange(): + expected_active_srv_ids = sorted( + ['summer-weekday-1_agency_a_city_a', + 'summer-weekday-2_agency_a_city_a', + 'summer-weekday-3_agency_a_city_a', + 'summer-weekend-1_agency_a_city_a']) + return expected_active_srv_ids + + +@pytest.fixture +def expected_result_cal_dates_param_day(): + expected_active_srv_ids = sorted( + ['summer-weekend-1_agency_a_city_a', + 'fall-weekend-1_agency_a_city_a']) + return expected_active_srv_ids + + +@pytest.fixture +def expected_result_cal_dates_param_date(): + expected_active_srv_ids = sorted( + ['special-game-day-3_agency_a_city_a', + 'special-game-day-2_agency_a_city_a', + 'special-game-day-1_agency_a_city_a']) + return expected_active_srv_ids + + +@pytest.fixture +def expected_result_cal_dates_param_daterange(): + expected_active_srv_ids = sorted( + ['special-game-day-1_agency_a_city_a', + 'special-game-day-3_agency_a_city_a', + 'special-game-day-2_agency_a_city_a', + 'summer-weekend-1_agency_a_city_a']) + return expected_active_srv_ids + + +@pytest.fixture +def expected_result_cal_dates_param_lookup(): + expected_srvc_ids_add = sorted( + ['summer-weekend-1_agency_a_city_a', 'fall-weekend-1_agency_a_city_a']) + expected_srvc_ids_del = sorted( + ['summer-weekday-1_agency_a_city_a', + 'fall-weekday-3_agency_a_city_a', + 'fall-weekday-1_agency_a_city_a', + 'summer-weekday-3_agency_a_city_a', + 'fall-weekday-2_agency_a_city_a']) + return expected_srvc_ids_add, expected_srvc_ids_del + + +def test_select_cal_service_ids_w_day( + calendar_agency_a, expected_result_cal_param_day, day_param): + result = gtfs_utils_cal._select_calendar_service_ids( + calendar_agency_a, + params={'day': day_param, + 'date': None, + 'date_range': None, + 'cal_dates_lookup': None, + 'has_cal': True, + 'has_day_param': True, + 'has_date_param': False, + 'has_date_range_param': False, + 'has_cal_dates': False, + 'has_cal_dates_param': False, + 'has_day_and_date_range_param': False}) + + assert isinstance(result, list) + assert len(result) == 5 + # result should only include service IDs in calendar where monday = 1 + assert sorted(result) == expected_result_cal_param_day + + +def test_select_cal_service_ids_w_date( + calendar_agency_a, date_param_cal, expected_result_cal_param_date): + result = gtfs_utils_cal._select_calendar_service_ids( + calendar_agency_a, + params={'day': None, + 'date': date_param_cal, + 'date_range': None, + 'cal_dates_lookup': None, + 'has_cal': True, + 'has_day_param': False, + 'has_date_param': True, + 'has_date_range_param': False, + 'has_cal_dates': False, + 'has_cal_dates_param': False, + 'has_day_and_date_range_param': False}) + + assert isinstance(result, list) + assert len(result) == 2 + # result should only include service IDs in calendar that are active on + # '2016-06-13' representing a monday sched + assert sorted(result) == expected_result_cal_param_date + + +def test_select_cal_service_ids_w_date_range( + calendar_agency_a, daterange_param, + expected_result_cal_param_daterange): + result = gtfs_utils_cal._select_calendar_service_ids( + calendar_agency_a, + params={'day': None, + 'date': None, + 'date_range': daterange_param, + 'cal_dates_lookup': None, + 'has_cal': True, + 'has_day_param': False, + 'has_date_param': False, + 'has_date_range_param': True, + 'has_cal_dates': False, + 'has_cal_dates_param': False, + 'has_day_and_date_range_param': False}) + + assert isinstance(result, list) + assert len(result) == 4 + # result should only include service IDs in calendar that are active + # between ['2016-06-01', '2016-08-31'] + assert sorted(result) == expected_result_cal_param_daterange + + +def test_select_cal_service_ids_w_day_and_date_range( + calendar_agency_a, day_param, daterange_param): + result = gtfs_utils_cal._select_calendar_service_ids( + calendar_agency_a, + params={'day': day_param, + 'date': None, + 'date_range': daterange_param, + 'cal_dates_lookup': None, + 'has_cal': True, + 'has_day_param': True, + 'has_date_param': False, + 'has_date_range_param': True, + 'has_cal_dates': False, + 'has_cal_dates_param': False, + 'has_day_and_date_range_param': True}) + + expected_active_srv_ids = sorted( + ['summer-weekday-3_agency_a_city_a', + 'summer-weekday-1_agency_a_city_a']) + + assert isinstance(result, list) + assert len(result) == 2 + # result should only include service IDs in calendar that are active + # between ['2016-06-01', '2016-08-31'] and 'monday' + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_day_param_w_cal_only( + calendar_agency_a, calendar_dates_empty, + expected_result_cal_param_day, day_param): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_empty, + day=day_param, + date=None, + date_range=None, + cal_dates_lookup=None) + + assert isinstance(result, list) + assert len(result) == 5 + # result should only include service IDs in calendar where monday = 1 + assert sorted(result) == expected_result_cal_param_day + + +def test_cal_srv_id_selector_day_param_w_cal_dates_only( + calendar_empty, calendar_dates_agency_a, day_param, + expected_result_cal_dates_param_day): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_empty, + calendar_dates_df=calendar_dates_agency_a, + day=day_param, + date=None, + date_range=None, + cal_dates_lookup=None) + + assert isinstance(result, list) + assert len(result) == 2 + # result should only include service IDs in calendar dates whose date is a + # monday and exception_type = 1 + assert sorted(result) == expected_result_cal_dates_param_day + + +def test_cal_srv_id_selector_day_param_w_cal_and_cal_dates( + calendar_agency_a, calendar_dates_agency_a): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_agency_a, + day='tuesday', + date=None, + date_range=None, + cal_dates_lookup=None) + + expected_active_srv_ids = sorted( + ['summer-weekday-1_agency_a_city_a', + 'summer-weekday-3_agency_a_city_a', + 'fall-weekday-1_agency_a_city_a', + 'fall-weekday-2_agency_a_city_a', + 'fall-weekday-3_agency_a_city_a', + 'special-game-day-1_agency_a_city_a', + 'special-game-day-2_agency_a_city_a', + 'special-game-day-3_agency_a_city_a']) + + assert isinstance(result, list) + assert len(result) == 8 + # result should only include service IDs in calendar dates whose date is a + # tuesday regardless of exception_type + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_date_param_w_cal_only( + calendar_agency_a, calendar_dates_empty, date_param_cal): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_empty, + day=None, + date=date_param_cal, + date_range=None, + cal_dates_lookup=None) + + expected_active_srv_ids = sorted( + ['summer-weekday-1_agency_a_city_a', + 'summer-weekday-3_agency_a_city_a']) + + assert isinstance(result, list) + assert len(result) == 2 + # result should only include service IDs in calendar that are active on + # '2016-06-13' representing a monday sched + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_date_param_w_cal_dates_only_case_1( + calendar_empty, calendar_dates_agency_a, date_param_cal_dates, + expected_result_cal_dates_param_date): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_empty, + calendar_dates_df=calendar_dates_agency_a, + day=None, + date=date_param_cal_dates, + date_range=None, + cal_dates_lookup=None) + + assert isinstance(result, list) + assert len(result) == 3 + # result should only include service IDs in calendar dates that are + # active on '2016-08-02' with exception_type = 1 + assert sorted(result) == expected_result_cal_dates_param_date + + +def test_cal_srv_id_selector_date_param_w_cal_dates_only_case_2( + calendar_empty, calendar_dates_agency_a): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_empty, + calendar_dates_df=calendar_dates_agency_a, + day=None, + date='2016-07-04', + date_range=None, + cal_dates_lookup=None) + + expected_active_srv_ids = ['summer-weekend-1_agency_a_city_a'] + + assert isinstance(result, list) + assert len(result) == 1 + # result should only include service IDs in calendar dates that are + # active on '2016-07-04' with exception_type = 1 + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_date_param_w_cal_and_cal_dates_case_1( + calendar_agency_a, calendar_dates_agency_a, date_param_cal_dates): + # case 1: add service IDs via cal dates + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_agency_a, + day=None, + date=date_param_cal_dates, + date_range=None, + cal_dates_lookup=None) + + expected_active_srv_ids = sorted( + ['special-game-day-1_agency_a_city_a', + 'special-game-day-2_agency_a_city_a', + 'special-game-day-3_agency_a_city_a', + 'summer-weekday-1_agency_a_city_a', + 'summer-weekday-3_agency_a_city_a', + 'fall-weekday-1_agency_a_city_a' + ]) + + assert isinstance(result, list) + assert len(result) == 6 + # result should only include service IDs in calendar that are active + # on '2016-08-02' and add service IDs from calendar dates where + # exception = 1 and remove service IDs from calendar dates where + # exception = 2 + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_date_param_w_cal_and_cal_dates_case_2( + calendar_agency_a, calendar_dates_agency_a): + # case 2: remove service IDs via cal dates + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_agency_a, + day=None, + date='2016-07-04', + date_range=None, + cal_dates_lookup=None) + + expected_active_srv_ids = ['summer-weekend-1_agency_a_city_a'] + + assert isinstance(result, list) + assert len(result) == 1 + # result should only include service IDs in calendar that are active + # on '2016-07-04' and add service IDs from calendar dates where + # exception = 1 and remove service IDs from calendar dates where + # exception = 2 + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_date_range_param_w_cal_only( + calendar_agency_a, calendar_dates_empty, daterange_param, + expected_result_cal_param_daterange): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_empty, + day=None, + date=None, + date_range=daterange_param, + cal_dates_lookup=None) + + assert isinstance(result, list) + assert len(result) == 4 + # result should only include service IDs in calendar that are active + # between ['2016-06-01', '2016-08-31'] + assert sorted(result) == expected_result_cal_param_daterange + + +def test_cal_srv_id_selector_date_range_param_w_cal_dates_only( + calendar_empty, calendar_dates_agency_a, daterange_param, + expected_result_cal_dates_param_daterange): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_empty, + calendar_dates_df=calendar_dates_agency_a, + day=None, + date=None, + date_range=daterange_param, + cal_dates_lookup=None) + + assert isinstance(result, list) + assert len(result) == 4 + # result should only include service IDs in calendar dates that are + # active between ['2016-06-01', '2016-08-31'] with exception_type = 1 + assert sorted(result) == expected_result_cal_dates_param_daterange + + +def test_cal_srv_id_selector_date_range_param_w_cal_and_cal_dates( + calendar_agency_a, calendar_dates_agency_a, daterange_param): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_agency_a, + day=None, + date=None, + date_range=daterange_param, + cal_dates_lookup=None) + + expected_active_srv_ids = sorted( + ['special-game-day-1_agency_a_city_a', + 'special-game-day-2_agency_a_city_a', + 'special-game-day-3_agency_a_city_a', + 'summer-weekend-1_agency_a_city_a', + 'summer-weekday-2_agency_a_city_a']) + + assert isinstance(result, list) + assert len(result) == 5 + # result should only include service IDs in calendar that are active + # between ['2016-06-01', '2016-08-31'] and in calendar dates that are + # active between ['2016-06-01', '2016-08-31'] with exception_type = 1 + # and remove service IDs from calendar dates where exception = 2 + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_day_and_date_range_param_w_cal_only( + calendar_agency_a, calendar_dates_empty, day_param, daterange_param): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_empty, + day=day_param, + date=None, + date_range=daterange_param, + cal_dates_lookup=None) + + expected_active_srv_ids = sorted( + ['summer-weekday-3_agency_a_city_a', + 'summer-weekday-1_agency_a_city_a']) + + assert isinstance(result, list) + assert len(result) == 2 + # result should only include service IDs in calendar that are active + # between ['2016-06-01', '2016-08-31'] and on 'monday' + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_day_and_date_range_param_w_cal_dates_only( + calendar_empty, calendar_dates_agency_a, day_param, daterange_param): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_empty, + calendar_dates_df=calendar_dates_agency_a, + day=day_param, + date=None, + date_range=daterange_param, + cal_dates_lookup=None) + + expected_active_srv_ids = ['summer-weekend-1_agency_a_city_a'] + + assert isinstance(result, list) + assert len(result) == 1 + # result should only include service IDs in calendar dates that are + # active between ['2016-06-01', '2016-08-31'] on a 'monday' with + # exception_type = 1 + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_day_date_range_param_w_cal_and_cal_dates( + calendar_agency_a, calendar_dates_agency_a, day_param, + daterange_param): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_agency_a, + day=day_param, + date=None, + date_range=daterange_param, + cal_dates_lookup=None) + + expected_active_srv_ids = sorted(['summer-weekend-1_agency_a_city_a', + 'summer-weekday-3_agency_a_city_a', + 'summer-weekday-1_agency_a_city_a']) + + assert isinstance(result, list) + assert len(result) == 3 + # result should only include service IDs in calendar that are active + # between ['2016-06-01', '2016-08-31'] on a monday and in calendar dates + # that are active between ['2016-06-01', '2016-08-31'] on a monday with + # exception_type = 1 + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_lookup_param_w_cal_only_invalid( + calendar_agency_a, calendar_dates_empty, day_param, + cal_dates_lookup_param): + with pytest.raises(ValueError) as excinfo: + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_empty, + day=day_param, + date=None, + date_range=None, + cal_dates_lookup=cal_dates_lookup_param) + expected_error = ("Calendar_dates is empty. Unable to use the " + "'calendar_dates_lookup' parameter. Set to None.") + assert expected_error in str(excinfo.value) + + +def test_cal_srv_id_selector_lookup_param_w_cal_dates_only( + calendar_empty, calendar_dates_agency_a, day_param, + cal_dates_lookup_param, + expected_result_cal_dates_param_day): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_empty, + calendar_dates_df=calendar_dates_agency_a, + day=day_param, + date=None, + date_range=None, + cal_dates_lookup=cal_dates_lookup_param) + + assert isinstance(result, list) + assert len(result) == 2 + # result should only include service IDs in calendar dates that match + # the values in the query with exception_type = 1 and where date is a + # 'monday' + assert sorted(result) == expected_result_cal_dates_param_day + + +def test_cal_srv_id_selector_lookup_param_w_cal_and_cal_dates( + calendar_agency_a, calendar_dates_agency_a, day_param, + cal_dates_lookup_param): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_agency_a, + day=day_param, + date=None, + date_range=None, + cal_dates_lookup=cal_dates_lookup_param) + + expected_active_srv_ids = sorted( + ['fall-weekend-1_agency_a_city_a', + 'summer-weekend-1_agency_a_city_a']) + + assert isinstance(result, list) + assert len(result) == 2 + # result should only include service IDs in calendar dates that match + # the values in the query with exception_type = 1 and remove service IDs + # from calendar dates where exception = 2 and also service IDs from + # calendar that are active for 'monday' + assert sorted(result) == expected_active_srv_ids + + +def test_cal_srv_id_selector_warn(capsys, + calendar_agency_a, calendar_dates_empty): + result = gtfs_utils_cal._calendar_service_id_selector( + calendar_df=calendar_agency_a, + calendar_dates_df=calendar_dates_empty, + day=None, + date='2016-01-01', + date_range=None, + cal_dates_lookup=None) + + assert isinstance(result, list) + assert len(result) == 0 + + # check that expected print prints + captured = capsys.readouterr() + assert ('Warning: No active service_ids were found matching the specified ' + 'parameters.') in captured.out + + +def test_unique_service_id( + calendar_dates_agency_a, calendar_agency_a): + # remove the existing column from the test data + calendar_agency_a.drop(columns=['unique_service_id'], inplace=True) + df_list = [calendar_dates_agency_a, calendar_agency_a] + result = gtfs_utils_cal._unique_service_id(df_list) + assert isinstance(result, list) + assert isinstance(result[0], pd.core.frame.DataFrame) + assert isinstance(result[1], pd.core.frame.DataFrame) + assert result[0].empty is False + assert result[1].empty is False + assert 'unique_service_id' in result[0].columns + assert 'unique_service_id' in result[1].columns + assert result[0]['unique_service_id'].iloc[0] == \ + 'summer-weekday-1_agency_a_city_a' + + +def test_cal_date_dt_conversion_valid(calendar_agency_a): + result = gtfs_utils_cal._cal_date_dt_conversion( + df=calendar_agency_a, + date_cols=['start_date', 'end_date']) + assert isinstance(result, pd.core.frame.DataFrame) + assert result.dtypes['start_date'] == ' Date: Tue, 26 Oct 2021 13:32:56 -0700 Subject: [PATCH 29/72] minor formatting --- urbanaccess/gtfs/utils_calendar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/urbanaccess/gtfs/utils_calendar.py b/urbanaccess/gtfs/utils_calendar.py index e8cccad..ed2132b 100644 --- a/urbanaccess/gtfs/utils_calendar.py +++ b/urbanaccess/gtfs/utils_calendar.py @@ -944,7 +944,7 @@ def _select_calendar_dates_str_match( result_cnt = len(result_srvc_ids) log(' Found {:,} unique service_id(s) that matched query:' ' column: {} and string: {}.'.format( - result_cnt, col_name_key, text)) + result_cnt, col_name_key, text)) srvc_ids.extend(result_srvc_ids) srvc_ids = list(set(srvc_ids)) From 3a814ae99c0a357ec067da46cf2db332e7afcae7 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 26 Oct 2021 16:25:33 -0700 Subject: [PATCH 30/72] add docstrings, minor formatting --- urbanaccess/gtfs/network.py | 61 +++++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/urbanaccess/gtfs/network.py b/urbanaccess/gtfs/network.py index 3b9f8ba..3ef2d60 100644 --- a/urbanaccess/gtfs/network.py +++ b/urbanaccess/gtfs/network.py @@ -326,11 +326,35 @@ def create_transit_net( def _simplify_transit_net(edges, nodes): + """ + Simplifies edge and node tables by removing trips from the edge table + that have identical properties that result in duplicate edge attributes + for use in routing engines such as Pandana. Edges are grouped by + columns: 'node_id_from', 'node_id_to', 'weight', 'unique_agency_id', + 'sequence', 'route_type', and 'unique_route_id'. If multiple trip IDs are + found to have the exact same attributes in those columns, only one + representative trip ID and its corresponding edges will be preserved out + of the group. + + Parameters + ---------- + edges : pandas.DataFrame + edges DataFrame to simplify + nodes : pandas.DataFrame + nodes DataFrame to simplify + + Returns + ------- + simp_edges : pandas.DataFrame + simplified edge DataFrame + simp_nodes : pandas.DataFrame + simplified node DataFrame + """ # TODO: add option for groups that are duplicates, place trip ids # of the groups that are removed in a new column as list of - # strings for debugging or trip counts + # strings for downstream debugging or for informative trip counts # TODO: can simplify even more by reducing along edge shared attr in - # addition to trips, ensure that stop ids are connected + # addition to trips, must ensure that stop ids are connected start_time = time.time() log('Running transit network simplification...') @@ -361,7 +385,7 @@ def _simplify_transit_net(edges, nodes): from_id_col='node_id_from', to_id_col='node_id_to') # calculate various data len and print statistics - trip_remove_cnt = len(edges_wdup.loc[edges_wdup == True]) + trip_remove_cnt = len(edges_wdup.loc[edges_wdup == True]) # noqa trip_org_tot_cnt = len(edges[id_col].unique()) trip_proc_tot_cnt = len(simp_edges[id_col].unique()) trip_remove_pct = (trip_remove_cnt / trip_org_tot_cnt) * 100 @@ -414,7 +438,6 @@ def _interpolate_stop_times(stop_times_df, calendar_selected_trips_df): final_stop_times_df : pandas.DataFrame """ - start_time = time.time() # create unique trip IDs @@ -765,11 +788,11 @@ def _format_transit_net_edge(stop_times_df, time_aware=False): raise ValueError( "Unable to continue processing transit network. stop_times " "table has 0 records. Most likely the 'timerange' and or " - "calendar selection parameters resulted in " - "a network with no active trips within the schedule window " - "specified. It is suggested to expand the 'timerange' and or " - "alter the calendar selection parameters to enlarge the time " - "window for selecting active trips.") + "calendar selection parameters resulted in a network with no " + "active trips within the schedule window specified. It is " + "suggested to expand the 'timerange' and or alter the calendar " + "selection parameters to enlarge the time window for selecting " + "active trips.") # subset to only columns needed for processing cols_of_interest = ['unique_trip_id', 'stop_id', 'unique_stop_id', @@ -1298,6 +1321,26 @@ def _check_if_index_name_in_cols(df): def _remove_nodes_not_in_edges(nodes, edges, from_id_col, to_id_col): + """ + Helper function to remove nodes from node DataFrame that are not found + in the edges DataFrame + + Parameters + ---------- + nodes : pandas.DataFrame + nodes DataFrame + edges : pandas.DataFrame + edges DataFrame + from_id_col : str + name of from ID column + to_id_col : str + name of to ID column + + Returns + ------- + nodes_df : pandas.DataFrame + node DataFrame with only the nodes that exist in the edge DataFrame + """ edge_node_list = set(edges[from_id_col]) | set(edges[to_id_col]) nodes_df = nodes.loc[nodes.index.isin(edge_node_list)] return nodes_df From 510308c2d7cf22857e16f4f72c94ccd9b27a8470 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Tue, 26 Oct 2021 16:26:53 -0700 Subject: [PATCH 31/72] add unit tests for net simplification functions --- urbanaccess/tests/test_gtfs_network.py | 281 +++++++++++++++++++++++++ 1 file changed, 281 insertions(+) diff --git a/urbanaccess/tests/test_gtfs_network.py b/urbanaccess/tests/test_gtfs_network.py index b6241eb..a1cf564 100644 --- a/urbanaccess/tests/test_gtfs_network.py +++ b/urbanaccess/tests/test_gtfs_network.py @@ -522,6 +522,116 @@ def hdf5_file_on_disk_gtfsfeeds_dfs( return hdf5_save_path +@pytest.fixture +def edges_empty(transit_edges_to_simplify): + df_empty = transit_edges_to_simplify[0:0] + return df_empty + + +@pytest.fixture +def transit_edges_to_simplify(): + data = { + 'node_id_from': [ + '1_agency_a_city_a', '2_agency_a_city_a', '3_agency_a_city_a', + '4_agency_a_city_a', '3_agency_a_city_a', '2_agency_a_city_a', + '5_agency_a_city_a', '6_agency_a_city_a', '7_agency_a_city_a', + '5_agency_a_city_a', '6_agency_a_city_a', '7_agency_a_city_a', + '8_agency_a_city_a', '9_agency_a_city_a', '10_agency_a_city_a', + '8_agency_a_city_a', '9_agency_a_city_a', '10_agency_a_city_a' + ], + 'node_id_to': [ + '2_agency_a_city_a', '3_agency_a_city_a', '4_agency_a_city_a', + '3_agency_a_city_a', '2_agency_a_city_a', '1_agency_a_city_a', + '6_agency_a_city_a', '7_agency_a_city_a', '8_agency_a_city_a', + '6_agency_a_city_a', '7_agency_a_city_a', '8_agency_a_city_a', + '9_agency_a_city_a', '10_agency_a_city_a', '11_agency_a_city_a', + '9_agency_a_city_a', '10_agency_a_city_a', '11_agency_a_city_a'], + 'weight': [2.0, 4.0, 5.0, + 5.0, 4.0, 2.0, + 2.0, 2.0, 3.0, + 2.0, 2.0, 3.0, + 4.0, 3.0, 4.0, + 4.0, 3.0, 4.0], + 'unique_agency_id': ['agency_a_city_a'] * 18, + 'unique_trip_id': [ + '1_agency_a_city_a', '1_agency_a_city_a', '1_agency_a_city_a', + '2_agency_a_city_a', '2_agency_a_city_a', '2_agency_a_city_a', + '3_agency_a_city_a', '3_agency_a_city_a', '3_agency_a_city_a', + '4_agency_a_city_a', '4_agency_a_city_a', '4_agency_a_city_a', + '5_agency_a_city_a', '5_agency_a_city_a', '5_agency_a_city_a', + '6_agency_a_city_a', '6_agency_a_city_a', '6_agency_a_city_a'], + 'sequence': [1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3], + 'unique_route_id': [ + 'R1_agency_a_city_a', 'R1_agency_a_city_a', 'R1_agency_a_city_a', + 'R1_agency_a_city_a', 'R1_agency_a_city_a', 'R1_agency_a_city_a', + 'A1_agency_a_city_a', 'A1_agency_a_city_a', 'A1_agency_a_city_a', + 'A1_agency_a_city_a', 'A1_agency_a_city_a', 'A1_agency_a_city_a', + 'B1_agency_a_city_a', 'B1_agency_a_city_a', 'B1_agency_a_city_a', + 'C1_agency_a_city_a', 'C1_agency_a_city_a', 'C1_agency_a_city_a'], + 'route_type': [1] * 18, + 'net_type': ['transit'] * 18 + } + index = range(18) + df = pd.DataFrame(data, index) + df['id'] = (df['unique_trip_id'].str.cat( + df['sequence'].astype('str'), sep='_')) + return df + + +@pytest.fixture +def transit_nodes_to_simplify(): + data = { + 'node_id': [ + '1_agency_a_city_a', '2_agency_a_city_a', '3_agency_a_city_a', + '4_agency_a_city_a', '5_agency_a_city_a', '6_agency_a_city_a', + '7_agency_a_city_a', '8_agency_a_city_a', '9_agency_a_city_a', + '10_agency_a_city_a', '11_agency_a_city_a'], + 'x': [-122.268962, -122.273627, -122.276709, -122.272114, + -122.266589, -122.268842, -122.271760, -122.265859, + -122.274823, -122.271057, -122.266551], + 'y': [37.807730, 37.800201, 37.795430, 37.793342, + 37.806846, 37.803413, 37.798581, 37.796250, + 37.805727, 37.804455, 37.802497], + 'unique_agency_id': ['agency_a_city_a'] * 11, + 'route_type': [1] * 11, + 'stop_id': range(1, 12), + 'stop_name': [ + 'ave a', 'ave b', 'ave c', 'ave d', 'ave 1', 'ave 2', + 'ave 3', 'ave 4', 'st a', 'st b', 'st c'], + 'net_type': ['transit'] * 11 + } + index = range(11) + df = pd.DataFrame(data, index) + df.set_index('node_id', drop=True, inplace=True) + return df + + +@pytest.fixture +def transit_nodes_to_simplify_extra_nodes(transit_nodes_to_simplify): + data = { + 'node_id': ['12_agency_a_city_a'], + 'x': [-122.266551], + 'y': [37.802497], + 'unique_agency_id': ['agency_a_city_a'], + 'route_type': [1], + 'stop_id': [13], + 'stop_name': ['st d'], + 'net_type': ['transit'] + } + index = range(1) + df = pd.DataFrame(data, index) + df.set_index('node_id', drop=True, inplace=True) + df = pd.concat([transit_nodes_to_simplify, df], axis=1) + + return df + + +@pytest.fixture +def stop_times_empty(selected_int_stop_times_from_feed_wo_calendar_dates): + df_empty = selected_int_stop_times_from_feed_wo_calendar_dates[0:0] + return df_empty + + def test_create_transit_net_wo_calendar_dates_w_high_freq_trips( gtfs_feed_wo_calendar_dates, expected_urbanaccess_network_keys, @@ -683,6 +793,49 @@ def test_create_transit_net_w_calendar_and_calendar_dates( assert result_edge.equals(expected_result) +def test_create_transit_net_w_calendar_and_calendar_dates_w_simplify( + gtfs_feed_w_calendar_and_calendar_dates, + expected_urbanaccess_network_keys, + expected_final_transit_edge_from_feed_w_cal_and_cal_dates): + # TODO: note that gtfs_feed_w_calendar_and_calendar_dates does not + # result in any edges that require simplification. Test can be improved + # by changing the test data to one that can be simplified. + expected_result = \ + expected_final_transit_edge_from_feed_w_cal_and_cal_dates.copy() + transit_net = gtfs_network.create_transit_net( + gtfs_feed_w_calendar_and_calendar_dates, day=None, + timerange=['07:00:00', '10:00:00'], + calendar_dates_lookup=None, + overwrite_existing_stop_times_int=False, + use_existing_stop_times_int=False, + save_processed_gtfs=False, + save_dir=None, + save_filename=None, + date_range=['2016-12-24', '2017-03-18'], + simplify=True) + assert isinstance(transit_net, urbanaccess_network) + urbanaccess_network_info = vars(transit_net) + expected_dfs = ['transit_nodes', 'transit_edges'] + assert expected_urbanaccess_network_keys == sorted(list( + urbanaccess_network_info.keys())) + for key, value in urbanaccess_network_info.items(): + assert isinstance(value, pd.core.frame.DataFrame) + # check that df is not empty + if key in expected_dfs: + assert value.empty is False + + result_edge = transit_net.transit_edges.copy() + # test that output df is identical to expected df + result_edge = result_edge.reindex( + sorted(result_edge.columns), axis=1) + expected_result = expected_result.reindex( + sorted(expected_result.columns), axis=1) + # ensure 'sequence' is int32 for test as other OS sometimes reads this as + # int64 and will cause tests to fail when using equals() + result_edge['sequence'] = result_edge['sequence'].astype('int32') + assert result_edge.equals(expected_result) + + def test_create_transit_net_w_calendar_and_calendar_dates_multi_agency( gtfs_feed_w_calendar_and_calendar_dates_multi_agency, expected_urbanaccess_network_keys, @@ -1422,6 +1575,15 @@ def test_format_transit_net_edge_timeaware_True( assert result.equals(expected_result) +def test_format_transit_net_edge_empty_stop_times(stop_times_empty): + with pytest.raises(ValueError) as excinfo: + result = gtfs_network._format_transit_net_edge(stop_times_empty) + expected_error = ( + "Unable to continue processing transit network. stop_times table " + "has 0 records.") + assert expected_error in str(excinfo.value) + + def test_convert_imp_time_units( transit_edge_from_feed_wo_calendar_dates): # test with minutes @@ -1734,3 +1896,122 @@ def test_load_processed_gtfs_data( # check that df is empty if key in expected_dfs_empty: assert value.empty + + +def test_simplify_transit_net( + transit_edges_to_simplify, transit_nodes_to_simplify): + remove_edge_ids = ['4_agency_a_city_a_1', '4_agency_a_city_a_2', + '4_agency_a_city_a_3'] + expected_edges = transit_edges_to_simplify.loc[ + ~transit_edges_to_simplify['id'].isin(remove_edge_ids)] + expected_edges.reset_index(inplace=True, drop=True) + result_edges, result_nodes = gtfs_network._simplify_transit_net( + transit_edges_to_simplify, transit_nodes_to_simplify) + + # expect all nodes to persist + assert len(result_nodes) == 11 + assert result_nodes.equals(transit_nodes_to_simplify) + + # expect 5_agency_a_city_a_1 to 3 and 6_agency_a_city_a_1 to 3 to + # persist since they have different attributes in unique_route_id col + # expect 1_agency_a_city_a_1 to 3 and 2_agency_a_city_a_1 to 3 to + # persist since node_id_from and node_id_to are different + # expect only one of 3_agency_a_city_a_1 to 3 or 4_agency_a_city_a_1 to 3 + # to persist but not both since they share the same attributes + assert len(result_edges) == 15 + result_edges.reset_index(inplace=True, drop=True) + assert result_edges.equals(expected_edges) + + +def test_simplify_transit_net_already_simplified( + transit_edges_to_simplify, transit_nodes_to_simplify): + remove_edge_ids = ['4_agency_a_city_a_1', '4_agency_a_city_a_2', + '4_agency_a_city_a_3'] + expected_edges = transit_edges_to_simplify.loc[ + ~transit_edges_to_simplify['id'].isin(remove_edge_ids)] + expected_edges.reset_index(inplace=True, drop=True) + + # use the simplified edge table as input to test + result_edges, result_nodes = gtfs_network._simplify_transit_net( + expected_edges, transit_nodes_to_simplify) + + # expect all nodes to persist + assert len(result_nodes) == 11 + assert result_nodes.equals(transit_nodes_to_simplify) + + # expect 5_agency_a_city_a_1 to 3 and 6_agency_a_city_a_1 to 3 to + # persist since they have different attributes in unique_route_id col + # expect 1_agency_a_city_a_1 to 3 and 2_agency_a_city_a_1 to 3 to + # persist since node_id_from and node_id_to are different + # expect only one of 3_agency_a_city_a_1 to 3 or 4_agency_a_city_a_1 to 3 + # to persist but not both since they share the same attributes + assert len(result_edges) == 15 + result_edges.reset_index(inplace=True, drop=True) + assert result_edges.equals(expected_edges) + + +def test_simplify_transit_net_remove_extra_nodes( + transit_edges_to_simplify, transit_nodes_to_simplify_extra_nodes): + remove_edge_ids = ['4_agency_a_city_a_1', '4_agency_a_city_a_2', + '4_agency_a_city_a_3'] + expected_edges = transit_edges_to_simplify.loc[ + ~transit_edges_to_simplify['id'].isin(remove_edge_ids)] + expected_edges.reset_index(inplace=True, drop=True) + + remove_node_ids = ['12_agency_a_city_a'] + expected_nodes = transit_nodes_to_simplify_extra_nodes.loc[ + ~transit_nodes_to_simplify_extra_nodes.index.isin(remove_node_ids)] + + result_edges, result_nodes = gtfs_network._simplify_transit_net( + transit_edges_to_simplify, transit_nodes_to_simplify_extra_nodes) + + # expect all nodes except for 12_agency_a_city_a to persist + assert len(result_nodes) == 11 + assert result_nodes.equals(expected_nodes) + + # expect 5_agency_a_city_a_1 to 3 and 6_agency_a_city_a_1 to 3 to + # persist since they have different attributes in unique_route_id col + # expect 1_agency_a_city_a_1 to 3 and 2_agency_a_city_a_1 to 3 to + # persist since node_id_from and node_id_to are different + # expect only one of 3_agency_a_city_a_1 to 3 or 4_agency_a_city_a_1 to 3 + # to persist but not both since they share the same attributes + assert len(result_edges) == 15 + result_edges.reset_index(inplace=True, drop=True) + assert result_edges.equals(expected_edges) + + +def test_simplify_transit_net_empty_edges( + edges_empty, transit_nodes_to_simplify): + with pytest.raises(ValueError) as excinfo: + result_edges, result_nodes = gtfs_network._simplify_transit_net( + edges_empty, transit_nodes_to_simplify) + expected_error = ( + 'Unable to simplify transit network. Edges have 0 records to ' + 'simplify.') + assert expected_error in str(excinfo.value) + + +def test_remove_nodes_not_in_edges( + transit_edges_to_simplify, transit_nodes_to_simplify_extra_nodes): + remove_node_ids = ['12_agency_a_city_a'] + expected_nodes = transit_nodes_to_simplify_extra_nodes.loc[ + ~transit_nodes_to_simplify_extra_nodes.index.isin(remove_node_ids)] + + result_nodes = gtfs_network._remove_nodes_not_in_edges( + transit_nodes_to_simplify_extra_nodes, transit_edges_to_simplify, + from_id_col='node_id_from', to_id_col='node_id_to') + + # expect all nodes except for 12_agency_a_city_a to persist + assert len(result_nodes) == 11 + assert result_nodes.equals(expected_nodes) + + +def test_remove_nodes_not_in_edges_none_to_remove( + transit_edges_to_simplify, transit_nodes_to_simplify): + result_nodes = gtfs_network._remove_nodes_not_in_edges( + transit_nodes_to_simplify, transit_edges_to_simplify, + from_id_col='node_id_from', to_id_col='node_id_to') + + # expect all nodes to persist + assert len(result_nodes) == 11 + assert result_nodes.equals(transit_nodes_to_simplify) From 4214acd9b1358c8008c3d21fbfc5111652cf6ec7 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 15 Nov 2021 10:12:02 -0800 Subject: [PATCH 32/72] update from, to_yaml(), add_feed() param and data validation steps, minor formatting --- urbanaccess/gtfsfeeds.py | 90 ++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 46 deletions(-) diff --git a/urbanaccess/gtfsfeeds.py b/urbanaccess/gtfsfeeds.py index 4dbd706..e83888d 100644 --- a/urbanaccess/gtfsfeeds.py +++ b/urbanaccess/gtfsfeeds.py @@ -52,14 +52,14 @@ def from_yaml(cls, gtfsfeeddir=os.path.join(config.settings.data_folder, ------- urbanaccess_gtfsfeeds """ - + dtype_raise_error_msg = '{} must be a string' if not isinstance(gtfsfeeddir, str): - raise ValueError('gtfsfeeddir must be a string') + raise ValueError(dtype_raise_error_msg.format('gtfsfeeddir')) if not os.path.exists(gtfsfeeddir): raise ValueError('{} does not exist or was not found'.format( gtfsfeeddir)) if not isinstance(yamlname, str): - raise ValueError('yaml must be a string') + raise ValueError(dtype_raise_error_msg.format('yaml')) yaml_file = os.path.join(gtfsfeeddir, yamlname) @@ -67,7 +67,7 @@ def from_yaml(cls, gtfsfeeddir=os.path.join(config.settings.data_folder, yaml_config = yaml.safe_load(f) if not isinstance(yaml_config, dict): - raise ValueError('{} yamlname is not a dict'.format(yamlname)) + raise ValueError('{} is not a dict'.format(yamlname)) validkey = 'gtfs_feeds' if validkey not in yaml_config.keys(): @@ -75,23 +75,22 @@ def from_yaml(cls, gtfsfeeddir=os.path.join(config.settings.data_folder, for key in yaml_config['gtfs_feeds'].keys(): if not isinstance(key, str): - raise ValueError('{} must be a string'.format(key)) - for value in yaml_config['gtfs_feeds'][key]: - if not isinstance(value, str): - raise ValueError('{} must be a string'.format(value)) + raise ValueError(dtype_raise_error_msg.format(key)) + value = yaml_config['gtfs_feeds'][key] + if not isinstance(value, str): + raise ValueError(dtype_raise_error_msg.format(value)) unique_url_count = len( - pd.DataFrame.from_dict(yaml_config['gtfs_feeds'], orient='index')[ - 0].unique()) + pd.DataFrame.from_dict( + yaml_config['gtfs_feeds'], orient='index')[0].unique()) url_count = len(yaml_config['gtfs_feeds']) if unique_url_count != url_count: raise ValueError( - 'duplicate values were found when the passed add_dict ' - 'dictionary was added to the existing dictionary. Feed URL ' - 'values must be unique.') + 'duplicate values were found in YAML file: {}. Feed URL ' + 'values must be unique.'.format(yaml_file)) gtfsfeeds = cls(gtfs_feeds=yaml_config.get('gtfs_feeds', {})) - log('{} YAML successfully loaded with {} feeds.'.format(yaml_file, len( - yaml_config['gtfs_feeds']))) + log('{} YAML successfully loaded with {:,} feeds.'.format( + yaml_file, len(yaml_config['gtfs_feeds']))) return gtfsfeeds @@ -122,27 +121,29 @@ def add_feed(self, add_dict, replace=False): raise ValueError('add_dict is not a dict') if not isinstance(replace, bool): raise ValueError('replace is not bool') + dtype_raise_error_msg = '{} must be a string' - if replace is not True: - + # TODO: refactor to remove the need to have repeat code if + # replace True/False + if replace is False: for key in add_dict.keys(): if key in self.gtfs_feeds.keys(): - raise ValueError( - '{} passed in add_dict already exists in gtfs_feeds. ' - 'Only unique keys are allowed to be added.'.format( - key)) + msg = ('{} passed in add_dict already exists in ' + 'gtfs_feeds. Only unique keys are allowed to be ' + 'added.') + raise ValueError(msg.format(key)) if not isinstance(key, str): - raise ValueError('{} must be a string'.format(key)) - for value in add_dict[key]: - if not isinstance(value, str): - raise ValueError('{} must be a string'.format(value)) + raise ValueError(dtype_raise_error_msg.format(key)) + value = add_dict[key] + if not isinstance(value, str): + raise ValueError(dtype_raise_error_msg.format(value)) for key, value in add_dict.items(): if value in self.gtfs_feeds.values(): - raise ValueError('duplicate values were found when the ' - 'passed add_dict dictionary was added to ' - 'the existing dictionary. Feed URL ' - 'values must be unique.') + msg = ('duplicate values were found when the passed ' + 'add_dict dictionary was added to the existing ' + 'dictionary. Feed URL values must be unique.') + raise ValueError(msg) gtfs_feeds = self.gtfs_feeds.update(add_dict) else: @@ -151,14 +152,14 @@ def add_feed(self, add_dict, replace=False): log('{} passed in add_dict will replace existing {} feed ' 'in gtfs_feeds.'.format(key, key)) if not isinstance(key, str): - raise ValueError('{} must be a string'.format(key)) - for value in add_dict[key]: - if not isinstance(value, str): - raise ValueError('{} must be a string'.format(value)) - + raise ValueError(dtype_raise_error_msg.format(key)) + value = add_dict[key] + if not isinstance(value, str): + raise ValueError(dtype_raise_error_msg.format(value)) gtfs_feeds = self.gtfs_feeds.update(add_dict) - log('Added {} feeds to gtfs_feeds: {}'.format(len(add_dict), add_dict)) + log('Added {:,} feeds to gtfs_feeds: {}'.format( + len(add_dict), add_dict)) return gtfs_feeds @@ -196,9 +197,8 @@ def remove_feed(self, del_key=None, remove_all=False): for key in del_key: if key not in self.gtfs_feeds.keys(): - raise ValueError( - '{} key to delete was not found in gtfs_feeds'.format( - key)) + msg = '{} key to delete was not found in gtfs_feeds' + raise ValueError(msg.format(key)) del self.gtfs_feeds[key] log('Removed {} feed from gtfs_feeds'.format(key)) @@ -222,21 +222,19 @@ def to_yaml(self, gtfsfeeddir=os.path.join(config.settings.data_folder, ------- Nothing """ - + dtype_raise_error_msg = '{} must be a string' if not isinstance(gtfsfeeddir, str): - raise ValueError('gtfsfeeddir must be a string') + raise ValueError(dtype_raise_error_msg.format('gtfsfeeddir')) if not os.path.exists(gtfsfeeddir): - log( - '{} does not exist or was not found and will be ' - 'created'.format( - gtfsfeeddir)) + log('{} does not exist or was not found and will be ' + 'created'.format(gtfsfeeddir)) os.makedirs(gtfsfeeddir) if not isinstance(yamlname, str): - raise ValueError('yaml must be a string') + raise ValueError(dtype_raise_error_msg.format('yamlname')) yaml_file = os.path.join(gtfsfeeddir, yamlname) if overwrite is False and os.path.isfile(yaml_file) is True: raise ValueError( - '{} already exists. Rename or turn overwrite to True'.format( + '{} already exists. Rename or set overwrite to True'.format( yamlname)) else: with open(yaml_file, 'w') as f: From 0203ea4f0609c9b4d9eef2be8ddbf31840117bc0 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 15 Nov 2021 10:17:03 -0800 Subject: [PATCH 33/72] minor formatting, docstring updates --- urbanaccess/gtfsfeeds.py | 163 +++++++++++++++++++++++---------------- 1 file changed, 98 insertions(+), 65 deletions(-) diff --git a/urbanaccess/gtfsfeeds.py b/urbanaccess/gtfsfeeds.py index e83888d..88a5127 100644 --- a/urbanaccess/gtfsfeeds.py +++ b/urbanaccess/gtfsfeeds.py @@ -23,21 +23,21 @@ class urbanaccess_gtfsfeeds(object): ---------- gtfs_feeds : dict dictionary of the name of the transit service or agency GTFS feed - as the key -note: this name will be used as the feed folder name. + as the key (Note: this name will be used as the feed folder name. If the GTFS feed does not have a agency name in the agency.txt file - this key will be used to name the agency- and + this key will be used to name the agency) and the GTFS feed URL as the value to pass to the GTFS downloader as: {unique name of GTFS feed or transit service/agency : URL of feed} """ - def __init__(self, - gtfs_feeds={}): + def __init__(self, gtfs_feeds={}): self.gtfs_feeds = gtfs_feeds @classmethod - def from_yaml(cls, gtfsfeeddir=os.path.join(config.settings.data_folder, - 'gtfsfeeds'), + def from_yaml(cls, + gtfsfeeddir=os.path.join( + config.settings.data_folder, 'gtfsfeeds'), yamlname='gtfsfeeds.yaml'): """ Create an urbanaccess_gtfsfeeds instance from a saved YAML. @@ -48,9 +48,10 @@ def from_yaml(cls, gtfsfeeddir=os.path.join(config.settings.data_folder, Directory to load a YAML file. yamlname : str or file like, optional File name from which to load a YAML file. + Returns ------- - urbanaccess_gtfsfeeds + gtfsfeeds : object """ dtype_raise_error_msg = '{} must be a string' if not isinstance(gtfsfeeddir, str): @@ -97,6 +98,10 @@ def from_yaml(cls, gtfsfeeddir=os.path.join(config.settings.data_folder, def to_dict(self): """ Return a dict representation of an urbanaccess_gtfsfeeds instance. + + Returns + ------- + {'gtfs_feeds': urbanaccess_gtfsfeeds.gtfs_feeds} : dict """ return {'gtfs_feeds': self.gtfs_feeds} @@ -113,8 +118,12 @@ def add_feed(self, add_dict, replace=False): as: {unique name of GTFS feed or transit service/agency : URL of feed} replace : bool, optional - If key of dict is already in the UrbanAccess replace - the existing dict value with the value passed + If key of dict is already in the urbanaccess_gtfsfeeds object, + replace the existing dict value with the value passed + + Returns + ------- + urbanaccess_gtfsfeeds : object """ if not isinstance(add_dict, dict): @@ -175,17 +184,19 @@ def remove_feed(self, del_key=None, remove_all=False): remove_all : bool, optional if true, remove all keys from existing urbanaccess_gtfsfeeds instance - """ + Returns + ------- + Nothing + + """ if not isinstance(remove_all, bool): raise ValueError('remove_all is not bool') if del_key is None and remove_all: self.gtfs_feeds = {} log('Removed all feeds from gtfs_feeds') - else: - if not isinstance(del_key, (list, str)): raise ValueError('del_key must be a string or list of strings') if remove_all: @@ -202,10 +213,9 @@ def remove_feed(self, del_key=None, remove_all=False): del self.gtfs_feeds[key] log('Removed {} feed from gtfs_feeds'.format(key)) - def to_yaml(self, gtfsfeeddir=os.path.join(config.settings.data_folder, - 'gtfsfeeds'), - yamlname='gtfsfeeds.yaml', - overwrite=False): + def to_yaml(self, gtfsfeeddir=os.path.join( + config.settings.data_folder, 'gtfsfeeds'), + yamlname='gtfsfeeds.yaml', overwrite=False): """ Save an urbanaccess_gtfsfeeds representation to a YAML file. @@ -278,12 +288,10 @@ def search(api='gtfsdataexch', search_text=None, search_field=None, search_result_df : pandas.DataFrame Dataframe of search results displaying full feed metadata """ - - log( - 'Note: Your use of a GTFS feed is governed by each GTFS feed author ' - 'license terms. It is suggested you read the respective license ' - 'terms for the appropriate use of a GTFS feed.', - level=lg.WARNING) + msg = ('Note: Your use of a GTFS feed is governed by each GTFS feed author' + ' license terms. It is suggested you read the respective license ' + 'terms for the appropriate use of a GTFS feed.') + log(msg, level=lg.WARNING) if not isinstance(api, str): raise ValueError('{} must be a string'.format(api)) @@ -291,30 +299,27 @@ def search(api='gtfsdataexch', search_text=None, search_field=None, raise ValueError('{} is not currently a supported API'.format(api)) if config.settings.gtfs_api[api] is None or not isinstance( config.settings.gtfs_api[api], str): - raise ValueError('{} is not defined or defined incorrectly'.format( - api)) + raise ValueError('{} API is not defined or is ' + 'defined incorrectly'.format(api)) if not isinstance(match, str) or match not in ['contains', 'exact']: raise ValueError('match must be either: contains or exact') if not isinstance(add_feed, bool): raise ValueError('add_feed must be bool') if api == 'gtfsdataexch': - log( - 'Warning: The GTFSDataExchange is no longer being maintained as ' - 'of Summer 2016. ' - 'Data accessed here may be out of date.', level=lg.WARNING) + log('Warning: The GTFSDataExchange is no longer being maintained as ' + 'of Summer 2016. Data accessed here may be out of date.', + level=lg.WARNING) feed_table = pd.read_table(config.settings.gtfs_api[api], sep=',') - feed_table['date_added'] = pd.to_datetime(feed_table['date_added'], - unit='s') + feed_table['date_added'] = pd.to_datetime( + feed_table['date_added'], unit='s') feed_table['date_last_updated'] = pd.to_datetime( feed_table['date_last_updated'], unit='s') if search_text is None: - log( - 'No search parameters were passed. Returning full list of {} ' - 'GTFS feeds:'.format( - len(feed_table))) + log('No search parameters were passed. Returning full list of {} ' + 'GTFS feeds:'.format(len(feed_table))) return feed_table else: pass @@ -343,12 +348,12 @@ def search(api='gtfsdataexch', search_text=None, search_field=None, for text in search_text: if match == 'contains': search_result = feed_table[ - feed_table[col].str.contains(text, case=False, - na=False)] + feed_table[col].str.contains( + text, case=False, na=False)] if match == 'exact': search_result = feed_table[ - feed_table[col].str.match(text, case=False, - na=False)] + feed_table[col].str.match( + text, case=False, na=False)] search_result_df = search_result_df.append(search_result) search_result_df.drop_duplicates(inplace=True) @@ -356,7 +361,6 @@ def search(api='gtfsdataexch', search_text=None, search_field=None, len(search_result_df), search_text, search_field)) if len(search_result_df) != 0: - if add_feed: if overwrite_feed: zip_url = search_result_df[ @@ -365,10 +369,8 @@ def search(api='gtfsdataexch', search_text=None, search_field=None, search_result_dict = search_result_df.set_index('name')[ 'dataexchange_url'].to_dict() feeds.gtfs_feeds = search_result_dict - log( - 'Replaced all records in gtfs_feed list with the {} ' - 'found records:'.format( - len(search_result_df))) + log('Replaced all records in gtfs_feed list with the {} ' + 'found records:'.format(len(search_result_df))) else: zip_url = search_result_df[ 'dataexchange_url'] + 'latest.zip' @@ -379,7 +381,6 @@ def search(api='gtfsdataexch', search_text=None, search_field=None, log('Added {} records to gtfs_feed list:'.format( len(search_result_df))) return search_result_dict - else: return search_result_df @@ -399,17 +400,24 @@ def download(data_folder=os.path.join(config.settings.data_folder), data_folder : str, optional directory to download GTFS feed data to feed_name : str, optional - name of transit agency or service to use to name downloaded zipfile + name of transit agency or service to use to name downloaded zipfile. + If using feed_name and feed_url, cannot use feed_dict. feed_url : str, optional corresponding URL to the feed_name to use to download GTFS feed zipfile feed_dict : dict, optional Dictionary specifying the name of the transit service or agency GTFS feed as the key and the GTFS feed URL as the value: - {unique name of GTFS feed or transit service/agency : URL of feed} + {unique name of GTFS feed or transit service/agency : URL of feed}. + If using feed_dict, cannot use feed_name and feed_url. error_pause_duration : int, optional how long to pause in seconds before re-trying requests if error delete_zips : bool, optional - if true the downloaded zipfiles will be removed + if true the directory created by this function called 'gtfsfeed_zips' + that holds the downloaded zipfiles from each URL will be deleted. + This directory is located inside of the root data directory + specified in the 'data_folder' parameter under the the GTFS feed + directory for that zipfile for example: + data_folder\\feed_folder\\gtfsfeed_zips, default is False raise_url_error : bool, optional if true error will be raised when a request to a URL fails, URL failure response will be returned, else if false URL failures will @@ -417,7 +425,7 @@ def download(data_folder=os.path.join(config.settings.data_folder), to continue. Returns ------- - nothing + Nothing """ if (feed_name is not None and feed_url is None) or ( @@ -476,8 +484,8 @@ def download(data_folder=os.path.join(config.settings.data_folder), # TODO: add file counter and print number to user for feed_name_key, feed_url_value in feeds.gtfs_feeds.items(): start_time2 = time.time() - zipfile_path = os.path.join(download_folder, '{}.zip'.format( - feed_name_key)) + zipfile_name = '{}.zip'.format(feed_name_key) + zipfile_path = os.path.join(download_folder, zipfile_name) # resolve issues where request results in certificate verify failure ssl._create_default_https_context = ssl._create_unverified_context @@ -501,6 +509,8 @@ def download(data_folder=os.path.join(config.settings.data_folder), log(msg_download_succeed.format( feed_name_key, time.time() - start_time2, os.path.getsize(zipfile_path))) + # deal with 429 Too Many Requests and 504 Gateway Timeout + # separately elif status_code in [429, 504]: msg = ('URL at {} returned status code {} and no data. ' 'Re-trying request in {:.2f} seconds.') @@ -535,13 +545,12 @@ def download(data_folder=os.path.join(config.settings.data_folder), msg_no_connection.format( feed_url_value, traceback.format_exc())) else: + # for non http links such as FTP try: file = request.urlopen(feed_url_value) _zipfile_type_check(file=file, feed_url_value=feed_url_value) - file_path = os.path.join(download_folder, '{}.zip'.format( - feed_name_key)) - with open(file_path, "wb") as local_file: + with open(zipfile_path, "wb") as local_file: local_file.write(file.read()) log(msg_download_succeed.format( feed_name_key, time.time() - start_time2, @@ -562,6 +571,28 @@ def download(data_folder=os.path.join(config.settings.data_folder), def _unzip_util(zipfile_read_path, unzip_file_path, has_subzips): + """ + unzip GTFS feed zipfile in a root directory with resulting text files + in the root folder: gtfsfeed_text. If zipfile contains only zipfiles in + its subdirectory, recursively these zips will be extracted separately + into their own directories inside of root folder: gtfsfeed_text + + Parameters + ---------- + zipfile_read_path : string + full path to directory where zipfile to unzip is located + unzip_file_path : string + full path to directory where to unzip zipfile + has_subzips : bool + if true the parent zipfile contains zipfiles in its subdirectory. If + true subdirectory zipfiles will be extracted as separate feed folders. + + Returns + ------- + sub_zip_filelist : list + list of zipfile names that exist inside of the parent zipfile + directory, if there are none, will return Nothing + """ with zipfile.ZipFile(zipfile_read_path) as z: # required to deal with zipfiles that have subdirectories and # that were created on OSX @@ -619,13 +650,13 @@ def unzip(zip_rootpath, delete_zips=True): Returns ------- - nothing + Nothing """ start_time = time.time() - unzip_rootpath = os.path.join(os.path.dirname(zip_rootpath), - 'gtfsfeed_text') + unzip_rootpath = os.path.join( + os.path.dirname(zip_rootpath), 'gtfsfeed_text') if not os.path.exists(unzip_rootpath): os.makedirs(unzip_rootpath) @@ -652,8 +683,8 @@ def unzip(zip_rootpath, delete_zips=True): unzipfile_name, sub_zipfile.replace('.zip', '')) sub_unzipfile_path = os.path.join( os.path.split(unzip_file_path)[0], sub_unzipfile_name) - _unzip_util(sub_zipfile_path, sub_unzipfile_path, - has_subzips=True) + _unzip_util( + sub_zipfile_path, sub_unzipfile_path, has_subzips=True) msg = ('{} with nested zipfile: {} successfully extracted ' 'to: {}') log(msg.format(zfile, sub_zipfile, sub_unzipfile_path)) @@ -673,24 +704,26 @@ def unzip(zip_rootpath, delete_zips=True): def _zipfile_type_check(file, feed_url_value): """ - zipfile format checker helper + Helper function to check if HTTP response contains a zipfile. + Valid application content are: 'zip' or 'octet-stream' Parameters ---------- file : http.client.HTTPResponse - loaded zipfile object in memory + loaded zipfile HTTPResponse object in memory feed_url_value : str - URL to download GTFS feed zipfile + URL to download GTFS feed zipfile for informative purposes Returns ------- - nothing + Nothing """ # TODO: check for a better method to support cases that are zips but # dont have header info example: sanfordcommunityredevelopmentagency # GTFS feed URL - if 'zip' not in file.info().get('Content-Type') is True \ - or 'octet' not in file.info().get('Content-Type') is True: + content_type = file.info().get('Content-Type') + if 'zip' not in content_type is True \ + or 'octet' not in content_type is True: raise ValueError( 'data requested at {} is not a zipfile. ' - 'Data must be a zipfile'.format(feed_url_value)) + 'Data must be a zipfile.'.format(feed_url_value)) From b47071ebf04d5531c74fe66028ec15d47b0a80bf Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 15 Nov 2021 10:23:14 -0800 Subject: [PATCH 34/72] clean up validation logic for feed params in download() --- urbanaccess/gtfsfeeds.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/urbanaccess/gtfsfeeds.py b/urbanaccess/gtfsfeeds.py index 88a5127..1345a6e 100644 --- a/urbanaccess/gtfsfeeds.py +++ b/urbanaccess/gtfsfeeds.py @@ -403,7 +403,8 @@ def download(data_folder=os.path.join(config.settings.data_folder), name of transit agency or service to use to name downloaded zipfile. If using feed_name and feed_url, cannot use feed_dict. feed_url : str, optional - corresponding URL to the feed_name to use to download GTFS feed zipfile + corresponding URL to the feed_name to use to download GTFS feed + zipfile. If using feed_name and feed_url, cannot use feed_dict. feed_dict : dict, optional Dictionary specifying the name of the transit service or agency GTFS feed as the key and the GTFS feed URL as the value: @@ -427,7 +428,7 @@ def download(data_folder=os.path.join(config.settings.data_folder), ------- Nothing """ - + dtype_raise_error_msg = '{} must be a string' if (feed_name is not None and feed_url is None) or ( feed_url is not None and feed_name is None): raise ValueError( @@ -435,37 +436,33 @@ def download(data_folder=os.path.join(config.settings.data_folder), if feed_name is not None and feed_url is not None: if feed_dict is not None: - raise ValueError('feed_dict is not specified') + raise ValueError('only feed_dict or feed_name and ' + 'feed_url can be used at once. ' + 'Both cannot be used.') if not isinstance(feed_name, str) or not isinstance(feed_url, str): raise ValueError('either feed_name and or feed_url are not string') feeds.gtfs_feeds = {feed_name: feed_url} elif feed_dict is not None: - if feed_name is not None or feed_url is not None: - raise ValueError('either feed_name and or feed_url are not None') if not isinstance(feed_dict, dict): raise ValueError('feed_dict is not dict') for key in feed_dict.keys(): if not isinstance(key, str): - raise ValueError('{} must be a string'.format(key)) - for value in feed_dict[key]: - if not isinstance(value, str): - raise ValueError('{} must be a string'.format(value)) + raise ValueError(dtype_raise_error_msg.format(key)) + value = feed_dict[key] + if not isinstance(value, str): + raise ValueError(dtype_raise_error_msg.format(value)) - for key, value in feed_dict.items(): - if value in feeds.gtfs_feeds.values(): - raise ValueError( - 'duplicate values were found when the passed add_dict ' - 'dictionary was added to the existing dictionary. Feed ' - 'URL values must be unique.') + feed_dict_urls = list(feed_dict.values()) + has_dup_urls = len(set(feed_dict_urls)) != len(feed_dict_urls) + if has_dup_urls: + raise ValueError('duplicate values were found in feed_dict. ' + 'Feed URL values must be unique.') feeds.gtfs_feeds = feed_dict elif feed_name is None and feed_url is None and feed_dict is None: if len(feeds.gtfs_feeds) == 0: raise ValueError('No records were found in passed feed_dict') - feeds.gtfs_feeds - else: - raise ValueError('Passed parameters were incorrect or not specified.') download_folder = os.path.join(data_folder, 'gtfsfeed_zips') From 8403149c8df4d54f37fbb1ce2d408a1952fb59e7 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 15 Nov 2021 10:24:39 -0800 Subject: [PATCH 35/72] update _unzip_util() to handle cases where the same zip is inside a sub zip and GTFS txt exists inside of zipfile --- urbanaccess/gtfsfeeds.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/urbanaccess/gtfsfeeds.py b/urbanaccess/gtfsfeeds.py index 1345a6e..40e762e 100644 --- a/urbanaccess/gtfsfeeds.py +++ b/urbanaccess/gtfsfeeds.py @@ -591,15 +591,27 @@ def _unzip_util(zipfile_read_path, unzip_file_path, has_subzips): directory, if there are none, will return Nothing """ with zipfile.ZipFile(zipfile_read_path) as z: + zip_file_list = z.namelist() # required to deal with zipfiles that have subdirectories and # that were created on OSX - filelist = [file for file in z.namelist() if - file.endswith(".txt") and not file.startswith( - "__MACOSX")] + filelist = [file for file in zip_file_list if + file.endswith(".txt") and not file.startswith("__MACOSX")] # in cases where the zip contains multiple zips of GTFS feeds # such as the case with the SEPTA GTFS feed - sub_zip_filelist = [file for file in z.namelist() if + sub_zip_filelist = [file for file in zip_file_list if file.endswith(".zip")] + # Note: there must only be zips inside of the sub directory, if its a + # mix of zips and txt files we assume that the parent zip contained + # both its zipped txt contents and a copy of itself as a zip in which + # case the GTFS txt files will be dealt with correctly but we + # will ignore the subdirectory zip + if filelist and sub_zip_filelist: + msg = ('Warning: Zipfile contains {:,} zipfile(s): {} in addition ' + 'to GTFS txt files: {}. Zipfile(s) inside of the parent ' + 'zipfile will not be extracted.') + log(msg.format(len(sub_zip_filelist), sub_zip_filelist, filelist), + level=lg.WARNING) + sub_zip_filelist = [] # null subdirectory zipfile list if len(filelist) == 0 and len(sub_zip_filelist) > 0: msg = ('Zipfile contains {:,} zipfiles: {}. These will be ' 'extracted as separate feeds.') From 0f3fa848e68dea7bfc7accae1303a53110474ecb Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 15 Nov 2021 10:26:04 -0800 Subject: [PATCH 36/72] fix dir removal in unzip() that as not working correctly, change default to False --- urbanaccess/gtfsfeeds.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/urbanaccess/gtfsfeeds.py b/urbanaccess/gtfsfeeds.py index 40e762e..67b7590 100644 --- a/urbanaccess/gtfsfeeds.py +++ b/urbanaccess/gtfsfeeds.py @@ -7,6 +7,7 @@ import time import ssl from six.moves.urllib import request +import shutil from urbanaccess.utils import log from urbanaccess import config @@ -645,7 +646,7 @@ def _unzip_util(zipfile_read_path, unzip_file_path, has_subzips): return sub_zip_filelist -def unzip(zip_rootpath, delete_zips=True): +def unzip(zip_rootpath, delete_zips=False): """ unzip all GTFS feed zipfiles in a root directory with resulting text files in the root folder: gtfsfeed_text @@ -655,13 +656,13 @@ def unzip(zip_rootpath, delete_zips=True): zip_rootpath : string root directory to place downloaded GTFS feed zipfiles delete_zips : bool, optional - if true the downloaded zipfiles will be removed + if true the directory specified in zip_rootpath will be deleted which + will delete all the downloaded zipfiles, default is False Returns ------- Nothing """ - start_time = time.time() unzip_rootpath = os.path.join( @@ -704,8 +705,8 @@ def unzip(zip_rootpath, delete_zips=True): zfile, unzip_file_path)) if delete_zips: - os.remove(zip_rootpath) - log('Deleted {} folder'.format(zip_rootpath)) + shutil.rmtree(zip_rootpath) + log('Deleted {} folder and all its contents.'.format(zip_rootpath)) msg = ('GTFS feed zipfile extraction completed. Took {:,.2f} seconds ' 'for {:,} file(s)') log(msg.format(time.time() - start_time, len(zipfilelist))) From 167dbe8e61cd95450f116e313a8bdf4045bc81e4 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 15 Nov 2021 10:29:16 -0800 Subject: [PATCH 37/72] update unit tests and expand unit test coverage for gtfsfeeds.py --- urbanaccess/tests/test_gtfsfeeds.py | 874 +++++++++++++++++++++++++++- 1 file changed, 860 insertions(+), 14 deletions(-) diff --git a/urbanaccess/tests/test_gtfsfeeds.py b/urbanaccess/tests/test_gtfsfeeds.py index 7dc7858..0646dfe 100644 --- a/urbanaccess/tests/test_gtfsfeeds.py +++ b/urbanaccess/tests/test_gtfsfeeds.py @@ -2,9 +2,161 @@ import os import pandas as pd import yaml +import zipfile +from six.moves.urllib import request from urbanaccess import gtfsfeeds from urbanaccess.gtfsfeeds import feeds +from urbanaccess import config + + +@pytest.fixture +def dir_w_no_zips(tmpdir): + # make dir with no zip in it + dir_path = os.path.join(tmpdir.strpath, 'test') + os.makedirs(dir_path) + return dir_path + + +@pytest.fixture +def gtfs_feed_zipfile( + tmpdir, + agency_feed_1, stop_times_feed_1, stops_feed_1, + routes_feed_1, trips_feed_1, calendar_feed_1, + calendar_dates_feed_1): + agency_name = 'agency_a' + feed_file_dict = {'agency': agency_feed_1, + 'stop_times': stop_times_feed_1, + 'stops': stops_feed_1, + 'routes': routes_feed_1, + 'trips': trips_feed_1, + 'calendar': calendar_feed_1, + 'calendar_dates': calendar_dates_feed_1} + feed_path = os.path.join(tmpdir.strpath, agency_name) + os.makedirs(feed_path) + print('writing test data to dir: {}'.format(feed_path)) + filelist = [] + for feed_file, feed_df in feed_file_dict.items(): + feed_file_name = '{}.txt'.format(feed_file) + filelist.extend([feed_file_name]) + feed_df.to_csv(os.path.join(feed_path, feed_file_name), index=False) + + os.makedirs(os.path.join(feed_path, 'gtfsfeed_zips')) + zip_path = os.path.join(feed_path, 'gtfsfeed_zips', agency_name + '.zip') + filelist_abs_path = [os.path.join( + os.path.abspath(feed_path), item) for item in filelist] + with zipfile.ZipFile(zip_path, 'w') as z: + for file in filelist_abs_path: + print('writing file: {} to zip: {}'.format(file, zip_path)) + z.write(file, os.path.basename(file), + compress_type=zipfile.ZIP_DEFLATED) + # remove intermediate data + os.remove(file) + + return zip_path + + +@pytest.fixture +def gtfs_feed_zipfile_w_subzips( + tmpdir, + agency_feed_1, stop_times_feed_1, stops_feed_1, + routes_feed_1, trips_feed_1, calendar_feed_1, + calendar_dates_feed_1): + feed_file_dict = {'agency': agency_feed_1, + 'stop_times': stop_times_feed_1, + 'stops': stops_feed_1, + 'routes': routes_feed_1, + 'trips': trips_feed_1, + 'calendar': calendar_feed_1, + 'calendar_dates': calendar_dates_feed_1} + + feed_path = os.path.join(tmpdir.strpath) + os.makedirs(os.path.join(feed_path, 'gtfsfeed_zips')) + zipfile_list = [] + for agency_name in ['agency_a', 'agency_b']: + print('writing test data to dir: {}'.format(feed_path)) + filelist = [] + for feed_file, feed_df in feed_file_dict.items(): + feed_file_name = '{}.txt'.format(feed_file) + filelist.extend([feed_file_name]) + feed_df.to_csv(os.path.join(feed_path, feed_file_name), + index=False) + + zip_path = os.path.join( + feed_path, 'gtfsfeed_zips', agency_name + '.zip') + filelist_abs_path = [os.path.join( + os.path.abspath(feed_path), item) for item in filelist] + with zipfile.ZipFile(zip_path, 'w') as z: + for file in filelist_abs_path: + print('writing file: {} to zip: {}'.format(file, zip_path)) + z.write(file, os.path.basename(file), + compress_type=zipfile.ZIP_DEFLATED) + # remove intermediate data + os.remove(file) + + zipfile_list.extend([agency_name + '.zip']) + + filelist_abs_path = [os.path.join( + os.path.dirname(zip_path), item) for item in zipfile_list] + parent_zip_path = os.path.join( + feed_path, 'gtfsfeed_zips', 'zip_w_subzips.zip') + with zipfile.ZipFile(parent_zip_path, 'w') as z: + for file in filelist_abs_path: + print('writing file: {} to zip: {}'.format(file, parent_zip_path)) + z.write(file, os.path.basename(file), + compress_type=zipfile.ZIP_DEFLATED) + os.remove(file) + + return parent_zip_path + + +@pytest.fixture +def gtfs_feed_zipfile_w_subzips_and_txt( + tmpdir, + agency_feed_1, stop_times_feed_1, stops_feed_1, + routes_feed_1, trips_feed_1, calendar_feed_1, + calendar_dates_feed_1): + feed_file_dict = {'agency': agency_feed_1, + 'stop_times': stop_times_feed_1, + 'stops': stops_feed_1, + 'routes': routes_feed_1, + 'trips': trips_feed_1, + 'calendar': calendar_feed_1, + 'calendar_dates': calendar_dates_feed_1} + + feed_path = os.path.join(tmpdir.strpath) + os.makedirs(os.path.join(feed_path, 'gtfsfeed_zips')) + agency_name = 'agency_a' + print('writing test data to dir: {}'.format(feed_path)) + filelist = [] + for feed_file, feed_df in feed_file_dict.items(): + feed_file_name = '{}.txt'.format(feed_file) + filelist.extend([feed_file_name]) + feed_df.to_csv(os.path.join(feed_path, feed_file_name), index=False) + + zip_path = os.path.join(feed_path, agency_name + '.zip') + filelist_abs_path = [os.path.join( + os.path.abspath(feed_path), item) for item in filelist] + with zipfile.ZipFile(zip_path, 'w') as z: + for file in filelist_abs_path: + print('writing file: {} to zip: {}'.format(file, zip_path)) + z.write(file, os.path.basename(file), + compress_type=zipfile.ZIP_DEFLATED) + + filelist.extend([agency_name + '.zip']) + + filelist_abs_path = [os.path.join(feed_path, item) for item in filelist] + parent_zip_path = os.path.join( + feed_path, 'gtfsfeed_zips', 'zip_w_subzip_and_txt.zip') + with zipfile.ZipFile(parent_zip_path, 'w') as z: + for file in filelist_abs_path: + print('writing file: {} to zip: {}'.format(file, parent_zip_path)) + z.write(file, os.path.basename(file), + compress_type=zipfile.ZIP_DEFLATED) + # remove intermediate data + os.remove(file) + + return parent_zip_path @pytest.fixture @@ -14,6 +166,12 @@ def feed_dict1(): 'http://www.actransit.org/wp-content/uploads/GTFSJune182017B.zip'} +@pytest.fixture +def feed_object(feed_dict1): + feeds.add_feed(add_dict=feed_dict1) + return feeds + + @pytest.fixture def feed_dict2(): return { @@ -32,6 +190,38 @@ def feed_dict3(): '/latest.zip'} +@pytest.fixture +def feed_dict4(): + return { + 'septa': 'https://github.com/septadev/GTFS/releases/download/' + 'v202106061/gtfs_public.zip'} + + +@pytest.fixture +def feed_dict_invalid_1(): + return { + 1: 'http://www.actransit.org/wp-content/uploads/GTFSJune182017B.zip', + 'Bay Area Rapid Transit': + 'http://www.gtfs-data-exchange.com/agency/bay-area-rapid-transit' + '/latest.zip'} + + +@pytest.fixture +def feed_dict_invalid_2(): + return { + 'ac transit': 1, + 'Bay Area Rapid Transit': + 'http://www.gtfs-data-exchange.com/agency/bay-area-rapid-transit' + '/latest.zip'} + + +@pytest.fixture +def feed_dict_invalid_3(): + return { + 'ac transit v2': 'http://www.actransit.org/wp-content/uploads/' + 'GTFSJune182017B.zip'} + + @pytest.fixture def feed_yaml(tmpdir): yaml_dict = { @@ -48,11 +238,101 @@ def feed_yaml(tmpdir): return tmpdir.strpath +@pytest.fixture +def feed_yaml_invalid_1(tmpdir): + yaml_dict = [{ + 'gtfs_feeds': { + 'ac transit': 'http://www.actransit.org/wp-content/uploads' + '/GTFSJune182017B.zip', + 'Bay Area Rapid Transit': + 'http://www.gtfs-data-exchange.com/agency/bay-area-rapid' + '-transit/latest.zip'}}] + + yaml_path = os.path.join(tmpdir.strpath, 'gtfsfeeds_invalid_1.yaml') + with open(yaml_path, 'w') as f: + yaml.dump(yaml_dict, f, default_flow_style=False) + return tmpdir.strpath + + +@pytest.fixture +def feed_yaml_invalid_2(tmpdir): + yaml_dict = { + 'test_gtfs_feeds': { + 'ac transit': 'http://www.actransit.org/wp-content/uploads' + '/GTFSJune182017B.zip', + 'Bay Area Rapid Transit': + 'http://www.gtfs-data-exchange.com/agency/bay-area-rapid' + '-transit/latest.zip'}} + + yaml_path = os.path.join(tmpdir.strpath, 'gtfsfeeds_invalid_2.yaml') + with open(yaml_path, 'w') as f: + yaml.dump(yaml_dict, f, default_flow_style=False) + return tmpdir.strpath + + +@pytest.fixture +def feed_yaml_invalid_3(tmpdir): + yaml_dict = { + 'gtfs_feeds': { + 'ac transit': 1, + 'Bay Area Rapid Transit': + 'http://www.gtfs-data-exchange.com/agency/bay-area-rapid' + '-transit/latest.zip'}} + + yaml_path = os.path.join(tmpdir.strpath, 'gtfsfeeds_invalid_3.yaml') + with open(yaml_path, 'w') as f: + yaml.dump(yaml_dict, f, default_flow_style=False) + return tmpdir.strpath + + +@pytest.fixture +def feed_yaml_invalid_4(tmpdir): + yaml_dict = { + 'gtfs_feeds': { + 1: 'http://www.actransit.org/wp-content/uploads' + '/GTFSJune182017B.zip', + 'Bay Area Rapid Transit': + 'http://www.gtfs-data-exchange.com/agency/bay-area-rapid' + '-transit/latest.zip'}} + + yaml_path = os.path.join(tmpdir.strpath, 'gtfsfeeds_invalid_4.yaml') + with open(yaml_path, 'w') as f: + yaml.dump(yaml_dict, f, default_flow_style=False) + return tmpdir.strpath + + +@pytest.fixture +def feed_yaml_invalid_5(tmpdir): + yaml_dict = { + 'gtfs_feeds': { + 'ac transit': + 'http://www.gtfs-data-exchange.com/agency/bay-area-rapid' + '-transit/latest.zip', + 'Bay Area Rapid Transit': + 'http://www.gtfs-data-exchange.com/agency/bay-area-rapid' + '-transit/latest.zip'}} + + yaml_path = os.path.join(tmpdir.strpath, 'gtfsfeeds_invalid_5.yaml') + with open(yaml_path, 'w') as f: + yaml.dump(yaml_dict, f, default_flow_style=False) + return tmpdir.strpath + + def test_feed_object(): assert isinstance(gtfsfeeds.feeds, gtfsfeeds.urbanaccess_gtfsfeeds) assert isinstance(feeds.to_dict(), dict) +def test_to_dict(feed_object): + assert isinstance(feed_object.to_dict(), dict) + assert feed_object.to_dict() == { + 'gtfs_feeds': { + 'ac transit': 'http://www.actransit.org/wp-content/uploads' + '/GTFSJune182017B.zip'}} + # clear feeds from global memory + feeds.remove_feed(remove_all=True) + + def test_add_feed(feed_dict1, feed_dict2): feeds.add_feed(add_dict=feed_dict1) assert len(feeds.gtfs_feeds.keys()) == 1 @@ -69,6 +349,55 @@ def test_add_feed(feed_dict1, feed_dict2): feeds.remove_feed(remove_all=True) +def test_add_feed_invalid(feed_dict1, feed_dict3, feed_dict_invalid_1, + feed_dict_invalid_2, feed_dict_invalid_3): + with pytest.raises(ValueError) as excinfo: + feeds.add_feed(add_dict=['test feed']) + expected_error = 'add_dict is not a dict' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + feeds.add_feed(add_dict=feed_dict1, replace=1) + expected_error = 'replace is not bool' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + feeds.add_feed(add_dict=feed_dict1, replace=False) + feeds.add_feed(add_dict=feed_dict3, replace=False) + expected_error = ('ac transit passed in add_dict already exists in ' + 'gtfs_feeds. Only unique keys are allowed to be ' + 'added.') + assert expected_error in str(excinfo.value) + # clear feeds from global memory + feeds.remove_feed(remove_all=True) + with pytest.raises(ValueError) as excinfo: + feeds.add_feed(add_dict=feed_dict_invalid_1, replace=False) + expected_error = '1 must be a string' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + feeds.add_feed(add_dict=feed_dict_invalid_2, replace=False) + expected_error = '1 must be a string' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + feeds.add_feed(add_dict=feed_dict_invalid_1, replace=True) + expected_error = '1 must be a string' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + feeds.add_feed(add_dict=feed_dict_invalid_2, replace=True) + expected_error = '1 must be a string' + assert expected_error in str(excinfo.value) + # clear feeds from global memory + feeds.remove_feed(remove_all=True) + with pytest.raises(ValueError) as excinfo: + # add dict with same value twice to test adding to another dict + feeds.add_feed(add_dict=feed_dict1, replace=False) + feeds.add_feed(add_dict=feed_dict_invalid_3, replace=False) + expected_error = ('duplicate values were found when the passed ' + 'add_dict dictionary was added to the existing ' + 'dictionary. Feed URL values must be unique.') + assert expected_error in str(excinfo.value) + # clear feeds from global memory + feeds.remove_feed(remove_all=True) + + def test_remove_feed(feed_dict3): feeds.add_feed(add_dict=feed_dict3) feeds.remove_feed(del_key='ac transit') @@ -81,6 +410,29 @@ def test_remove_feed(feed_dict3): feeds.remove_feed(remove_all=True) +def test_remove_feed_invalid(feed_dict3): + feeds.add_feed(add_dict=feed_dict3) + with pytest.raises(ValueError) as excinfo: + feeds.remove_feed(del_key='ac transit', remove_all=1) + expected_error = 'remove_all is not bool' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + feeds.remove_feed(del_key=1, remove_all=False) + expected_error = 'del_key must be a string or list of strings' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + feeds.remove_feed(del_key='ac transit', remove_all=True) + expected_error = ('remove_all must be False in order to remove ' + 'individual records: ac transit') + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + feeds.remove_feed(del_key='ac transit 2', remove_all=False) + expected_error = 'ac transit 2 key to delete was not found in gtfs_feeds' + assert expected_error in str(excinfo.value) + # clear feeds from global memory + feeds.remove_feed(remove_all=True) + + def test_to_yaml_feed(tmpdir, feed_dict3): feeds.add_feed(add_dict=feed_dict3) feeds.to_yaml(tmpdir.strpath, overwrite=True) @@ -93,9 +445,45 @@ def test_to_yaml_feed(tmpdir, feed_dict3): feeds.remove_feed(remove_all=True) +def test_to_yaml_feed_invalid(tmpdir, feed_dict3): + with pytest.raises(ValueError) as excinfo: + feeds.add_feed(add_dict=feed_dict3) + feeds.to_yaml( + gtfsfeeddir=1, yamlname='gtfsfeeds.yaml', + overwrite=True) + expected_error = 'gtfsfeeddir must be a string' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + feeds.to_yaml( + gtfsfeeddir=tmpdir.strpath, yamlname=1, + overwrite=True) + expected_error = 'yamlname must be a string' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + feeds.to_yaml(tmpdir.strpath, overwrite=True) + feeds.to_yaml(tmpdir.strpath, overwrite=False) + expected_error = ('gtfsfeeds.yaml already exists. ' + 'Rename or set overwrite to True') + assert expected_error in str(excinfo.value) + # clear feeds from global memory + feeds.remove_feed(remove_all=True) + + +def test_to_yaml_feed_create_dir(tmpdir, feed_dict3): + path_to_create = os.path.join(tmpdir.strpath, 'temp') + feeds.add_feed(add_dict=feed_dict3) + feeds.to_yaml(path_to_create, overwrite=True) + + yaml_path = os.path.join(path_to_create, 'gtfsfeeds.yaml') + with open(yaml_path, 'r') as f: + yaml_config = yaml.safe_load(f) + assert yaml_config['gtfs_feeds'] == feed_dict3 + # clear feeds from global memory + feeds.remove_feed(remove_all=True) + + def test_from_yaml_feed(feed_yaml): - yaml_path = feed_yaml - feeds_from_yaml = feeds.from_yaml(yaml_path, 'gtfsfeeds.yaml') + feeds_from_yaml = feeds.from_yaml(feed_yaml, 'gtfsfeeds.yaml') assert isinstance(feeds_from_yaml, gtfsfeeds.urbanaccess_gtfsfeeds) assert len(feeds_from_yaml.gtfs_feeds.keys()) == 2 @@ -111,11 +499,80 @@ def test_from_yaml_feed(feed_yaml): feeds.remove_feed(remove_all=True) +def test_from_yaml_feed_invalid( + feed_yaml, feed_yaml_invalid_1, feed_yaml_invalid_2, + feed_yaml_invalid_3, feed_yaml_invalid_4, feed_yaml_invalid_5): + with pytest.raises(ValueError) as excinfo: + feeds_from_yaml = feeds.from_yaml( + gtfsfeeddir=1, yamlname='gtfsfeeds.yaml') + expected_error = 'gtfsfeeddir must be a string' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + feeds_from_yaml = feeds.from_yaml( + gtfsfeeddir='test_dir', yamlname='test.yaml') + expected_error = 'test_dir does not exist or was not found' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + feeds_from_yaml = feeds.from_yaml( + gtfsfeeddir=feed_yaml, yamlname=1) + expected_error = 'yaml must be a string' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + feeds_from_yaml = feeds.from_yaml( + gtfsfeeddir=feed_yaml_invalid_1, + yamlname='gtfsfeeds_invalid_1.yaml') + expected_error = 'gtfsfeeds_invalid_1.yaml is not a dict' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + feeds_from_yaml = feeds.from_yaml( + gtfsfeeddir=feed_yaml_invalid_2, + yamlname='gtfsfeeds_invalid_2.yaml') + expected_error = 'key gtfs_feeds was not found in YAML file' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + feeds_from_yaml = feeds.from_yaml( + gtfsfeeddir=feed_yaml_invalid_3, + yamlname='gtfsfeeds_invalid_3.yaml') + expected_error = '1 must be a string' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + feeds_from_yaml = feeds.from_yaml( + gtfsfeeddir=feed_yaml_invalid_4, + yamlname='gtfsfeeds_invalid_4.yaml') + expected_error = '1 must be a string' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + feeds_from_yaml = feeds.from_yaml( + gtfsfeeddir=feed_yaml_invalid_5, + yamlname='gtfsfeeds_invalid_5.yaml') + expected_error = ( + 'duplicate values were found in YAML file: {}. Feed URL values must ' + 'be unique.'.format(os.path.join( + feed_yaml_invalid_5, 'gtfsfeeds_invalid_5.yaml'))) + assert expected_error in str(excinfo.value) + + +def test_search_all_gtfs_data_exchange(): + search_result = gtfsfeeds.search( + api='gtfsdataexch') + + assert isinstance(search_result, pd.DataFrame) + assert search_result.empty is False + # expect all records to be returned + assert len(search_result) == 1000 + + col_list = ['dataexchange_url', 'dataexchange_id', 'name'] + for col in col_list: + assert col in search_result.columns + assert search_result[col].isnull().all() == False # noqa + + def test_search_contains_gtfs_data_exchange(): - search_result = gtfsfeeds.search(api='gtfsdataexch', - search_text=['ac transit', 'santa rosa'], - search_field=None, match='contains', - add_feed=False, overwrite_feed=False) + search_result = gtfsfeeds.search( + api='gtfsdataexch', + search_text=['ac transit', 'santa rosa'], + search_field=None, match='contains', + add_feed=False, overwrite_feed=False) assert isinstance(search_result, pd.DataFrame) assert search_result.empty is False @@ -141,10 +598,11 @@ def test_search_contains_add_feed_gtfs_data_exchange(): assert 'AC Transit' in feeds.gtfs_feeds.keys() # test overwrite feed - gtfsfeeds.search(api='gtfsdataexch', - search_text='Bay Area Rapid Transit', - search_field=None, match='exact', - add_feed=True, overwrite_feed=True) + gtfsfeeds.search( + api='gtfsdataexch', + search_text='Bay Area Rapid Transit', + search_field=None, match='exact', + add_feed=True, overwrite_feed=True) assert len(feeds.gtfs_feeds.keys()) == 1 assert 'Bay Area Rapid Transit' in feeds.gtfs_feeds.keys() @@ -154,15 +612,93 @@ def test_search_contains_add_feed_gtfs_data_exchange(): def test_search_exact_search_field_gtfs_data_exchange(): # test search field - search_result = gtfsfeeds.search(api='gtfsdataexch', - search_text='San Francisco Bay Area', - search_field=['area'], match='exact', - add_feed=False, overwrite_feed=False) + search_result = gtfsfeeds.search( + api='gtfsdataexch', + search_text='San Francisco Bay Area', + search_field=['area'], match='exact', + add_feed=False, overwrite_feed=False) assert len(search_result) == 8 +def test_search_invalid(): + with pytest.raises(ValueError) as excinfo: + search_result = gtfsfeeds.search( + api=1, + search_text=['ac transit', 'santa rosa'], + search_field=None, match='contains', + add_feed=False, overwrite_feed=False) + expected_error = '1 must be a string' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + search_result = gtfsfeeds.search( + api='test api', + search_text=['ac transit', 'santa rosa'], + search_field=None, match='contains', + add_feed=False, overwrite_feed=False) + expected_error = 'test api is not currently a supported API' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + # set config url to None to throw error + config.settings.gtfs_api = {'gtfsdataexch': None} + search_result = gtfsfeeds.search( + api='gtfsdataexch', + search_text=['ac transit', 'santa rosa'], + search_field=None, match='contains', + add_feed=False, overwrite_feed=False) + expected_error = ('gtfsdataexch API is not defined or is defined ' + 'incorrectly') + assert expected_error in str(excinfo.value) + # reset config url + config.settings.gtfs_api = { + 'gtfsdataexch': + 'http://www.gtfs-data-exchange.com/api/agencies?format=csv'} + with pytest.raises(ValueError) as excinfo: + search_result = gtfsfeeds.search( + api='gtfsdataexch', + search_text=['ac transit', 'santa rosa'], + search_field=None, match='contains test', + add_feed=False, overwrite_feed=False) + expected_error = 'match must be either: contains or exact' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + search_result = gtfsfeeds.search( + api='gtfsdataexch', + search_text=['ac transit', 'santa rosa'], + search_field=None, match='contains', + add_feed=1, overwrite_feed=False) + expected_error = 'add_feed must be bool' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + search_result = gtfsfeeds.search( + api='gtfsdataexch', + search_text=['ac transit', 'santa rosa'], + search_field='test field', match='contains', + add_feed=False, overwrite_feed=False) + expected_error = 'search_field is not list' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + search_result = gtfsfeeds.search( + api='gtfsdataexch', + search_text=['ac transit', 'santa rosa'], + search_field=['test field'], match='contains', + add_feed=False, overwrite_feed=False) + expected_error = 'test field column not found in available feed table' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + search_result = gtfsfeeds.search( + api='gtfsdataexch', + search_text=1, + search_field=['name'], match='contains', + add_feed=False, overwrite_feed=False) + expected_error = 'search_text is not list' + assert expected_error in str(excinfo.value) + + def test_download_gtfs_feed_via_feed_object(feed_dict3, tmpdir): feeds.add_feed(add_dict=feed_dict3) + # Note: ac transit url used here is an old link whose zipfile contains + # a sub directory zipfile that is an edge case in the zipfile structure + # expected however this edge case is handled by the download function tmp_path = tmpdir.strpath gtfsfeeds.download(data_folder=tmp_path) @@ -184,6 +720,32 @@ def test_download_gtfs_feed_via_feed_object(feed_dict3, tmpdir): feeds.remove_feed(remove_all=True) +def test_download_gtfs_feed_w_subzips(feed_dict4, tmpdir): + feeds.add_feed(add_dict=feed_dict4) + # Note: septa url used here contains a zipfile with 2 sub directory + # zipfiles + tmp_path = tmpdir.strpath + gtfsfeeds.download(data_folder=tmp_path) + + zipfilelist = ['septa.zip'] + filelist = ['septa_google_bus', 'septa_google_rail'] + txtlist = ['calendar.txt', 'routes.txt', 'stop_times.txt', + 'stops.txt', 'trips.txt'] + zip_path = os.path.join(tmp_path, 'gtfsfeed_zips') + txt_path = os.path.join(tmp_path, 'gtfsfeed_text') + for zipfile in zipfilelist: + assert os.path.exists(os.path.join(zip_path, zipfile)) is True + for folder in filelist: + check_path = os.path.join(txt_path, folder.replace('.zip', '')) + assert os.path.exists(check_path) is True + for txt in txtlist: + check_path = os.path.join( + txt_path, folder.replace('.zip', ''), txt) + assert os.path.exists(check_path) is True + # clear feeds from global memory + feeds.remove_feed(remove_all=True) + + def test_download_gtfs_feed_via_feed_name_and_dict(tmpdir): tmp_path = tmpdir.strpath gtfsfeeds.download( @@ -217,3 +779,287 @@ def test_download_gtfs_feed_via_feed_name_and_dict(tmpdir): assert os.path.exists(check_path) is True # clear feeds from global memory feeds.remove_feed(remove_all=True) + + +def test_download_invalid(tmpdir): + tmp_path = tmpdir.strpath + with pytest.raises(ValueError) as excinfo: + gtfsfeeds.download( + data_folder=tmp_path, + feed_name='test_agency', + feed_url=None, + feed_dict=None, + error_pause_duration=5, delete_zips=False) + expected_error = 'Both feed_name and feed_url parameters are required.' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + gtfsfeeds.download( + data_folder=tmp_path, + feed_name='test_agency', + feed_url=('http://www.gtfs-data-exchange.com/' + 'agency/bay-area-rapid-transit/latest.zip'), + feed_dict={ + 'test_agency_dict': 'http://www.gtfs-data-exchange.com/agency/' + 'ac-transit/latest.zip'}, + error_pause_duration=5, delete_zips=False) + expected_error = ('only feed_dict or feed_name and ' + 'feed_url can be used at once. Both cannot be used.') + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + gtfsfeeds.download( + data_folder=tmp_path, + feed_name=1, + feed_url=('http://www.gtfs-data-exchange.com/' + 'agency/bay-area-rapid-transit/latest.zip'), + feed_dict=None, + error_pause_duration=5, delete_zips=False) + expected_error = 'either feed_name and or feed_url are not string' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + gtfsfeeds.download( + data_folder=tmp_path, + feed_name=None, + feed_url=None, + feed_dict='test_agency_dict', + error_pause_duration=5, delete_zips=False) + expected_error = 'feed_dict is not dict' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + gtfsfeeds.download( + data_folder=tmp_path, + feed_name=None, + feed_url=None, + feed_dict={1: 'http://www.gtfs-data-exchange.com/agency/' + 'ac-transit/latest.zip'}, + error_pause_duration=5, delete_zips=False) + expected_error = '1 must be a string' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + gtfsfeeds.download( + data_folder=tmp_path, + feed_name=None, + feed_url=None, + feed_dict={'test_agency_dict': 1}, + error_pause_duration=5, delete_zips=False) + expected_error = '1 must be a string' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + gtfsfeeds.download( + data_folder=tmp_path, + feed_name=None, + feed_url=None, + feed_dict=None, + error_pause_duration=5, delete_zips=False) + expected_error = 'No records were found in passed feed_dict' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + gtfsfeeds.download( + data_folder=tmp_path, + feed_name=None, + feed_url=None, + feed_dict={ + 'test_agency_dict': + 'http://www.gtfs-data-exchange.com/agency/' + 'ac-transit/latest.zip', + 'test_agency_dict 2': + 'http://www.gtfs-data-exchange.com/agency/' + 'ac-transit/latest.zip'}, + error_pause_duration=5, delete_zips=False) + expected_error = ('duplicate values were found in feed_dict. Feed ' + 'URL values must be unique.') + assert expected_error in str(excinfo.value) + # clear feeds from global memory + feeds.remove_feed(remove_all=True) + + +def test_zipfile_type_check_valid(feed_dict1): + feed_url_value = feed_dict1['ac transit'] + file = request.urlopen(feed_url_value) + gtfsfeeds._zipfile_type_check(file, feed_url_value) + + +def test_unzip_util(gtfs_feed_zipfile): + unzip_dir = os.path.join(os.path.dirname(gtfs_feed_zipfile).replace( + 'gtfsfeed_zips', 'gtfsfeed_text')) + sub_zip_filelist = gtfsfeeds._unzip_util( + zipfile_read_path=gtfs_feed_zipfile, + unzip_file_path=unzip_dir, has_subzips=False) + + assert isinstance(sub_zip_filelist, list) + assert sub_zip_filelist == [] + + txtlist = ['agency.txt', 'calendar.txt', 'calendar_dates.txt', + 'routes.txt', 'stop_times.txt', 'stops.txt', 'trips.txt'] + # test that expected files have been extracted + assert os.path.exists(unzip_dir) is True + for txt in txtlist: + assert os.path.exists(os.path.join(unzip_dir, txt)) is True + + +def test_unzip_util_w_subzips_w_False_w_subzips(gtfs_feed_zipfile_w_subzips): + zip_root_dir = os.path.dirname(gtfs_feed_zipfile_w_subzips) + unzip_dir = os.path.join(zip_root_dir.replace( + 'gtfsfeed_zips', 'gtfsfeed_text')) + sub_zip_filelist = gtfsfeeds._unzip_util( + zipfile_read_path=gtfs_feed_zipfile_w_subzips, + unzip_file_path=unzip_dir, has_subzips=False) + + assert isinstance(sub_zip_filelist, list) + # returns all the zips that exist in the subdirectory that need to be + # extracted + expected_sub_zip_filelist = ['agency_a.zip', 'agency_b.zip'] + assert sub_zip_filelist == expected_sub_zip_filelist + + for zipfile in expected_sub_zip_filelist: + assert os.path.exists(os.path.join(unzip_dir, zipfile)) is True + + +def test_unzip_util_w_subzips_w_True_w_subzips(gtfs_feed_zipfile_w_subzips): + zip_root_dir = os.path.dirname(gtfs_feed_zipfile_w_subzips) + unzip_dir = os.path.join(zip_root_dir.replace( + 'gtfsfeed_zips', 'gtfsfeed_text')) + sub_zip_filelist = gtfsfeeds._unzip_util( + zipfile_read_path=gtfs_feed_zipfile_w_subzips, + unzip_file_path=unzip_dir, has_subzips=True) + + # expect it to extract all the zips so we expect no more zips to exist + # in the subdirectory and return nothing + assert sub_zip_filelist is None + + +def test_unzip_util_w_subzips_w_True_wo_subzips(gtfs_feed_zipfile): + zip_root_dir = os.path.dirname(gtfs_feed_zipfile) + unzip_dir = os.path.join(zip_root_dir.replace( + 'gtfsfeed_zips', 'gtfsfeed_text')) + sub_zip_filelist = gtfsfeeds._unzip_util( + zipfile_read_path=gtfs_feed_zipfile, + unzip_file_path=unzip_dir, has_subzips=True) + + # zip had no zips in its subdirectory so we dont expect any in list + # and instead expect nothing + assert sub_zip_filelist is None + + txtlist = ['agency.txt', 'calendar.txt', 'calendar_dates.txt', + 'routes.txt', 'stop_times.txt', 'stops.txt', 'trips.txt'] + + # test that all expected extracted data exists + assert os.path.exists(unzip_dir) is True + # test that subzip does not exist in dir + assert os.path.exists(os.path.join( + unzip_dir, 'agency_a.zip')) is False + for txt in txtlist: + check_path = os.path.join(unzip_dir, txt) + assert os.path.exists(check_path) is True + + +def test_unzip_util_w_subzips_w_True_w_subzips_and_txt( + gtfs_feed_zipfile_w_subzips_and_txt): + unzip_rootpath = os.path.join( + os.path.dirname(gtfs_feed_zipfile_w_subzips_and_txt).replace( + 'gtfsfeed_zips', 'gtfsfeed_text'), 'agency_a') + unzip_dir = os.path.dirname(gtfs_feed_zipfile_w_subzips_and_txt) + sub_zip_filelist = gtfsfeeds._unzip_util( + zipfile_read_path=gtfs_feed_zipfile_w_subzips_and_txt, + unzip_file_path=unzip_rootpath, has_subzips=False) + + assert isinstance(sub_zip_filelist, list) + # zip had no zips in its subdirectory so we dont expect any in list + assert sub_zip_filelist == [] + + txtlist = ['agency.txt', 'calendar.txt', 'calendar_dates.txt', + 'routes.txt', 'stop_times.txt', 'stops.txt', 'trips.txt'] + txt_path = os.path.join( + unzip_dir.replace('gtfsfeed_zips', 'gtfsfeed_text'), 'agency_a') + # test that the subzip is not in the unzip directory + subzip_path = os.path.join(txt_path, 'agency_a.zip') + assert os.path.exists(subzip_path) is False + assert os.path.exists(txt_path) is True + for txt in txtlist: + check_path = os.path.join(txt_path, txt) + assert os.path.exists(check_path) is True + + +def test_unzip_w_delete_False(gtfs_feed_zipfile): + zip_rootpath = os.path.dirname(gtfs_feed_zipfile) + gtfsfeeds.unzip(zip_rootpath=zip_rootpath, delete_zips=False) + + txtlist = ['agency.txt', 'calendar.txt', 'calendar_dates.txt', + 'routes.txt', 'stop_times.txt', 'stops.txt', 'trips.txt'] + folder_name = os.path.basename(gtfs_feed_zipfile).replace('.zip', '') + txt_path = os.path.join(zip_rootpath.replace( + 'gtfsfeed_zips', 'gtfsfeed_text'), folder_name) + + # test that directory and files have not been removed + assert os.path.exists(zip_rootpath) is True + assert os.path.exists(gtfs_feed_zipfile) is True + + # test that all expected extracted data exists + assert os.path.exists(txt_path) is True + for txt in txtlist: + check_path = os.path.join(txt_path, txt) + assert os.path.exists(check_path) is True + + +def test_unzip_w_delete_true(gtfs_feed_zipfile): + zip_rootpath = os.path.dirname(gtfs_feed_zipfile) + gtfsfeeds.unzip(zip_rootpath=zip_rootpath, delete_zips=True) + + txtlist = ['agency.txt', 'calendar.txt', 'calendar_dates.txt', + 'routes.txt', 'stop_times.txt', 'stops.txt', 'trips.txt'] + folder_name = os.path.basename(gtfs_feed_zipfile).replace('.zip', '') + txt_path = os.path.join(zip_rootpath.replace( + 'gtfsfeed_zips', 'gtfsfeed_text'), folder_name) + + # test that directory and files have been removed + assert os.path.exists(zip_rootpath) is False + assert os.path.exists(gtfs_feed_zipfile) is False + + # test that all expected extracted data exists + assert os.path.exists(txt_path) is True + for txt in txtlist: + check_path = os.path.join(txt_path, txt) + assert os.path.exists(check_path) is True + + +def test_unzip_w_subzips(gtfs_feed_zipfile_w_subzips): + zip_rootpath = os.path.dirname(gtfs_feed_zipfile_w_subzips) + gtfsfeeds.unzip(zip_rootpath=zip_rootpath, delete_zips=True) + + txtlist = ['agency.txt', 'calendar.txt', 'calendar_dates.txt', + 'routes.txt', 'stop_times.txt', 'stops.txt', 'trips.txt'] + txt_path = zip_rootpath.replace('gtfsfeed_zips', 'gtfsfeed_text') + expected_sub_dir = ['zip_w_subzips_agency_a', 'zip_w_subzips_agency_b'] + + # test that all expected extracted data exists + for sub_dir in expected_sub_dir: + assert os.path.exists(os.path.join(txt_path, sub_dir)) is True + for txt in txtlist: + check_path = os.path.join(txt_path, sub_dir, txt) + assert os.path.exists(check_path) is True + + +def test_unzip_w_subzips_and_txt(gtfs_feed_zipfile_w_subzips_and_txt): + zip_rootpath = os.path.dirname(gtfs_feed_zipfile_w_subzips_and_txt) + gtfsfeeds.unzip(zip_rootpath=zip_rootpath, delete_zips=True) + + txtlist = ['agency.txt', 'calendar.txt', 'calendar_dates.txt', + 'routes.txt', 'stop_times.txt', 'stops.txt', 'trips.txt'] + txt_path = zip_rootpath.replace('gtfsfeed_zips', 'gtfsfeed_text') + + # test that all expected extracted data exists + sub_dir = 'zip_w_subzip_and_txt' + assert os.path.exists(os.path.join(txt_path, sub_dir)) is True + # test that subzip does not exist in dir + assert os.path.exists(os.path.join( + txt_path, sub_dir, 'agency_a.zip')) is False + for txt in txtlist: + check_path = os.path.join(txt_path, sub_dir, txt) + assert os.path.exists(check_path) is True + + +def test_unzip_error(dir_w_no_zips): + with pytest.raises(ValueError) as excinfo: + gtfsfeeds.unzip(zip_rootpath=dir_w_no_zips, delete_zips=False) + expected_error = ('No zipfiles were found in specified ' + 'directory: {}'.format(dir_w_no_zips)) + assert expected_error in str(excinfo.value) From b63ffcc1f7d42eb40c64d3246fc1efa23985dbf6 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Wed, 17 Nov 2021 12:27:03 -0800 Subject: [PATCH 38/72] make nearest_neighbor() and connector_edges() public, move to network_utils.py and move unit tests --- urbanaccess/network.py | 106 +----------------------- urbanaccess/network_utils.py | 100 ++++++++++++++++++++++ urbanaccess/tests/test_network_utils.py | 83 +++++++++++++++++++ 3 files changed, 186 insertions(+), 103 deletions(-) create mode 100644 urbanaccess/network_utils.py create mode 100644 urbanaccess/tests/test_network_utils.py diff --git a/urbanaccess/network.py b/urbanaccess/network.py index 372dcff..7a8db9f 100644 --- a/urbanaccess/network.py +++ b/urbanaccess/network.py @@ -1,21 +1,12 @@ import time import os -import geopy -from geopy import distance - -from sklearn.neighbors import KDTree import pandas as pd from urbanaccess.utils import log, df_to_hdf5, hdf5_to_df +from urbanaccess.network_utils import connector_edges from urbanaccess import config -if int(geopy.__version__[0]) < 2: - dist_calc = distance.vincenty -else: - dist_calc = distance.geodesic - - class urbanaccess_network(object): """ An urbanaccess object of pandas.DataFrames representing @@ -53,34 +44,6 @@ def __init__(self, ua_network = urbanaccess_network() -def _nearest_neighbor(df1, df2): - """ - For a DataFrame of xy coordinates find the nearest xy - coordinates in a subsequent DataFrame - - Parameters - ---------- - df1 : pandas.DataFrame - DataFrame of records to return as the nearest record to records in df2 - df2 : pandas.DataFrame - DataFrame of records with xy coordinates for which to find the - nearest record in df1 for - Returns - ------- - df1.index.values[indexes] : pandas.Series - index of records in df1 that are nearest to the coordinates in df2 - """ - try: - df1_matrix = df1.to_numpy() - df2_matrix = df2.to_numpy() - except AttributeError: - df1_matrix = df1.values - df2_matrix = df2.values - kdt = KDTree(df1_matrix) - indexes = kdt.query(df2_matrix, k=1, return_distance=False) - return df1.index.values[indexes] - - def integrate_network(urbanaccess_network, headways=False, urbanaccess_gtfsfeeds_df=None, headway_statistic='mean'): """ @@ -140,7 +103,6 @@ def integrate_network(urbanaccess_network, headways=False, raise ValueError('headways must be bool type') if headways: - if urbanaccess_gtfsfeeds_df is None or \ urbanaccess_gtfsfeeds_df.headways.empty or \ urbanaccess_gtfsfeeds_df.stops.empty: @@ -177,7 +139,7 @@ def integrate_network(urbanaccess_network, headways=False, stops_df=urbanaccess_gtfsfeeds_df.stops, edges_w_routes=urbanaccess_network.transit_edges) - net_connector_edges = _connector_edges( + net_connector_edges = connector_edges( osm_nodes=urbanaccess_network.osm_nodes, transit_nodes=urbanaccess_network.transit_nodes, travel_speed_mph=3) @@ -188,7 +150,7 @@ def integrate_network(urbanaccess_network, headways=False, headway_statistic=headway_statistic) else: - urbanaccess_network.net_connector_edges = _connector_edges( + urbanaccess_network.net_connector_edges = connector_edges( osm_nodes=urbanaccess_network.osm_nodes, transit_nodes=urbanaccess_network.transit_nodes, travel_speed_mph=3) @@ -351,68 +313,6 @@ def _route_id_to_node(stops_df, edges_w_routes): return transit_nodes_wroutes -def _connector_edges(osm_nodes, transit_nodes, travel_speed_mph=3): - """ - Generate the connector edges between the OSM and transit edges and - weight by travel time - - Parameters - ---------- - osm_nodes : pandas.DataFrame - OSM nodes DataFrame - transit_nodes : pandas.DataFrame - transit nodes DataFrame - travel_speed_mph : int, optional - travel speed to use to calculate travel time across a - distance on an edge. units are in miles per hour (MPH) - for pedestrian travel this is assumed to be 3 MPH - - Returns - ------- - net_connector_edges : pandas.DataFrame - - """ - start_time = time.time() - - transit_nodes['nearest_osm_node'] = _nearest_neighbor( - osm_nodes[['x', 'y']], - transit_nodes[['x', 'y']]) - - net_connector_edges = [] - - for transit_node_id, row in transit_nodes.iterrows(): - # create new edge between the node in df2 (transit) - # and the node in OpenStreetMap (pedestrian) - - osm_node_id = int(row['nearest_osm_node']) - osm_row = osm_nodes.loc[osm_node_id] - - distance = dist_calc((row['y'], row['x']), - (osm_row['y'], osm_row['x'])).miles - time_ped_to_transit = distance / travel_speed_mph * 60 - time_transit_to_ped = distance / travel_speed_mph * 60 - - # save the edge - net_type = 'transit to osm' - net_connector_edges.append((transit_node_id, osm_node_id, - time_transit_to_ped, net_type)) - # make the edge bi-directional - net_type = 'osm to transit' - net_connector_edges.append((osm_node_id, transit_node_id, - time_ped_to_transit, net_type)) - - net_connector_edges = pd.DataFrame(net_connector_edges, - columns=["from", "to", - "weight", "net_type"]) - - log( - 'Connector edges between the OSM and transit network nodes ' - 'successfully completed. Took {:,.2f} seconds'.format( - time.time() - start_time)) - - return net_connector_edges - - def _format_pandana_edges_nodes(edge_df, node_df): """ Perform final formatting on nodes and edge DataFrames to prepare them diff --git a/urbanaccess/network_utils.py b/urbanaccess/network_utils.py new file mode 100644 index 0000000..26fb0c7 --- /dev/null +++ b/urbanaccess/network_utils.py @@ -0,0 +1,100 @@ +import time +import geopy +from geopy import distance + +from sklearn.neighbors import KDTree +import pandas as pd + +from urbanaccess.utils import log + + +if int(geopy.__version__[0]) < 2: + dist_calc = distance.vincenty +else: + dist_calc = distance.geodesic + + +def nearest_neighbor(df1, df2): + """ + For a DataFrame of xy coordinates find the nearest xy + coordinates in a subsequent DataFrame + + Parameters + ---------- + df1 : pandas.DataFrame + DataFrame of records to return as the nearest record to records in df2 + df2 : pandas.DataFrame + DataFrame of records with xy coordinates for which to find the + nearest record in df1 for + Returns + ------- + df1.index.values[indexes] : pandas.Series + index of records in df1 that are nearest to the coordinates in df2 + """ + try: + df1_matrix = df1.to_numpy() + df2_matrix = df2.to_numpy() + except AttributeError: + df1_matrix = df1.values + df2_matrix = df2.values + kdt = KDTree(df1_matrix) + indexes = kdt.query(df2_matrix, k=1, return_distance=False) + return df1.index.values[indexes] + + +def connector_edges(osm_nodes, transit_nodes, travel_speed_mph=3): + """ + Generate the connector edges between the OSM and transit edges and + weight by travel time + + Parameters + ---------- + osm_nodes : pandas.DataFrame + OSM nodes DataFrame + transit_nodes : pandas.DataFrame + transit nodes DataFrame + travel_speed_mph : int, optional + travel speed to use to calculate travel time across a + distance on an edge. units are in miles per hour (MPH) + for pedestrian travel this is assumed to be 3 MPH + + Returns + ------- + net_connector_edges : pandas.DataFrame + + """ + start_time = time.time() + + transit_nodes['nearest_osm_node'] = nearest_neighbor( + osm_nodes[['x', 'y']], + transit_nodes[['x', 'y']]) + + net_connector_edges = [] + for transit_node_id, row in transit_nodes.iterrows(): + # create new edge between the node in df2 (transit) + # and the node in OpenStreetMap (pedestrian) + osm_node_id = int(row['nearest_osm_node']) + osm_row = osm_nodes.loc[osm_node_id] + + distance_mi = dist_calc( + (row['y'], row['x']), (osm_row['y'], osm_row['x'])).miles + time_ped_to_transit = distance_mi / travel_speed_mph * 60 + time_transit_to_ped = distance_mi / travel_speed_mph * 60 + + # save the edge + net_type = 'transit to osm' + net_connector_edges.append((transit_node_id, osm_node_id, + time_transit_to_ped, net_type)) + # make the edge bi-directional + net_type = 'osm to transit' + net_connector_edges.append((osm_node_id, transit_node_id, + time_ped_to_transit, net_type)) + + net_connector_edges = pd.DataFrame( + net_connector_edges, columns=["from", "to", "weight", "net_type"]) + + log('Connector edges between the OSM and transit network nodes ' + 'successfully completed. Took {:,.2f} seconds'.format( + time.time() - start_time)) + + return net_connector_edges diff --git a/urbanaccess/tests/test_network_utils.py b/urbanaccess/tests/test_network_utils.py new file mode 100644 index 0000000..64a5be2 --- /dev/null +++ b/urbanaccess/tests/test_network_utils.py @@ -0,0 +1,83 @@ +import pytest +import pandas as pd +from urbanaccess import network_utils + + +@pytest.fixture +def osm_nodes_df(): + data = { + 'id': (1, 2, 3), + 'x': [-122.267546, -122.264479, -122.219119], + 'y': [37.802919, 37.808042, 37.782288] + } + osm_nodes = pd.DataFrame(data).set_index('id') + return osm_nodes + + +@pytest.fixture +def transit_nodes_df(): + data = { + 'node_id_route': ['1_transit_a', '2_transit_a', + '3_transit_a', '4_transit_a'], + 'x': [-122.265417, -122.266910, -122.269741, -122.238638], + 'y': [37.806372, 37.802687, 37.799480, 37.797234] + } + transit_nodes = pd.DataFrame(data).set_index('node_id_route') + return transit_nodes + + +@pytest.fixture +def expected_transit_nodes_neighbor_df(transit_nodes_df): + data = {'node_id_route': ['1_transit_a', '2_transit_a', + '3_transit_a', '4_transit_a'], + 'nearest_osm_node': [2, 1, 1, 3]} + index = range(4) + expected_transit_nodes = pd.concat( + [transit_nodes_df, + pd.DataFrame(data, index).set_index('node_id_route')], + axis=1) + return expected_transit_nodes + + +@pytest.fixture +def expected_connector_edge_df(): + data = {'from': ['1_transit_a', 2, + '2_transit_a', 1, + '3_transit_a', 1, + '4_transit_a', 3], + 'to': [2, '1_transit_a', + 1, '2_transit_a', + 1, '3_transit_a', + 3, '4_transit_a'], + 'weight': [2.521901, 2.521901, 0.766106, 0.766106, + 5.317242, 5.317242, 29.690540, 29.690540], + 'net_type': ['transit to osm', 'osm to transit', + 'transit to osm', 'osm to transit', + 'transit to osm', 'osm to transit', + 'transit to osm', 'osm to transit']} + + index = range(8) + expected_connector_edge = pd.DataFrame(data, index) + return expected_connector_edge + + +def test_nearest_neighbor(osm_nodes_df, transit_nodes_df, + expected_transit_nodes_neighbor_df): + transit_nodes_df['nearest_osm_node'] = network_utils.nearest_neighbor( + osm_nodes_df[['x', 'y']], transit_nodes_df[['x', 'y']]) + + assert expected_transit_nodes_neighbor_df.equals(transit_nodes_df) + + +def test_connector_edges(osm_nodes_df, transit_nodes_df, + expected_connector_edge_df): + net_connector_edges = network_utils.connector_edges( + osm_nodes_df, transit_nodes_df, travel_speed_mph=3) + net_connector_edges['weight'] = net_connector_edges['weight'].round(6) + expected_connector_edge_df['weight'] = \ + expected_connector_edge_df['weight'].round(6) + + col_order = ['from', 'to', 'weight', 'net_type'] + expected_connector_edge_df = expected_connector_edge_df[col_order] + net_connector_edges = net_connector_edges[col_order] + assert expected_connector_edge_df.equals(net_connector_edges) From 214c853f6de79f8130eda473e8b8100d4e7356a8 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Wed, 17 Nov 2021 12:36:27 -0800 Subject: [PATCH 39/72] added initial general unit test for integrate_network() wo headways --- urbanaccess/tests/test_network.py | 353 ++++++++++++++++++++++++------ urbanaccess/tests/test_plot.py | 2 + 2 files changed, 289 insertions(+), 66 deletions(-) diff --git a/urbanaccess/tests/test_network.py b/urbanaccess/tests/test_network.py index c0f5521..5b7baf2 100644 --- a/urbanaccess/tests/test_network.py +++ b/urbanaccess/tests/test_network.py @@ -1,85 +1,306 @@ import pytest import pandas as pd +from geopy import distance +from sklearn.neighbors import KDTree + from urbanaccess import network +from urbanaccess.network import urbanaccess_network as ua_net + + +def _build_expected_nearest_osm_node_data(osm_nodes=None, transit_nodes=None): + df1_matrix = osm_nodes[['x', 'y']].values + df2_matrix = transit_nodes[['x', 'y']].values + kdt = KDTree(df1_matrix) + indexes = kdt.query(df2_matrix, k=1, return_distance=False) + transit_nodes['nearest_osm_node'] = osm_nodes.index.values[indexes] + return transit_nodes + + +def _build_expected_connector_edges_data(osm_nodes=None, transit_nodes=None): + travel_speed_mph = 3 + net_connector_edges = [] + for transit_node_id, row in transit_nodes.iterrows(): + osm_node_id = int(row['nearest_osm_node']) + osm_row = osm_nodes.loc[osm_node_id] + + distance_mi = distance.geodesic( + (row['y'], row['x']), (osm_row['y'], osm_row['x'])).miles + travel_time = distance_mi / travel_speed_mph * 60 + + net_type = 'transit to osm' + net_connector_edges.append((transit_node_id, osm_node_id, + travel_time, net_type)) + # make the edge bi-directional + net_type = 'osm to transit' + net_connector_edges.append((osm_node_id, transit_node_id, + travel_time, net_type)) + + net_connector_edges = pd.DataFrame( + net_connector_edges, columns=["from", "to", "weight", "net_type"]) + return net_connector_edges + + +def _build_expected_intermediate_transit_node_edge_data( + transit_edges=None, transit_nodes=None): + transit_edges.rename( + columns={'node_id_from': 'from', 'node_id_to': 'to'}, inplace=True) + transit_nodes.reset_index(inplace=True, drop=False) + transit_nodes.rename(columns={'node_id': 'id'}, inplace=True) + return transit_edges, transit_nodes + + +def _build_expected_node_edge_net_data( + transit_edges=None, transit_nodes=None, walk_edges=None, + walk_nodes=None, expected_connector_edges=None): + edges_df = pd.concat( + [transit_edges, walk_edges, expected_connector_edges], + axis=0, ignore_index=True) + nodes_df = pd.concat( + [transit_nodes, walk_nodes], + axis=0, ignore_index=True) + + nodes_df['id_int'] = range(1, len(nodes_df) + 1) + + edges_df.rename(columns={'id': 'edge_id'}, inplace=True) + tmp = pd.merge(edges_df, nodes_df[['id', 'id_int']], left_on='from', + right_on='id', sort=False, copy=False, how='left') + tmp['from_int'] = tmp['id_int'] + tmp.drop(['id_int', 'id'], axis=1, inplace=True) + edges_df_wnumericid = pd.merge( + tmp, nodes_df[['id', 'id_int']], left_on='to', + right_on='id', sort=False, copy=False, + how='left') + edges_df_wnumericid['to_int'] = edges_df_wnumericid['id_int'] + edges_df_wnumericid.drop(['id_int', 'id'], axis=1, inplace=True) + col_list = edges_df_wnumericid.select_dtypes(include=['object']).columns + edges_df_wnumericid[col_list] = edges_df_wnumericid[col_list].astype(str) + + nodes_df.set_index('id_int', drop=True, inplace=True) + nodes_df['id'] = nodes_df['id'].astype(str) + nodes_df.drop(['nearest_osm_node'], axis=1, inplace=True) + return edges_df_wnumericid, nodes_df @pytest.fixture -def osm_nodes_df(): +def transit_edges_1_agency(): + # edges for outbound direction data = { - 'id': (1, 2, 3), - 'x': [-122.267546, -122.264479, -122.219119], - 'y': [37.802919, 37.808042, 37.782288] + 'node_id_from': ['1_agency_a_city_a', '2_agency_a_city_a', + '3_agency_a_city_a', '4_agency_a_city_a', + '5_agency_a_city_a', + '7_agency_a_city_a', '8_agency_a_city_a'], + 'node_id_to': ['2_agency_a_city_a', '3_agency_a_city_a', + '4_agency_a_city_a', '5_agency_a_city_a', + '6_agency_a_city_a', + '8_agency_a_city_a', '9_agency_a_city_a'], + 'weight': [2, 4, 4, 8, 6, 3, 5], + 'unique_agency_id': ['agency_a_city_a'] * 7, + 'unique_trip_id': ['a1_agency_a_city_a', 'a1_agency_a_city_a', + 'a1_agency_a_city_a', 'a1_agency_a_city_a', + 'a1_agency_a_city_a', + 'b1_agency_a_city_a', 'b1_agency_a_city_a'], + 'sequence': [1, 2, 3, 4, 5, 1, 2], + 'route_type': [3, 3, 3, 3, 3, 2, 2], + 'unique_route_id': ['10-101_agency_a_city_a', '10-101_agency_a_city_a', + '10-101_agency_a_city_a', '10-101_agency_a_city_a', + '10-101_agency_a_city_a', + 'B1_agency_a_city_a', 'B1_agency_a_city_a'], + 'net_type': ['transit'] * 7 } - osm_nodes = pd.DataFrame(data).set_index('id') - return osm_nodes + index = range(7) + direction_0 = pd.DataFrame(data, index) + + # edges for inbound direction + data = { + 'node_id_from': ['5_agency_a_city_a', '4_agency_a_city_a', + '3_agency_a_city_a', '2_agency_a_city_a', + '1_agency_a_city_a', + '8_agency_a_city_a', '7_agency_a_city_a'], + 'node_id_to': ['6_agency_a_city_a', '5_agency_a_city_a', + '4_agency_a_city_a', '3_agency_a_city_a', + '2_agency_a_city_a', + '9_agency_a_city_a', '8_agency_a_city_a'], + 'weight': [6, 8, 4, 4, 2, 5, 3], + 'unique_agency_id': ['agency_a_city_a'] * 7, + 'unique_trip_id': ['a2_agency_a_city_a', 'a2_agency_a_city_a', + 'a2_agency_a_city_a', 'a2_agency_a_city_a', + 'a2_agency_a_city_a', + 'b2_agency_a_city_a', 'b2_agency_a_city_a'], + 'sequence': [1, 2, 3, 4, 5, 1, 2], + 'route_type': [3, 3, 3, 3, 3, 2, 2], + 'unique_route_id': ['10-101_agency_a_city_a', '10-101_agency_a_city_a', + '10-101_agency_a_city_a', '10-101_agency_a_city_a', + '10-101_agency_a_city_a', + 'B1_agency_a_city_a', 'B1_agency_a_city_a'], + 'net_type': ['transit'] * 7 + } + index = range(7) + direction_1 = pd.DataFrame(data, index) + + df = pd.concat([direction_1, direction_0], ignore_index=True) + + # add ID + df['id'] = ( + df['unique_trip_id'].str.cat( + df['sequence'].astype('str'), sep='_')) + + return df @pytest.fixture -def transit_nodes_df(): +def transit_nodes_1_agency(): data = { - 'node_id_route': ['1_transit_a', '2_transit_a', - '3_transit_a', '4_transit_a'], - 'x': [-122.265417, -122.266910, -122.269741, -122.238638], - 'y': [37.806372, 37.802687, 37.799480, 37.797234] + 'stop_id': ['1', '2', '3', '4', '5', '6', + '7', '8', '9'], + 'stop_name': ['ave a', 'ave b', 'ave c', 'ave d', 'ave e', 'ave f', + '1st st', '2nd st', '3rd st'], + 'x': [-122.265609, -122.224274, -122.271604, -122.269029, + -122.267227, -122.251793, -122.444116, -122.017867, + -122.067423], + 'y': [37.797484, 37.774963, 37.803664, 37.80787, 37.828415, + 37.844601, 37.664174, 37.591208, 37.905628], + 'location_type': [1, 1, 1, 1, 1, 1, + 2, 2, 2], + 'wheelchair_boarding': [1, 0, 0, 0, 0, 0, + 1, 1, 1], + 'unique_agency_id': ['agency_a_city_a'] * 9, + 'route_type': [3] * 9, + 'net_type': ['transit'] * 9 } - transit_nodes = pd.DataFrame(data).set_index('node_id_route') - return transit_nodes + index = range(9) + df = pd.DataFrame(data, index) + + # add node ID as index + df['node_id'] = ( + df['stop_id'].str.cat( + df['unique_agency_id'].astype('str'), sep='_')) + df.set_index('node_id', inplace=True, drop=True) + + return df @pytest.fixture -def expected_transit_nodes_neighbor_df(transit_nodes_df): - data = {'node_id_route': ['1_transit_a', '2_transit_a', - '3_transit_a', '4_transit_a'], - 'nearest_osm_node': [2, 1, 1, 3]} - index = range(4) - expected_transit_nodes = pd.concat( - [transit_nodes_df, - pd.DataFrame(data, index).set_index('node_id_route')], - axis=1) - return expected_transit_nodes +def walk_nodes(): + data = { + 'id': [1, 2, 3, 4, 5, 6, 7, 8, 9], + 'x': [ + -122.265474, -122.272543, -122.273680, -122.262834, -122.269889, + -122.271170, -122.268333, -122.266974, -122.264433], + 'y': [ + 37.796897, 37.799683, 37.800206, 37.800964, 37.803884, + 37.804270, 37.809158, 37.808645, 37.807921], + # name is not expected in OSM nodes but is used here as placeholder + # for custom columns and as a reference for tests + 'name': [ + '1 8th & Oak', '2 8th & Franklin', '3 8th & Broadway', + '4 14th & Oak', '5 14th & Franklin', '6 14th & Broadway', + '7 Berkley & Broadway', '8 Berkley & Franklin', + '9 Berkley & Harrison'], + 'net_type': ['walk'] * 9} + index = range(9) + df = pd.DataFrame(data, index) + df.set_index('id', inplace=True, drop=False) + + return df + + +@pytest.fixture +def walk_edges(): + data = { + 'from': [1, 2, 3, 6, 7, 8, 9, 4, 4, 5, 2, 5], + 'to': [2, 3, 6, 7, 8, 9, 4, 1, 5, 6, 5, 8], + 'name': ['8th', '8th', 'Broadway', 'Broadway', 'Berkley', 'Berkley', + 'Lakeside', 'Oak', '14th', '14th', 'Franklin', 'Franklin'], + 'highway': ['residential', 'residential', 'primary', 'primary', + 'primary', 'primary', 'residential', 'residential', + 'primary', 'primary', 'residential', 'residential'], + 'weight': [0.3, 0.3, 0.5, 0.5, 0.6, 0.6, 1, 0.8, 0.8, 0.8, 0.4, 0.4], + 'oneway': ['yes', 'yes', 'no', 'no', 'no', 'no', 'yes', 'yes', 'no', + 'no', 'yes', 'yes'], + 'net_type': ['walk'] * 12, + } + index = range(12) + df = pd.DataFrame(data, index) + + # since this is a walk network we generate a bi-directed graph + twoway_df = df.copy() + twoway_df.rename(columns={'from': 'to', 'to': 'from'}, inplace=True) + + edge_df = pd.concat([df, twoway_df], axis=0, ignore_index=True) + edge_df.set_index(['from', 'to'], inplace=True, drop=False) + + return edge_df + + +@pytest.fixture +def expected_connector_edges(transit_nodes_1_agency, walk_nodes): + # build expected connector edge df + transit_nodes = _build_expected_nearest_osm_node_data( + walk_nodes, transit_nodes_1_agency) + net_connector_edges = _build_expected_connector_edges_data( + walk_nodes, transit_nodes) + + return net_connector_edges @pytest.fixture -def expected_connector_edge_df(): - data = {'from': ['1_transit_a', 2, - '2_transit_a', 1, - '3_transit_a', 1, - '4_transit_a', 3], - 'to': [2, '1_transit_a', - 1, '2_transit_a', - 1, '3_transit_a', - 3, '4_transit_a'], - 'weight': [2.521901, 2.521901, 0.766106, 0.766106, - 5.317242, 5.317242, 29.690540, 29.690540], - 'net_type': ['transit to osm', 'osm to transit', - 'transit to osm', 'osm to transit', - 'transit to osm', 'osm to transit', - 'transit to osm', 'osm to transit']} - - index = range(8) - expected_connector_edge = pd.DataFrame(data, index) - return expected_connector_edge - - -def test_nearest_neighbor(osm_nodes_df, transit_nodes_df, - expected_transit_nodes_neighbor_df): - transit_nodes_df['nearest_osm_node'] = network._nearest_neighbor( - osm_nodes_df[['x', 'y']], - transit_nodes_df[['x', 'y']]) - - assert expected_transit_nodes_neighbor_df.equals(transit_nodes_df) - - -def test_connector_edges(osm_nodes_df, transit_nodes_df, - expected_connector_edge_df): - net_connector_edges = network._connector_edges(osm_nodes_df, - transit_nodes_df, - travel_speed_mph=3) - net_connector_edges['weight'] = net_connector_edges['weight'].round(6) - expected_connector_edge_df['weight'] = \ - expected_connector_edge_df['weight'].round(6) - - col_order = ['from', 'to', 'weight', 'net_type'] - expected_connector_edge_df = expected_connector_edge_df[col_order] - net_connector_edges = net_connector_edges[col_order] - assert expected_connector_edge_df.equals(net_connector_edges) +def intermediate_transit_edges_nodes( + transit_edges_1_agency, transit_nodes_1_agency, walk_nodes, + expected_connector_edges): + # build intermediate data used downstream + transit_edges, transit_nodes = \ + _build_expected_intermediate_transit_node_edge_data( + transit_edges_1_agency, transit_nodes_1_agency) + # nodes are expected to have 'nearest_osm_node' column so generate the col + transit_nodes = _build_expected_nearest_osm_node_data \ + (walk_nodes, transit_nodes) + return transit_edges, transit_nodes + + +@pytest.fixture +def expected_net_edges_nodes( + intermediate_transit_edges_nodes, + walk_edges, walk_nodes, expected_connector_edges): + # build expected resulting integrated network edge and node tables + transit_edges, transit_nodes = intermediate_transit_edges_nodes + + edges_df, nodes_df = _build_expected_node_edge_net_data( + transit_edges, transit_nodes, walk_edges, walk_nodes, + expected_connector_edges) + + return nodes_df, edges_df + + +@pytest.fixture +def ua_net_object_w_transit_walk( + transit_edges_1_agency, transit_nodes_1_agency, + walk_edges, walk_nodes): + ua_net.transit_edges = transit_edges_1_agency.copy() + ua_net.transit_nodes = transit_nodes_1_agency.copy() + ua_net.osm_edges = walk_edges.copy() + ua_net.osm_nodes = walk_nodes.copy() + return ua_net + + +def test_integrate_network_wo_headways( + ua_net_object_w_transit_walk, intermediate_transit_edges_nodes, + walk_edges, walk_nodes, expected_connector_edges, + expected_net_edges_nodes): + expected_net_nodes, expected_net_edges = expected_net_edges_nodes + expected_transit_edges, expected_transit_nodes = \ + intermediate_transit_edges_nodes + + ua_net_object_result = network.integrate_network( + urbanaccess_network=ua_net_object_w_transit_walk, + headways=False, + urbanaccess_gtfsfeeds_df=None) + + # check that results match expected tables + assert ua_net_object_result.transit_edges.equals(expected_transit_edges) + assert ua_net_object_result.transit_nodes.equals(expected_transit_nodes) + assert ua_net_object_result.osm_edges.equals(walk_edges) + assert ua_net_object_result.osm_nodes.equals(walk_nodes) + assert ua_net_object_result.net_connector_edges.equals( + expected_connector_edges) + assert ua_net_object_result.net_edges.equals(expected_net_edges) + assert ua_net_object_result.net_nodes.equals(expected_net_nodes) diff --git a/urbanaccess/tests/test_plot.py b/urbanaccess/tests/test_plot.py index 43ec910..55f4cea 100644 --- a/urbanaccess/tests/test_plot.py +++ b/urbanaccess/tests/test_plot.py @@ -152,6 +152,8 @@ def drive_nodes(): 'y': [ 37.796897, 37.799683, 37.800206, 37.800964, 37.803884, 37.804270, 37.809158, 37.808645, 37.807921], + # name is not expected in OSM nodes but is used here as placeholder + # for custom columns and as a reference for tests 'name': [ '1 8th & Oak', '2 8th & Franklin', '3 8th & Broadway', '4 14th & Oak', '5 14th & Franklin', '6 14th & Broadway', From 74c7d35308b5f290b1a98361e7b06fcc7fb6b059 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 13:44:37 -0800 Subject: [PATCH 40/72] consolidated dict_to_yaml, yaml_to_dict, unique_stop_id, route_id, trip_id, stop_route, node_id_route to reusable functions, rename _unique_service_id -> _add_unique_service_id and _add_unique_agencyid -> _add_unique_agency_id --- urbanaccess/config.py | 35 ++------ urbanaccess/gtfs/headways.py | 62 ++++++++++---- urbanaccess/gtfs/load.py | 2 +- urbanaccess/gtfs/network.py | 31 +++---- urbanaccess/gtfs/utils_calendar.py | 18 ++-- urbanaccess/gtfs/utils_format.py | 2 +- urbanaccess/gtfsfeeds.py | 39 +-------- urbanaccess/network.py | 6 +- urbanaccess/utils.py | 132 +++++++++++++++++++++++++++++ 9 files changed, 209 insertions(+), 118 deletions(-) diff --git a/urbanaccess/config.py b/urbanaccess/config.py index c5f23c6..1076627 100644 --- a/urbanaccess/config.py +++ b/urbanaccess/config.py @@ -2,6 +2,8 @@ import yaml import numpy as np +from urbanaccess.utils import _dict_to_yaml, _yaml_to_dict + def _format_check(settings): """ @@ -105,19 +107,7 @@ def from_yaml(cls, configdir='configs', ------- urbanaccess_config """ - - if not isinstance(configdir, str): - raise ValueError('configdir must be a string') - if not os.path.exists(configdir): - raise ValueError('{} does not exist or was not found'.format( - configdir)) - if not isinstance(yamlname, str): - raise ValueError('yaml must be a string') - - yaml_file = os.path.join(configdir, yamlname) - - with open(yaml_file, 'r') as f: - yaml_config = yaml.safe_load(f) + yaml_config = _yaml_to_dict(yaml_dir=configdir, yaml_name=yamlname) settings = cls( data_folder=yaml_config.get('data_folder', 'data'), @@ -174,23 +164,8 @@ def to_yaml(self, configdir='configs', yamlname='urbanaccess_config.yaml', ------- Nothing """ - - if not isinstance(configdir, str): - raise ValueError('configdir must be a string') - if not os.path.exists(configdir): - raise ValueError('{} does not exist or was not found'.format( - configdir)) - os.makedirs(configdir) - if not isinstance(yamlname, str): - raise ValueError('yaml must be a string') - yaml_file = os.path.join(configdir, yamlname) - if overwrite is False and os.path.isfile(yaml_file) is True: - raise ValueError( - '{} already exists. Rename or turn overwrite to True'.format( - yamlname)) - else: - with open(yaml_file, 'w') as f: - yaml.dump(self.to_dict(), f, default_flow_style=False) + _dict_to_yaml(dictionary=self.to_dict(), yaml_dir=configdir, + yaml_name=yamlname, overwrite=overwrite) # set global variables diff --git a/urbanaccess/gtfs/headways.py b/urbanaccess/gtfs/headways.py index e0ac0e3..0ec98e8 100644 --- a/urbanaccess/gtfs/headways.py +++ b/urbanaccess/gtfs/headways.py @@ -2,7 +2,7 @@ import pandas as pd import time -from urbanaccess.utils import log +from urbanaccess.utils import log, _add_unique_trip_id, _add_unique_route_id from urbanaccess.gtfs.utils_validation import _check_time_range_format from urbanaccess.gtfs.network import _time_selector @@ -29,9 +29,7 @@ def _calc_headways_by_route_stop(df): start_time = time.time() - df['unique_stop_route'] = ( - df['unique_stop_id'].str.cat( - df['unique_route_id'].astype('str'), sep=',')) + df = _add_unique_stop_route(df) stop_route_groups = df.groupby('unique_stop_route') log('Starting route stop headway calculation for {:,} route ' @@ -87,12 +85,8 @@ def _headway_handler(interpolated_stop_times_df, trips_df, start_time = time.time() # add unique trip and route ID - trips_df['unique_trip_id'] = ( - trips_df['trip_id'].str.cat( - trips_df['unique_agency_id'].astype('str'), sep='_')) - trips_df['unique_route_id'] = ( - trips_df['route_id'].str.cat( - trips_df['unique_agency_id'].astype('str'), sep='_')) + trips_df = _add_unique_trip_id(trips_df) + trips_df = _add_unique_route_id(trips_df) columns = ['unique_route_id', 'service_id', 'unique_trip_id', 'unique_agency_id'] @@ -105,9 +99,7 @@ def _headway_handler(interpolated_stop_times_df, trips_df, trips_df = trips_df[columns] # add unique route ID - routes_df['unique_route_id'] = ( - routes_df['route_id'].str.cat( - routes_df['unique_agency_id'].astype('str'), sep='_')) + routes_df = _add_unique_route_id(routes_df) columns = ['unique_route_id', 'route_long_name', 'route_type', 'unique_agency_id'] @@ -133,9 +125,7 @@ def _headway_handler(interpolated_stop_times_df, trips_df, merge_df[['unique_stop_route', 'unique_stop_id', 'unique_route_id']], how='left', left_index=True, right_on='unique_stop_route', sort=False) headway_by_routestop_df.drop('unique_stop_route', axis=1, inplace=True) - headway_by_routestop_df['node_id_route'] = ( - headway_by_routestop_df['unique_stop_id'].str.cat( - headway_by_routestop_df['unique_route_id'].astype('str'), sep='_')) + headway_by_routestop_df = _add_node_id_route(headway_by_routestop_df) log('Headway calculation complete. Took {:,.2f} seconds.'.format( time.time() - start_time)) @@ -182,3 +172,43 @@ def headways(gtfsfeeds_df, headway_timerange): gtfsfeeds_df.headways = headways_df return gtfsfeeds_df + + +def _add_unique_stop_route(df): + """ + Create 'unique_stop_route' column and values in a pandas.DataFrame. + Intended for use with headways. + + Parameters + ---------- + df : pandas.DataFrame + pandas.DataFrame to generate 'unique_stop_route' column + + Returns + ------- + df : pandas.DataFrame + pandas.DataFrames with 'unique_stop_route' column added + """ + df['unique_stop_route'] = df['unique_stop_id'].str.cat( + df['unique_route_id'].astype('str'), sep=',') + return df + + +def _add_node_id_route(df): + """ + Create 'node_id_route' column and values in a pandas.DataFrame. + Intended for use with headways. + + Parameters + ---------- + df : pandas.DataFrame + pandas.DataFrame to generate 'node_id_route' column + + Returns + ------- + df : pandas.DataFrame + pandas.DataFrames with 'node_id_route' column added + """ + df['node_id_route'] = df['unique_stop_id'].str.cat( + df['unique_route_id'].astype('str'), sep='_') + return df diff --git a/urbanaccess/gtfs/load.py b/urbanaccess/gtfs/load.py index ea2a59b..ce43595 100644 --- a/urbanaccess/gtfs/load.py +++ b/urbanaccess/gtfs/load.py @@ -334,7 +334,7 @@ def gtfsfeed_to_df(gtfsfeed_path=None, validation=False, verbose=True, agency_df = pd.DataFrame() stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ - calendar_dates_df = utils_format._add_unique_agencyid( + calendar_dates_df = utils_format._add_unique_agency_id( agency_df=agency_df, stops_df=stops_df, routes_df=routes_df, diff --git a/urbanaccess/gtfs/network.py b/urbanaccess/gtfs/network.py index 3ef2d60..5bb9a7d 100644 --- a/urbanaccess/gtfs/network.py +++ b/urbanaccess/gtfs/network.py @@ -5,7 +5,9 @@ from datetime import datetime, timedelta import logging as lg -from urbanaccess.utils import log, df_to_hdf5, hdf5_to_df +from urbanaccess.utils import log, df_to_hdf5, hdf5_to_df, \ + _add_unique_trip_id, _add_unique_stop_id, _add_unique_route_id + from urbanaccess.gtfs.utils_validation import _check_time_range_format from urbanaccess.gtfs.utils_calendar import _calendar_service_id_selector, \ _trip_selector, _highest_freq_trips_date @@ -444,8 +446,7 @@ def _interpolate_stop_times(stop_times_df, calendar_selected_trips_df): df_list = [calendar_selected_trips_df, stop_times_df] for index, df in enumerate(df_list): - df['unique_trip_id'] = (df['trip_id'].str.cat( - df['unique_agency_id'].astype('str'), sep='_')) + df = _add_unique_trip_id(df) df_list[index] = df # sort stop times inplace based on first to last stop in @@ -631,9 +632,7 @@ def _interpolate_stop_times(stop_times_df, calendar_selected_trips_df): final_stop_times_df['departure_time_sec_interpolate'].astype(int) # add unique stop ID - final_stop_times_df['unique_stop_id'] = ( - final_stop_times_df['stop_id'].str.cat( - final_stop_times_df['unique_agency_id'].astype('str'), sep='_')) + final_stop_times_df = _add_unique_stop_id(final_stop_times_df) if missing_stop_times_count > 0: log('Departure stop time interpolation complete. ' @@ -918,9 +917,7 @@ def _stops_in_edge_table_selector(input_stops_df, input_stop_times_df): start_time = time.time() # add unique stop ID - input_stops_df['unique_stop_id'] = ( - input_stops_df['stop_id'].str.cat( - input_stops_df['unique_agency_id'].astype('str'), sep='_')) + input_stops_df = _add_unique_stop_id(input_stops_df) # Select stop IDs that match stop IDs in the subset stop time data that # match day and time selection @@ -954,9 +951,7 @@ def _format_transit_net_nodes(df): # add unique stop ID if 'unique_stop_id' not in df.columns: - df['unique_stop_id'] = ( - df['stop_id'].str.cat( - df['unique_agency_id'].astype('str'), sep='_')) + df = _add_unique_stop_id(df) final_node_df = pd.DataFrame() final_node_df['node_id'] = df['unique_stop_id'] @@ -1001,9 +996,7 @@ def _route_type_to_edge(transit_edge_df, stop_time_df): start_time = time.time() # create unique trip IDs - stop_time_df['unique_trip_id'] = ( - stop_time_df['trip_id'].str.cat( - stop_time_df['unique_agency_id'].astype('str'), sep='_')) + stop_time_df = _add_unique_trip_id(stop_time_df) # join route_id to the edge table merged_df = pd.merge( @@ -1044,12 +1037,8 @@ def _route_id_to_edge(transit_edge_df, trips_df): if 'unique_route_id' not in transit_edge_df.columns: # create unique trip and route IDs - trips_df['unique_trip_id'] = ( - trips_df['trip_id'].str.cat( - trips_df['unique_agency_id'].astype('str'), sep='_')) - trips_df['unique_route_id'] = ( - trips_df['route_id'].str.cat( - trips_df['unique_agency_id'].astype('str'), sep='_')) + trips_df = _add_unique_trip_id(trips_df) + trips_df = _add_unique_route_id(trips_df) transit_edge_df_with_routes = pd.merge( transit_edge_df, trips_df[['unique_trip_id', 'unique_route_id']], diff --git a/urbanaccess/gtfs/utils_calendar.py b/urbanaccess/gtfs/utils_calendar.py index ed2132b..4361720 100644 --- a/urbanaccess/gtfs/utils_calendar.py +++ b/urbanaccess/gtfs/utils_calendar.py @@ -4,7 +4,7 @@ from datetime import datetime as dt import logging as lg -from urbanaccess.utils import log +from urbanaccess.utils import log, _add_unique_trip_id def _trip_schedule_selector_validate_params(calendar_dates_df, params): @@ -1130,7 +1130,7 @@ def _calendar_service_id_selector( df_list.extend([calendar_df]) if has_cal_dates: df_list.extend([calendar_dates_df]) - df_list = _unique_service_id(df_list) + df_list = _add_unique_service_id(df_list) if has_cal: srvc_ids = _select_calendar_service_ids( @@ -1197,7 +1197,7 @@ def _trip_selector(trips_df, service_ids, verbose=True): log('--------------------------------') log('Selecting trip_id(s) with active service_ids that match ' 'the specified calendar and or calendar date parameters...') - df_list = _unique_service_id([trips_df]) + df_list = _add_unique_service_id([trips_df]) subset_trip_df = trips_df.loc[ trips_df['unique_service_id'].isin(service_ids)] @@ -1286,7 +1286,7 @@ def _highest_freq_trips_date(trips_df, calendar_df, calendar_dates_df): df_list.extend([calendar_df]) if has_cal_dates: df_list.extend([calendar_dates_df]) - df_list = _unique_service_id(df_list) + df_list = _add_unique_service_id(df_list) if has_cal: # convert cols to datetime expected format: 'yyyymmdd' e.g.: '20130825' @@ -1354,8 +1354,7 @@ def _highest_freq_trips_date(trips_df, calendar_df, calendar_dates_df): {date: date_add_rmv_srv_id_dict[date]['add']}) # select the trips and count - trips_df['unique_trip_id'] = trips_df['trip_id'].str.cat( - trips_df['unique_agency_id'].astype('str'), sep='_') + trips_df = _add_unique_trip_id(trips_df) date_trip_cnt = {} for date in unique_date_list: subset_trips_df = _trip_selector( @@ -1417,7 +1416,7 @@ def _highest_freq_trips_date(trips_df, calendar_df, calendar_dates_df): return max_date -def _unique_service_id(df_list): +def _add_unique_service_id(df_list): """ Create 'unique_service_id' column and values for a list of pandas.DataFrames @@ -1431,10 +1430,9 @@ def _unique_service_id(df_list): ------- df_list : list list of pandas.DataFrames with 'unique_service_id' column added - """ for index, df in enumerate(df_list): - df['unique_service_id'] = (df['service_id'].str.cat( - df['unique_agency_id'].astype('str'), sep='_')) + df['unique_service_id'] = df['service_id'].str.cat( + df['unique_agency_id'].astype('str'), sep='_') df_list[index] = df return df_list diff --git a/urbanaccess/gtfs/utils_format.py b/urbanaccess/gtfs/utils_format.py index ef6c7ad..d0f041c 100644 --- a/urbanaccess/gtfs/utils_format.py +++ b/urbanaccess/gtfs/utils_format.py @@ -393,7 +393,7 @@ def _stop_times_agencyid(stop_times_df, routes_df, trips_df, return merged_df -def _add_unique_agencyid(agency_df, stops_df, routes_df, +def _add_unique_agency_id(agency_df, stops_df, routes_df, trips_df, stop_times_df, calendar_df, calendar_dates_df, feed_folder, nulls_as_folder=True): diff --git a/urbanaccess/gtfsfeeds.py b/urbanaccess/gtfsfeeds.py index 67b7590..6354ee8 100644 --- a/urbanaccess/gtfsfeeds.py +++ b/urbanaccess/gtfsfeeds.py @@ -1,4 +1,3 @@ -import yaml import pandas as pd import traceback import zipfile @@ -11,6 +10,7 @@ from urbanaccess.utils import log from urbanaccess import config +from urbanaccess.utils import _dict_to_yaml, _yaml_to_dict # TODO: make class CamelCase @@ -54,22 +54,7 @@ def from_yaml(cls, ------- gtfsfeeds : object """ - dtype_raise_error_msg = '{} must be a string' - if not isinstance(gtfsfeeddir, str): - raise ValueError(dtype_raise_error_msg.format('gtfsfeeddir')) - if not os.path.exists(gtfsfeeddir): - raise ValueError('{} does not exist or was not found'.format( - gtfsfeeddir)) - if not isinstance(yamlname, str): - raise ValueError(dtype_raise_error_msg.format('yaml')) - - yaml_file = os.path.join(gtfsfeeddir, yamlname) - - with open(yaml_file, 'r') as f: - yaml_config = yaml.safe_load(f) - - if not isinstance(yaml_config, dict): - raise ValueError('{} is not a dict'.format(yamlname)) + yaml_config = _yaml_to_dict(yaml_dir=gtfsfeeddir, yaml_name=yamlname) validkey = 'gtfs_feeds' if validkey not in yaml_config.keys(): @@ -233,24 +218,8 @@ def to_yaml(self, gtfsfeeddir=os.path.join( ------- Nothing """ - dtype_raise_error_msg = '{} must be a string' - if not isinstance(gtfsfeeddir, str): - raise ValueError(dtype_raise_error_msg.format('gtfsfeeddir')) - if not os.path.exists(gtfsfeeddir): - log('{} does not exist or was not found and will be ' - 'created'.format(gtfsfeeddir)) - os.makedirs(gtfsfeeddir) - if not isinstance(yamlname, str): - raise ValueError(dtype_raise_error_msg.format('yamlname')) - yaml_file = os.path.join(gtfsfeeddir, yamlname) - if overwrite is False and os.path.isfile(yaml_file) is True: - raise ValueError( - '{} already exists. Rename or set overwrite to True'.format( - yamlname)) - else: - with open(yaml_file, 'w') as f: - yaml.dump(self.to_dict(), f, default_flow_style=False) - log('{} file successfully created'.format(yaml_file)) + _dict_to_yaml(dictionary=self.to_dict(), yaml_dir=gtfsfeeddir, + yaml_name=yamlname, overwrite=overwrite) # instantiate the UrbanAccess GTFS feed object diff --git a/urbanaccess/network.py b/urbanaccess/network.py index 7a8db9f..c40de4b 100644 --- a/urbanaccess/network.py +++ b/urbanaccess/network.py @@ -2,7 +2,7 @@ import os import pandas as pd -from urbanaccess.utils import log, df_to_hdf5, hdf5_to_df +from urbanaccess.utils import log, df_to_hdf5, hdf5_to_df, _add_unique_stop_id from urbanaccess.network_utils import connector_edges from urbanaccess import config @@ -273,9 +273,7 @@ def _route_id_to_node(stops_df, edges_w_routes): start_time = time.time() # create unique stop IDs - stops_df['unique_stop_id'] = ( - stops_df['stop_id'].str.cat( - stops_df['unique_agency_id'].astype('str'), sep='_')) + stops_df = _add_unique_stop_id(stops_df) tmp1 = pd.merge(edges_w_routes[['node_id_from', 'node_id_route_from']], stops_df[['unique_stop_id', 'stop_lat', 'stop_lon']], diff --git a/urbanaccess/utils.py b/urbanaccess/utils.py index 90d443e..ffe73e1 100644 --- a/urbanaccess/utils.py +++ b/urbanaccess/utils.py @@ -3,6 +3,7 @@ # log, _get_logger: https://github.com/gboeing/osmnx/blob/master/osmnx/utils.py import logging as lg +import yaml import unicodedata import sys import datetime as dt @@ -275,3 +276,134 @@ def hdf5_to_df(dir=None, filename=None, key=None): key, store.keys())) return df + + +def _yaml_to_dict(yaml_dir, yaml_name): + """ + Load a YAML file into a dictionary. + + Parameters + ---------- + yaml_dir : str, optional + Directory to load a YAML file. + yaml_name : str or file like, optional + File name from which to load a YAML file. + + Returns + ------- + yaml_dict : dict + """ + dtype_raise_error_msg = '{} must be a string.' + if not isinstance(yaml_dir, str): + raise ValueError(dtype_raise_error_msg.format('yaml directory')) + if not os.path.exists(yaml_dir): + raise ValueError('{} does not exist or was not found.'.format( + yaml_dir)) + if not isinstance(yaml_name, str): + raise ValueError(dtype_raise_error_msg.format('yaml file name')) + + yaml_file = os.path.join(yaml_dir, yaml_name) + + with open(yaml_file, 'r') as f: + yaml_dict = yaml.safe_load(f) + if not isinstance(yaml_dict, dict): + raise ValueError('Yaml data must be a dictionary.') + return yaml_dict + + +def _dict_to_yaml(dictionary, yaml_dir, yaml_name, overwrite=False): + """ + Save dictionary to a YAML file. + + Parameters + ---------- + dictionary : dict + Dictionary to save in a YAML file. + yaml_dir : str, optional + Directory to save a YAML file. + yaml_name : str or file like, optional + File name to which to save a YAML file. + overwrite : bool, optional + if true, will overwrite an existing YAML + file in specified directory if file names are the same + Returns + ------- + Nothing + """ + if not isinstance(dictionary, dict): + raise ValueError('Data to convert to YAML must be a dictionary.') + dtype_raise_error_msg = '{} must be a string.' + if not isinstance(yaml_dir, str): + raise ValueError(dtype_raise_error_msg.format('yaml directory')) + if not os.path.exists(yaml_dir): + log('{} does not exist or was not found and will be ' + 'created.'.format(yaml_dir)) + os.makedirs(yaml_dir) + if not isinstance(yaml_name, str): + raise ValueError(dtype_raise_error_msg.format('yaml file name')) + yaml_file = os.path.join(yaml_dir, yaml_name) + if overwrite is False and os.path.isfile(yaml_file) is True: + raise ValueError( + '{} already exists. Rename or set overwrite to True.'.format( + yaml_name)) + else: + with open(yaml_file, 'w') as f: + yaml.dump(dictionary, f, default_flow_style=False) + log('{} file successfully created.'.format(yaml_file)) + + +def _add_unique_trip_id(df): + """ + Create 'unique_trip_id' column and values in a pandas.DataFrame + + Parameters + ---------- + df : pandas.DataFrame + pandas.DataFrame to generate 'unique_trip_id' column + + Returns + ------- + df : pandas.DataFrame + pandas.DataFrames with 'unique_trip_id' column added + """ + df['unique_trip_id'] = df['trip_id'].str.cat( + df['unique_agency_id'].astype('str'), sep='_') + return df + + +def _add_unique_route_id(df): + """ + Create 'unique_route_id' column and values in a pandas.DataFrame + + Parameters + ---------- + df : pandas.DataFrame + pandas.DataFrame to generate 'unique_route_id' column + + Returns + ------- + df : pandas.DataFrame + pandas.DataFrames with 'unique_route_id' column added + """ + df['unique_route_id'] = df['route_id'].str.cat( + df['unique_agency_id'].astype('str'), sep='_') + return df + + +def _add_unique_stop_id(df): + """ + Create 'unique_stop_id' column and values in a pandas.DataFrame + + Parameters + ---------- + df : pandas.DataFrame + pandas.DataFrame to generate 'unique_stop_id' column + + Returns + ------- + df : pandas.DataFrame + pandas.DataFrames with 'unique_stop_id' column added + """ + df['unique_stop_id'] = df['stop_id'].str.cat( + df['unique_agency_id'].astype('str'), sep='_') + return df From 098f705b35831e6e1af421b2161aef9fa1f1eca3 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 13:49:42 -0800 Subject: [PATCH 41/72] minor formatting and docstring updates --- urbanaccess/config.py | 4 +- urbanaccess/gtfs/headways.py | 20 ++++---- urbanaccess/gtfs/load.py | 12 +++-- urbanaccess/gtfs/network.py | 2 +- urbanaccess/gtfs/utils_format.py | 3 +- urbanaccess/gtfs/utils_validation.py | 5 +- urbanaccess/gtfsfeeds.py | 4 +- urbanaccess/network.py | 74 ++++++++++++++-------------- urbanaccess/osm/load.py | 3 +- urbanaccess/osm/network.py | 17 ++++--- urbanaccess/utils.py | 9 ++-- 11 files changed, 79 insertions(+), 74 deletions(-) diff --git a/urbanaccess/config.py b/urbanaccess/config.py index 1076627..9e68d1b 100644 --- a/urbanaccess/config.py +++ b/urbanaccess/config.py @@ -158,8 +158,8 @@ def to_yaml(self, configdir='configs', yamlname='urbanaccess_config.yaml', yamlname : str or file like, optional File name to which to save a YAML file. overwrite : bool, optional - if true, overwrite an existing same name YAML - file in specified directory + if true, will overwrite an existing YAML + file in specified directory if file names are the same Returns ------- Nothing diff --git a/urbanaccess/gtfs/headways.py b/urbanaccess/gtfs/headways.py index 0ec98e8..a63847f 100644 --- a/urbanaccess/gtfs/headways.py +++ b/urbanaccess/gtfs/headways.py @@ -24,7 +24,6 @@ def _calc_headways_by_route_stop(df): DataFrame : pandas.DataFrame DataFrame of statistics of route stop headways in units of minutes """ - # TODO: Optimize for speed start_time = time.time() @@ -35,20 +34,19 @@ def _calc_headways_by_route_stop(df): log('Starting route stop headway calculation for {:,} route ' 'stops...'.format(len(stop_route_groups))) - results = {} - # suppress RuntimeWarning: Mean of empty slice. for this code block with warnings.catch_warnings(): warnings.simplefilter("ignore", category='RuntimeWarning') - + results = {} + col = 'departure_time_sec_interpolate' for unique_stop_route, stop_route_group in stop_route_groups: - stop_route_group.sort_values(['departure_time_sec_interpolate'], - ascending=True, inplace=True) - next_bus_time = (stop_route_group['departure_time_sec_interpolate'] - .iloc[1:].values) - prev_bus_time = (stop_route_group['departure_time_sec_interpolate'] - .iloc[:-1].values) - stop_route_group_headways = (next_bus_time - prev_bus_time) / 60 + stop_route_group.sort_values([col], ascending=True, inplace=True) + next_veh_time = (stop_route_group[col].iloc[1:].values) + prev_veh_time = (stop_route_group[col].iloc[:-1].values) + # compute headway with: + # (next veh arrival time - previous veh arrival time) / 60 seconds + # = headway in min + stop_route_group_headways = (next_veh_time - prev_veh_time) / 60 results[unique_stop_route] = (pd.Series(stop_route_group_headways) .describe()) diff --git a/urbanaccess/gtfs/load.py b/urbanaccess/gtfs/load.py index ce43595..32a2405 100644 --- a/urbanaccess/gtfs/load.py +++ b/urbanaccess/gtfs/load.py @@ -223,20 +223,24 @@ def gtfsfeed_to_df(gtfsfeed_path=None, validation=False, verbose=True, start_time = time.time() + dir_error_msg = "Directory: '{}' does not exist." if gtfsfeed_path is None: gtfsfeed_path = os.path.join( config.settings.data_folder, 'gtfsfeed_text') if not os.path.exists(gtfsfeed_path): - raise ValueError('{} does not exist.'.format(gtfsfeed_path)) + raise ValueError(dir_error_msg.format(gtfsfeed_path)) else: if not os.path.exists(gtfsfeed_path): - raise ValueError('{} does not exist.'.format(gtfsfeed_path)) + raise ValueError(dir_error_msg.format(gtfsfeed_path)) if not isinstance(gtfsfeed_path, str): raise ValueError('gtfsfeed_path must be a string.') if validation: - if bbox is None or remove_stops_outsidebbox is None or verbose is \ - None: + has_bbox_param = bbox is not None + has_remove_stops_param = remove_stops_outsidebbox is not None + has_verbose_param = verbose is not None + if not has_bbox_param or not has_remove_stops_param or not \ + has_verbose_param: raise ValueError( 'Attempted to run validation but bbox, verbose, and or ' 'remove_stops_outsidebbox were set to None. These parameters ' diff --git a/urbanaccess/gtfs/network.py b/urbanaccess/gtfs/network.py index 5bb9a7d..3da0d0c 100644 --- a/urbanaccess/gtfs/network.py +++ b/urbanaccess/gtfs/network.py @@ -678,7 +678,7 @@ def _time_selector(df, starttime, endtime, timerange_pad=None): starttime : str 24 hour clock formatted time 1 endtime : str - 24 hour clock formatted time 2, + 24 hour clock formatted time 2 timerange_pad: str, optional string indicating the number of hours minutes seconds to pad after the end of the time interval specified in 'timerange'. Must follow format diff --git a/urbanaccess/gtfs/utils_format.py b/urbanaccess/gtfs/utils_format.py index d0f041c..78a8290 100644 --- a/urbanaccess/gtfs/utils_format.py +++ b/urbanaccess/gtfs/utils_format.py @@ -859,8 +859,7 @@ def _apply_gtfs_definition(df, desc_dict): return df -def _add_txt_definitions(stops_df, routes_df, stop_times_df, - trips_df): +def _add_txt_definitions(stops_df, routes_df, stop_times_df, trips_df): """ Append GTFS definitions to stops, routes, stop times, and trips DataFrames diff --git a/urbanaccess/gtfs/utils_validation.py b/urbanaccess/gtfs/utils_validation.py index 25737ee..897113c 100644 --- a/urbanaccess/gtfs/utils_validation.py +++ b/urbanaccess/gtfs/utils_validation.py @@ -186,7 +186,8 @@ def _validate_gtfs(stops_df, feed_folder, def _check_time_range_format(timerange): """ - Check time range value format for expected schema + Check time range value format for expected schema and raise value error + if format is invalid Parameters ---------- @@ -197,7 +198,7 @@ def _check_time_range_format(timerange): Returns ------- - None + Nothing """ if timerange is None: raise ValueError('timerange cannot be None.') diff --git a/urbanaccess/gtfsfeeds.py b/urbanaccess/gtfsfeeds.py index 6354ee8..8f7c6e4 100644 --- a/urbanaccess/gtfsfeeds.py +++ b/urbanaccess/gtfsfeeds.py @@ -212,8 +212,8 @@ def to_yaml(self, gtfsfeeddir=os.path.join( yamlname : str or file like, optional File name to which to save a YAML file. overwrite : bool, optional - if true, overwrite an existing same name YAML file in specified - directory + if true, will overwrite an existing YAML + file in specified directory if file names are the same Returns ------- Nothing diff --git a/urbanaccess/network.py b/urbanaccess/network.py index c40de4b..04b3897 100644 --- a/urbanaccess/network.py +++ b/urbanaccess/network.py @@ -60,9 +60,9 @@ def integrate_network(urbanaccess_network, headways=False, if true, route stop level headways calculated in a previous step will be applied to the OSM to transit connector edge travel time weights as an approximate measure - of average passenger transit stop waiting time. + of average passenger transit stop wait time. urbanaccess_gtfsfeeds_df : object, optional - required if headways is true; the gtfsfeeds_dfs object that holds + required if 'headways' is true; the gtfsfeeds_dfs object that holds the corresponding headways and stops DataFrames headway_statistic : {'mean', 'std', 'min', 'max'}, optional required if headways is true; route stop headway @@ -80,13 +80,15 @@ def integrate_network(urbanaccess_network, headways=False, urbanaccess_network.net_edges : pandas.DataFrame urbanaccess_network.net_nodes : pandas.DataFrame """ - if urbanaccess_network is None: raise ValueError('urbanaccess_network is not specified') - if urbanaccess_network.transit_edges.empty \ - or urbanaccess_network.transit_nodes.empty \ - or urbanaccess_network.osm_edges.empty \ - or urbanaccess_network.osm_nodes.empty: + is_transit_edges_empty = urbanaccess_network.transit_edges.empty + is_transit_nodes_empty = urbanaccess_network.transit_nodes.empty + is_osm_edges_empty = urbanaccess_network.osm_edges.empty + is_osm_nodes_empty = urbanaccess_network.osm_nodes.empty + + if is_transit_edges_empty or is_transit_nodes_empty or \ + is_osm_edges_empty or is_osm_nodes_empty: raise ValueError( 'one of the network objects: transit_edges, transit_nodes, ' 'osm_edges, or osm_nodes were found to be empty.') @@ -103,17 +105,18 @@ def integrate_network(urbanaccess_network, headways=False, raise ValueError('headways must be bool type') if headways: - if urbanaccess_gtfsfeeds_df is None or \ - urbanaccess_gtfsfeeds_df.headways.empty or \ - urbanaccess_gtfsfeeds_df.stops.empty: + is_headway_df_empty = urbanaccess_gtfsfeeds_df.headways.empty + is_stops_df_empty = urbanaccess_gtfsfeeds_df.stops.empty + is_ua_gtfsfeeds_empty = urbanaccess_gtfsfeeds_df is None + if is_ua_gtfsfeeds_empty or is_headway_df_empty or is_stops_df_empty: raise ValueError( - 'stops and headway DataFrames were not found in the ' + 'stops and or headway DataFrames were not found in the ' 'urbanaccess_gtfsfeeds object. Please create these ' 'DataFrames in order to use headways.') valid_stats = ['mean', 'std', 'min', 'max'] - if headway_statistic not in valid_stats or not isinstance( - headway_statistic, str): + if headway_statistic not in valid_stats or \ + not isinstance(headway_statistic, str): raise ValueError('{} is not a supported statistic or is not a ' 'string'.format(headway_statistic)) @@ -157,8 +160,8 @@ def integrate_network(urbanaccess_network, headways=False, # change cols in transit edges and nodes if headways: - urbanaccess_network.transit_edges.rename(columns={ - 'node_id_route_from': 'from', 'node_id_route_to': 'to'}, + urbanaccess_network.transit_edges.rename( + columns={'node_id_route_from': 'from', 'node_id_route_to': 'to'}, inplace=True) urbanaccess_network.transit_edges.drop(['node_id_from', 'node_id_to'], inplace=True, axis=1) @@ -169,8 +172,8 @@ def integrate_network(urbanaccess_network, headways=False, urbanaccess_network.transit_edges.rename( columns={'node_id_from': 'from', 'node_id_to': 'to'}, inplace=True) urbanaccess_network.transit_nodes.reset_index(inplace=True, drop=False) - urbanaccess_network.transit_nodes.rename(columns={'node_id': 'id'}, - inplace=True) + urbanaccess_network.transit_nodes.rename( + columns={'node_id': 'id'}, inplace=True) # concat all network components urbanaccess_network.net_edges = pd.concat( @@ -225,28 +228,25 @@ def _add_headway_impedance(ped_to_transit_edges_df, headways_df, osm_to_transit_wheadway : pandas.DataFrame """ - start_time = time.time() - log( - '{} route stop headway will be used for pedestrian to transit edge ' - 'impedance.'.format( - headway_statistic)) + log('{} route stop headway will be used for pedestrian to transit edge ' + 'impedance.'.format(headway_statistic)) - osm_to_transit_wheadway = pd.merge(ped_to_transit_edges_df, headways_df[ - [headway_statistic, 'node_id_route']], - how='left', left_on=['to'], - right_on=['node_id_route'], sort=False, - copy=False) + osm_to_transit_wheadway = pd.merge( + ped_to_transit_edges_df, + headways_df[[headway_statistic, 'node_id_route']], + how='left', left_on=['to'], right_on=['node_id_route'], sort=False, + copy=False) + # divide headway statistic result by 2 assuming uniform distribution + # of passenger arrival at stop osm_to_transit_wheadway['weight_tmp'] = osm_to_transit_wheadway[ - 'weight'] + ( - osm_to_transit_wheadway[ - headway_statistic] / 2.0) + 'weight'] + (osm_to_transit_wheadway[headway_statistic] / 2.0) osm_to_transit_wheadway['weight_tmp'].fillna( osm_to_transit_wheadway['weight'], inplace=True) osm_to_transit_wheadway.drop('weight', axis=1, inplace=True) - osm_to_transit_wheadway.rename(columns={'weight_tmp': 'weight'}, - inplace=True) + osm_to_transit_wheadway.rename( + columns={'weight_tmp': 'weight'}, inplace=True) log('Headway impedance calculation completed. Took {:,.2f} seconds'.format( time.time() - start_time)) @@ -256,7 +256,8 @@ def _add_headway_impedance(ped_to_transit_edges_df, headways_df, def _route_id_to_node(stops_df, edges_w_routes): """ - Assign route ids to the transit nodes table + Assign route ids to the transit nodes table. Intended for use with + headway workflow. Parameters ---------- @@ -394,11 +395,12 @@ def save_network(urbanaccess_network, filename, Returns ------- - None + Nothing """ log('Writing HDF5 store...') - if urbanaccess_network is None or urbanaccess_network.net_edges.empty or \ - urbanaccess_network.net_nodes.empty: + is_net_edges_empty = urbanaccess_network.net_edges.empty + is_net_nodes_empty = urbanaccess_network.net_nodes.empty + if urbanaccess_network is None or is_net_edges_empty or is_net_nodes_empty: raise ValueError('Either no urbanaccess_network specified or ' 'net_edges or net_nodes are empty.') diff --git a/urbanaccess/osm/load.py b/urbanaccess/osm/load.py index c2f6da3..ebeea1c 100644 --- a/urbanaccess/osm/load.py +++ b/urbanaccess/osm/load.py @@ -63,10 +63,9 @@ def ua_network_from_bbox(lat_min=None, lng_min=None, lat_max=None, Returns ------- - nodesfinal, edgesfinal : pandas.DataFrame + nodes, edges : pandas.DataFrame """ - start_time = time.time() # returned OSM data allows travel in both directions diff --git a/urbanaccess/osm/network.py b/urbanaccess/osm/network.py index 9eae162..3ca14a7 100644 --- a/urbanaccess/osm/network.py +++ b/urbanaccess/osm/network.py @@ -36,12 +36,14 @@ def create_osm_net(osm_edges, osm_nodes, start_time = time.time() if not isinstance(network_type, str) or network_type is None: - raise ValueError('{!s} network_type passed is either not a ' - 'string or is None'.format(network_type)) + raise ValueError("'{!s}' network_type passed is either not a " + "string or is None.".format(network_type)) - # assign impedance to OSM edges - osm_edges['weight'] = (osm_edges[ - 'distance'] / 1609.34) / travel_speed_mph * 60 + # assign impedance to OSM edges where: + # (distance in meters / 1 mile in meters) / + # speed in mph * mph in mile per minute + osm_edges['weight'] = \ + (osm_edges['distance'] / 1609.34) / travel_speed_mph * 60 # assign node and edge net type osm_edges['net_type'] = network_type @@ -50,9 +52,8 @@ def create_osm_net(osm_edges, osm_nodes, ua_network.osm_nodes = osm_nodes ua_network.osm_edges = osm_edges - log( - 'Created OSM network with travel time impedance using a travel speed ' - 'of {} MPH. Took {:,.2f} seconds'.format( + log('Created OSM network with travel time impedance using a travel speed ' + 'of {} MPH. Took {:,.2f} seconds.'.format( travel_speed_mph, time.time() - start_time)) return ua_network diff --git a/urbanaccess/utils.py b/urbanaccess/utils.py index ffe73e1..4c8165b 100644 --- a/urbanaccess/utils.py +++ b/urbanaccess/utils.py @@ -140,7 +140,7 @@ def create_hdf5(dir=None, filename=None, overwrite_hdf5=False): Returns ------- - None + Nothing """ if dir is None: dir = config.settings.data_folder @@ -203,20 +203,21 @@ def df_to_hdf5(data=None, key=None, overwrite_key=False, dir=None, Returns ------- - None + Nothing """ hdf5_save_path = create_hdf5( dir=dir, filename=filename, overwrite_hdf5=overwrite_hdf5) store = pd.HDFStore(hdf5_save_path, mode='r') + h5_key_str = ''.join(['/', key]) - if not ''.join(['/', key]) in store.keys(): + if h5_key_str not in store.keys(): store.close() data.to_hdf(hdf5_save_path, key=key, mode='a', format='table') log(' DataFrame: {} saved in HDF5 store: {}.'.format( key, hdf5_save_path)) - elif ''.join(['/', key]) in store.keys() and overwrite_key: + elif h5_key_str in store.keys() and overwrite_key: store.close() data.to_hdf(hdf5_save_path, key=key, mode='a', format='table') log(' Existing DataFrame: {} overwritten in HDF5 store: {}.'.format( From 9ba9c671ba70a552394507b21479326d69651c34 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 13:50:14 -0800 Subject: [PATCH 42/72] fix missing raise for valuerror --- urbanaccess/gtfs/network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/urbanaccess/gtfs/network.py b/urbanaccess/gtfs/network.py index 3da0d0c..48d0657 100644 --- a/urbanaccess/gtfs/network.py +++ b/urbanaccess/gtfs/network.py @@ -1127,8 +1127,8 @@ def edge_impedance_by_route_type( # used in this function in case changes are made in the config obj if set(sorted(route_type_dict.keys())) != set( sorted(var_mode_id_lookup.keys())): - ValueError('ROUTES_MODE_TYPE_LOOKUP keys do not match keys in ' - 'var_mode_id_lookup. Keys must match.') + raise ValueError('ROUTES_MODE_TYPE_LOOKUP keys do not match keys in ' + 'var_mode_id_lookup. Keys must match.') for key, value in route_type_dict.items(): route_type_dict[key] = {'name': value, 'multiplier': var_mode_id_lookup[key]} From 2797fbe2e40243a4ee1f2fa6f7bf4cdcb597c068 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 13:50:53 -0800 Subject: [PATCH 43/72] fix negative time check in _timetoseconds() --- urbanaccess/gtfs/utils_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/urbanaccess/gtfs/utils_format.py b/urbanaccess/gtfs/utils_format.py index 78a8290..5434f0c 100644 --- a/urbanaccess/gtfs/utils_format.py +++ b/urbanaccess/gtfs/utils_format.py @@ -797,7 +797,7 @@ def _timetoseconds(df, time_cols): series_df = col_series.to_frame(name=''.join([col, '_sec'])) # check if times are negative if so display warning - if series_df.values.any() < 0: + if (series_df > 1).any()[0]: log('Warning: Some stop times in {} column are negative. ' 'Time should be positive. Suggest checking original ' 'GTFS feed stop_time file before proceeding.'.format(col), From cedb64f48c4777f1f7018b6e93eae00c4917f9d0 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 13:52:13 -0800 Subject: [PATCH 44/72] simplify and update prints for low connectivity OSM node process --- urbanaccess/osm/load.py | 59 ++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/urbanaccess/osm/load.py b/urbanaccess/osm/load.py index ebeea1c..30fe4fd 100644 --- a/urbanaccess/osm/load.py +++ b/urbanaccess/osm/load.py @@ -82,33 +82,32 @@ def ua_network_from_bbox(lat_min=None, lng_min=None, lat_max=None, # remove low connectivity nodes and return cleaned nodes and edges if remove_lcn: log('Checking for low connectivity nodes...') - pandana_net = Network(nodes['x'], nodes['y'], - edges['from'], edges['to'], edges[['distance']]) - lcn = pandana_net.low_connectivity_nodes(impedance=10000, count=10, - imp_name='distance') - log( - '{:,} out of {:,} nodes ({:.2f} percent of total) were ' - 'identified as having low connectivity and have ' - 'been removed.'.format(len(lcn), len(nodes), - (len(lcn) / len(nodes)) * 100)) - - rm_nodes = set(lcn) - - nodes_to_keep = ~nodes.index.isin(rm_nodes) - edges_to_keep = ~( - edges['from'].isin(rm_nodes) | edges['to'].isin(rm_nodes)) - - nodes = nodes.loc[nodes_to_keep] - edges = edges.loc[edges_to_keep] - - log('Completed OSM data download and graph node and edge table ' - 'creation in {:,.2f} seconds'.format(time.time() - start_time)) - - return nodes, edges - - else: - - log('Completed OSM data download and graph node and edge table ' - 'creation in {:,.2f} seconds'.format(time.time() - start_time)) - - return nodes, edges + # use twoway=True only for low connectivity purposes + pandana_net = Network( + node_x=nodes['x'], node_y=nodes['y'], + edge_from=edges['from'], edge_to=edges['to'], + edge_weights=edges[['distance']], twoway=True) + lcn = pandana_net.low_connectivity_nodes( + impedance=10000, count=10, imp_name='distance') + len_lcn = len(lcn) + len_nodes = len(nodes) + if len_lcn > 0: + rm_nodes = set(lcn) + + nodes_to_keep = ~nodes.index.isin(rm_nodes) + edges_to_keep = ~( + edges['from'].isin(rm_nodes) | edges['to'].isin(rm_nodes)) + + nodes = nodes.loc[nodes_to_keep] + edges = edges.loc[edges_to_keep] + msg = ('{:,} out of {:,} node(s) ({:.2f} percent of total) were ' + 'identified as having low connectivity and have ' + 'been removed.') + log(msg.format(len_lcn, len_nodes, (len_lcn / len_nodes) * 100)) + else: + log('No nodes were identified as having low connectivity.') + + log('Completed OSM data download and graph node and edge table ' + 'creation in {:,.2f} seconds.'.format(time.time() - start_time)) + + return nodes, edges From 59929f212d951b5a17aac74ca0688e59c48a1ec2 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 13:55:40 -0800 Subject: [PATCH 45/72] update config settings format validation and added validation for gtfs api key --- urbanaccess/config.py | 46 +++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/urbanaccess/config.py b/urbanaccess/config.py index 9e68d1b..5dd25de 100644 --- a/urbanaccess/config.py +++ b/urbanaccess/config.py @@ -1,5 +1,4 @@ -import os -import yaml +import itertools import numpy as np from urbanaccess.utils import _dict_to_yaml, _yaml_to_dict @@ -17,20 +16,37 @@ def _format_check(settings): ------- Nothing """ - - valid_keys = ['data_folder', 'images_folder', 'image_filename', 'logs_folder', - 'log_file', 'log_console', 'log_name', 'log_filename', - 'txt_encoding', 'gtfs_api'] - - for key in settings.keys(): - if key not in valid_keys: - raise ValueError('{} not found in list of valid configuration ' - 'keys: {}.'.format(key, valid_keys)) - if not isinstance(key, str): - raise ValueError('{} must be a string.'.format(key)) - if key == 'log_file' or key == 'log_console': + gtfs_api_schema_keys = ['gtfsdataexch'] + str_keys = ['data_folder', 'images_folder', 'image_filename', + 'logs_folder', 'log_name', 'log_filename', 'txt_encoding'] + bool_keys = ['log_file', 'log_console'] + dict_keys = ['gtfs_api'] + valid_keys = list(itertools.chain(str_keys, bool_keys, dict_keys)) + valid_keys = sorted(valid_keys) + settings_keys = list(settings.keys()) + + if set(settings_keys) != set(valid_keys): + raise ValueError("Configuration keys: {} do not match required " + "keys: {}.".format(settings_keys, valid_keys)) + for key in settings_keys: + if key in str_keys: + if not isinstance(settings[key], str): + raise ValueError( + "Key: '{}' value must be string.".format(key)) + if key in bool_keys: if not isinstance(settings[key], bool): - raise ValueError('{} must be boolean.'.format(key)) + raise ValueError( + "Key: '{}' value must be boolean.".format(key)) + if key in dict_keys: + for gtfs_api_key in settings['gtfs_api'].keys(): + if gtfs_api_key not in gtfs_api_schema_keys: + raise ValueError( + "gtfs_api key: '{}' does not match valid key(s):" + " {}.".format(gtfs_api_key, gtfs_api_schema_keys)) + if not isinstance(settings['gtfs_api'][gtfs_api_key], str): + raise ValueError( + "gtfs_api key: '{}' value must be string.".format( + gtfs_api_key)) # TODO: make class CamelCase From 1180515f6ec39e2de69bbeb86076b8408e73df9b Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 13:56:42 -0800 Subject: [PATCH 46/72] update error msgs and prints for ua_gtfsfeeds.from_yaml() --- urbanaccess/gtfsfeeds.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/urbanaccess/gtfsfeeds.py b/urbanaccess/gtfsfeeds.py index 8f7c6e4..fafd9f8 100644 --- a/urbanaccess/gtfsfeeds.py +++ b/urbanaccess/gtfsfeeds.py @@ -60,6 +60,7 @@ def from_yaml(cls, if validkey not in yaml_config.keys(): raise ValueError('key gtfs_feeds was not found in YAML file') + dtype_raise_error_msg = '{} must be a string.' for key in yaml_config['gtfs_feeds'].keys(): if not isinstance(key, str): raise ValueError(dtype_raise_error_msg.format(key)) @@ -73,11 +74,11 @@ def from_yaml(cls, if unique_url_count != url_count: raise ValueError( 'duplicate values were found in YAML file: {}. Feed URL ' - 'values must be unique.'.format(yaml_file)) + 'values must be unique.'.format(yamlname)) gtfsfeeds = cls(gtfs_feeds=yaml_config.get('gtfs_feeds', {})) log('{} YAML successfully loaded with {:,} feeds.'.format( - yaml_file, len(yaml_config['gtfs_feeds']))) + yamlname, len(yaml_config['gtfs_feeds']))) return gtfsfeeds From 2968454e63116ccbee544067c86c1e89af23716d Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 13:58:07 -0800 Subject: [PATCH 47/72] update _route_id_to_node() to retain original stops gtfs columns when creating transit node table --- urbanaccess/network.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/urbanaccess/network.py b/urbanaccess/network.py index 04b3897..752dfa7 100644 --- a/urbanaccess/network.py +++ b/urbanaccess/network.py @@ -277,7 +277,7 @@ def _route_id_to_node(stops_df, edges_w_routes): stops_df = _add_unique_stop_id(stops_df) tmp1 = pd.merge(edges_w_routes[['node_id_from', 'node_id_route_from']], - stops_df[['unique_stop_id', 'stop_lat', 'stop_lon']], + stops_df, how='left', left_on='node_id_from', right_on='unique_stop_id', sort=False, copy=False) tmp1.rename(columns={'node_id_route_from': 'node_id_route', @@ -285,7 +285,7 @@ def _route_id_to_node(stops_df, edges_w_routes): 'stop_lat': 'y'}, inplace=True) tmp2 = pd.merge(edges_w_routes[['node_id_to', 'node_id_route_to']], - stops_df[['unique_stop_id', 'stop_lat', 'stop_lon']], + stops_df, how='left', left_on='node_id_to', right_on='unique_stop_id', sort=False, copy=False) @@ -294,12 +294,13 @@ def _route_id_to_node(stops_df, edges_w_routes): 'stop_lat': 'y'}, inplace=True) - transit_nodes_wroutes = pd.concat([tmp1[['node_id_route', 'x', 'y']], - tmp2[['node_id_route', 'x', 'y']]], - axis=0) + transit_nodes_wroutes = pd.concat([tmp1, tmp2], axis=0) transit_nodes_wroutes.drop_duplicates( subset='node_id_route', keep='first', inplace=True) + # drop temp columns + transit_nodes_wroutes.drop( + columns=['node_id_to', 'node_id_to'], inplace=True) # set node index to be unique stop ID transit_nodes_wroutes = transit_nodes_wroutes.set_index('node_id_route') From 2b06fb3f4a3498daa8c6e11a31883530f3fc6f1b Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 13:59:00 -0800 Subject: [PATCH 48/72] config.settings.log_level -> lg.INFO since settings has no log_level param yet --- urbanaccess/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/urbanaccess/utils.py b/urbanaccess/utils.py index 4c8165b..2146aad 100644 --- a/urbanaccess/utils.py +++ b/urbanaccess/utils.py @@ -90,9 +90,9 @@ def _get_logger(level=None, name=None, filename=None): ------- logger : logger.logger """ - + # TODO: consider placing default lg.INFO 'log_level' as new config setting if level is None: - level = config.settings.log_level + level = lg.INFO if name is None: name = config.settings.log_name if filename is None: From ceba146604d2af18810063e5d753e1866696dfe3 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 13:59:28 -0800 Subject: [PATCH 49/72] add TODOs --- urbanaccess/gtfs/headways.py | 6 +++++- urbanaccess/network.py | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/urbanaccess/gtfs/headways.py b/urbanaccess/gtfs/headways.py index a63847f..04a06e5 100644 --- a/urbanaccess/gtfs/headways.py +++ b/urbanaccess/gtfs/headways.py @@ -25,7 +25,9 @@ def _calc_headways_by_route_stop(df): DataFrame of statistics of route stop headways in units of minutes """ # TODO: Optimize for speed - + # TODO: Update to incorporate optional direction_id into group, if + # direction_id doesnt exist do not use it since its an optional col + # TODO: Assess if we can simplify the results to return only unique records start_time = time.time() df = _add_unique_stop_route(df) @@ -80,6 +82,8 @@ def _headway_handler(interpolated_stop_times_df, trips_df, DataFrame of statistics of route stop headways in units of minutes with relevant route and stop information """ + # TODO: reconsider the use of inplace or copy incoming dfs + # to memory to avoid changing in memory tables start_time = time.time() # add unique trip and route ID diff --git a/urbanaccess/network.py b/urbanaccess/network.py index 752dfa7..58a858a 100644 --- a/urbanaccess/network.py +++ b/urbanaccess/network.py @@ -121,6 +121,7 @@ def integrate_network(urbanaccess_network, headways=False, 'string'.format(headway_statistic)) transit_edge_cols = urbanaccess_network.transit_edges.columns + # TODO: verify if these checks are still required in various use cases if 'node_id_from' not in transit_edge_cols or 'from' in \ transit_edge_cols: urbanaccess_network.transit_edges.rename( @@ -338,6 +339,8 @@ def _format_pandana_edges_nodes(edge_df, node_df): # for edges make it the from and to columns node_df['id_int'] = range(1, len(node_df) + 1) + # TODO: reconsider the use of inplace or copy incoming dfs + # to memory to avoid changing in memory tables edge_df.rename(columns={'id': 'edge_id'}, inplace=True) tmp = pd.merge(edge_df, node_df[['id', 'id_int']], left_on='from', right_on='id', sort=False, copy=False, how='left') From aedb403a451eeafb0ea39d756e7a5b9f49dbb5bd Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 14:01:49 -0800 Subject: [PATCH 50/72] increase coverage adding tests for config.py --- urbanaccess/tests/test_config.py | 134 +++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 urbanaccess/tests/test_config.py diff --git a/urbanaccess/tests/test_config.py b/urbanaccess/tests/test_config.py new file mode 100644 index 0000000..8d6de48 --- /dev/null +++ b/urbanaccess/tests/test_config.py @@ -0,0 +1,134 @@ +import pytest +import yaml +import os + +from urbanaccess import config + + +@pytest.fixture +def settings_valid(): + # build current dict of configs + return config.settings.to_dict() + + +@pytest.fixture +def expected_settings_dict(): + return { + 'data_folder': 'data', + 'images_folder': 'images', + 'image_filename': 'urbanaccess_plot', + 'logs_folder': 'logs', + 'log_file': True, + 'log_console': False, + 'log_name': 'urbanaccess', + 'log_filename': 'urbanaccess', + 'txt_encoding': 'utf-8', + 'gtfs_api': { + 'gtfsdataexch': + 'http://www.gtfs-data-exchange.com/api/agencies' + '?format=csv'}} + + +@pytest.fixture +def valid_setting_keys(): + return sorted(['data_folder', 'images_folder', 'image_filename', + 'logs_folder', 'log_file', 'log_console', 'log_name', + 'log_filename', 'txt_encoding', 'gtfs_api']) + + +@pytest.fixture +def settings_from_yaml(tmpdir, settings_valid): + # make 1 change to test yaml loading downstream + settings_valid['data_folder'] = 'test' + yaml_path = os.path.join(tmpdir.strpath, 'urbanaccess_config.yaml') + with open(yaml_path, 'w') as f: + yaml.dump(settings_valid, f, default_flow_style=False) + return tmpdir.strpath + + +@pytest.fixture +def settings_to_yaml(tmpdir, settings_valid): + # make 1 change to test yaml loading downstream + config.settings.data_folder = 'test' + return config.settings + + +def test_config_object(): + assert isinstance(config.settings, config.urbanaccess_config) + assert isinstance(config.settings.to_dict(), dict) + + +def test_config_to_dict(expected_settings_dict): + assert isinstance(config.settings.to_dict(), dict) + assert config.settings.to_dict() == expected_settings_dict + + +def test_format_check(settings_valid): + config._format_check(settings_valid) + + +def test_format_check_invalid( + valid_setting_keys, expected_settings_dict): + key_error_msg = ("Configuration keys: {} do not match " + "required keys: {}.") + with pytest.raises(ValueError) as excinfo: + # remove valid key txt_encoding + invalid_dict = expected_settings_dict.copy() + del invalid_dict['txt_encoding'] + config._format_check(invalid_dict) + expected_error = key_error_msg.format( + list(invalid_dict.keys()), valid_setting_keys) + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + # add invalid key + invalid_dict = expected_settings_dict.copy() + invalid_dict.update({'invalid_key': 'test'}) + config._format_check(invalid_dict) + expected_error = key_error_msg.format( + list(invalid_dict.keys()), valid_setting_keys) + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + invalid_dict = expected_settings_dict.copy() + invalid_dict['log_filename'] = 1 + config._format_check(invalid_dict) + expected_error = "Key: 'log_filename' value must be string." + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + invalid_dict = expected_settings_dict.copy() + invalid_dict['log_console'] = 'invalid' + config._format_check(invalid_dict) + expected_error = "Key: 'log_console' value must be boolean." + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + invalid_dict = expected_settings_dict.copy() + invalid_dict['gtfs_api'] = {'invalid': 'test_url'} + config._format_check(invalid_dict) + expected_error = ("gtfs_api key: 'invalid' does not match valid " + "key(s): ['gtfsdataexch'].") + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + invalid_dict = expected_settings_dict.copy() + invalid_dict['gtfs_api']['gtfsdataexch'] = 1 + config._format_check(invalid_dict) + expected_error = ("gtfs_api key: 'gtfsdataexch' value must be " + "string.") + assert expected_error in str(excinfo.value) + + +def test_from_yaml_config(settings_from_yaml): + result_settings = config.settings.from_yaml( + configdir=settings_from_yaml, yamlname='urbanaccess_config.yaml') + result_settings_dict = result_settings.to_dict() + assert result_settings_dict['data_folder'] == 'test' + + +def test_to_yaml_config(tmpdir, settings_to_yaml): + yamlname = 'urbanaccess_config.yaml' + settings_to_yaml.to_yaml( + configdir=tmpdir.strpath, yamlname=yamlname, overwrite=True) + + yaml_path = os.path.join(tmpdir.strpath, yamlname) + with open(yaml_path, 'r') as f: + yaml_config = yaml.safe_load(f) + assert sorted(yaml_config) == sorted(settings_to_yaml.to_dict()) + assert yaml_config['data_folder'] == 'test' From 909f0621c6514b07037d34abe95af94a920a315d Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 14:02:47 -0800 Subject: [PATCH 51/72] increase coverage adding tests for gtfs/headways.py --- urbanaccess/tests/test_gtfs_headways.py | 270 ++++++++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 urbanaccess/tests/test_gtfs_headways.py diff --git a/urbanaccess/tests/test_gtfs_headways.py b/urbanaccess/tests/test_gtfs_headways.py new file mode 100644 index 0000000..4727ea6 --- /dev/null +++ b/urbanaccess/tests/test_gtfs_headways.py @@ -0,0 +1,270 @@ +import pytest +import pandas as pd + +import urbanaccess.gtfs.headways as headways +from urbanaccess.gtfs.gtfsfeeds_dataframe import urbanaccess_gtfs_df \ + as gtfsfeeds_df + + +def _build_expected_hw_intermediate_stop_times_data( + trips_df, routes_df, stop_times_df): + trips_df['unique_trip_id'] = trips_df['trip_id'].str.cat( + trips_df['unique_agency_id'].astype('str'), sep='_') + trips_df['unique_route_id'] = trips_df['route_id'].str.cat( + trips_df['unique_agency_id'].astype('str'), sep='_') + + columns = ['unique_route_id', 'service_id', 'unique_trip_id', + 'unique_agency_id', 'direction_id'] + trips_df = trips_df[columns] + + routes_df['unique_route_id'] = routes_df['route_id'].str.cat( + routes_df['unique_agency_id'].astype('str'), sep='_') + columns = ['unique_route_id', 'route_long_name', 'route_type', + 'unique_agency_id'] + routes_df = routes_df[columns] + + tmp1 = pd.merge(trips_df, routes_df, how='left', left_on='unique_route_id', + right_on='unique_route_id', sort=False) + merge_df = pd.merge(stop_times_df, tmp1, how='left', + left_on='unique_trip_id', right_on='unique_trip_id', + sort=False) + cols_to_drop = ['unique_agency_id_y', 'unique_agency_id_x'] + merge_df.drop(cols_to_drop, axis=1, inplace=True) + return merge_df + + +def _check_headway_result( + result, hw_stop_times_int_df_1_agency, expected_route_stop_ids, + expected_route_stop_headway_result_sample_1, + expected_route_stop_headway_result_sample_2): + expected_cols = ['count', 'mean', 'std', 'min', '25%', '50%', '75%', 'max', + 'unique_stop_id', 'unique_route_id', 'node_id_route'] + assert isinstance(result, pd.core.frame.DataFrame) + assert result.empty is False + # each stop_time has a corresponding headway so len should be the same + # but this can be simplified in the future to return only unique data + assert len(result) == len(hw_stop_times_int_df_1_agency) + for col in expected_cols: + assert col in result.columns + assert sorted(result['node_id_route'].unique()) == expected_route_stop_ids + + # check subset of values + subset = result.loc[result['node_id_route'] == + '1_agency_a_city_a_10-101_agency_a_city_a'] + subset.reset_index(inplace=True, drop=True) + subset = subset.round(2) + assert len(subset) == 12 + assert subset[0:1].equals(expected_route_stop_headway_result_sample_1) + + subset = result.loc[result['node_id_route'] == + '7_agency_a_city_a_B1_agency_a_city_a'] + subset.reset_index(inplace=True, drop=True) + subset = subset.round(2) + assert len(subset) == 12 + assert subset[0:1].equals(expected_route_stop_headway_result_sample_2) + + +@pytest.fixture +def ua_gtfsfeeds_df( + hw_routes_df_1_agency, hw_trips_df_1_agency, + hw_stop_times_int_df_1_agency): + gtfsfeeds_df.routes = hw_routes_df_1_agency.copy() + gtfsfeeds_df.trips = hw_trips_df_1_agency.copy() + gtfsfeeds_df.stop_times_int = hw_stop_times_int_df_1_agency.copy() + return gtfsfeeds_df + + +@pytest.fixture +def interm_stop_times_df_1_agency( + hw_trips_df_1_agency, hw_routes_df_1_agency, + hw_stop_times_int_df_1_agency): + int_stop_times = _build_expected_hw_intermediate_stop_times_data( + trips_df=hw_trips_df_1_agency, + routes_df=hw_routes_df_1_agency, + stop_times_df=hw_stop_times_int_df_1_agency) + return int_stop_times + + +@pytest.fixture +def expected_route_stop_headway_result_sample_1(): + data = { + 'count': [11.0], + 'mean': [1.818182], + 'std': [6.030227], + 'min': [0.0], + '25%': [0.0], '50%': [0.0], '75%': [0.0], + 'max': [20.0], + 'unique_stop_id': ['1_agency_a_city_a'], + 'unique_route_id': ['10-101_agency_a_city_a'], + 'node_id_route': ['1_agency_a_city_a_10-101_agency_a_city_a'] + } + index = range(1) + expected_subset = pd.DataFrame(data, index) + expected_subset = expected_subset.round(2) + return expected_subset + + +@pytest.fixture +def expected_route_stop_headway_result_sample_2(): + data = { + 'count': [11.0], + 'mean': [2.73], + 'std': [3.75], + 'min': [0.0], + '25%': [0.0], '50%': [0.0], '75%': [5.0], + 'max': [10.0], + 'unique_stop_id': ['7_agency_a_city_a'], + 'unique_route_id': ['B1_agency_a_city_a'], + 'node_id_route': ['7_agency_a_city_a_B1_agency_a_city_a'] + } + index = range(1) + expected_subset = pd.DataFrame(data, index) + expected_subset = expected_subset.round(2) + return expected_subset + + +@pytest.fixture +def expected_route_stop_ids(): + expected_route_stop_ids = sorted([ + '1_agency_a_city_a_10-101_agency_a_city_a', + '2_agency_a_city_a_10-101_agency_a_city_a', + '3_agency_a_city_a_10-101_agency_a_city_a', + '4_agency_a_city_a_10-101_agency_a_city_a', + '5_agency_a_city_a_10-101_agency_a_city_a', + '6_agency_a_city_a_10-101_agency_a_city_a', + '7_agency_a_city_a_B1_agency_a_city_a', + '8_agency_a_city_a_B1_agency_a_city_a', + '9_agency_a_city_a_B1_agency_a_city_a']) + return expected_route_stop_ids + + +def test_calc_headways_by_route_stop( + interm_stop_times_df_1_agency, + expected_route_stop_headway_result_sample_1, + expected_route_stop_headway_result_sample_2, + expected_route_stop_ids): + result = headways._calc_headways_by_route_stop( + df=interm_stop_times_df_1_agency) + assert isinstance(result, pd.core.frame.DataFrame) + assert result.empty is False + assert len(result) == 9 + + expected_cols = ['count', 'mean', 'std', 'min', '25%', '50%', '75%', 'max'] + for col in expected_cols: + assert col in result.columns + # check route stop ids + assert sorted(result.index.str.replace(',', '_').to_list()) == \ + expected_route_stop_ids + + # check subset of values + subset = result.loc[ + result.index == '1_agency_a_city_a,10-101_agency_a_city_a'] + subset.reset_index(inplace=True, drop=True) + subset = subset.round(2) + assert len(subset) == 1 + assert subset.equals( + expected_route_stop_headway_result_sample_1[expected_cols]) + + subset = result.loc[result.index == '7_agency_a_city_a,B1_agency_a_city_a'] + subset.reset_index(inplace=True, drop=True) + subset = subset.round(2) + assert len(subset) == 1 + assert subset.equals( + expected_route_stop_headway_result_sample_2[expected_cols]) + + +def test_headway_handeler( + hw_routes_df_1_agency, hw_stop_times_int_df_1_agency, + hw_trips_df_1_agency, + expected_route_stop_headway_result_sample_1, + expected_route_stop_headway_result_sample_2, + expected_route_stop_ids): + result = headways._headway_handler( + interpolated_stop_times_df=hw_stop_times_int_df_1_agency, + trips_df=hw_trips_df_1_agency, + routes_df=hw_routes_df_1_agency, + headway_timerange=['06:00:00', '12:00:00']) + + _check_headway_result(result, + hw_stop_times_int_df_1_agency, + expected_route_stop_ids, + expected_route_stop_headway_result_sample_1, + expected_route_stop_headway_result_sample_2) + + +def test_headways(ua_gtfsfeeds_df, + hw_routes_df_1_agency, hw_stop_times_int_df_1_agency, + hw_trips_df_1_agency, + expected_route_stop_headway_result_sample_1, + expected_route_stop_headway_result_sample_2, + expected_route_stop_ids): + ua_gtfsfeeds_df = headways.headways( + gtfsfeeds_df=ua_gtfsfeeds_df, + headway_timerange=['06:00:00', '12:00:00']) + # add expected cols to tables + hw_trips_df_1_agency['unique_trip_id'] = ( + hw_trips_df_1_agency['trip_id'].str.cat( + hw_trips_df_1_agency['unique_agency_id'].astype('str'), sep='_')) + hw_trips_df_1_agency['unique_route_id'] = ( + hw_trips_df_1_agency['route_id'].str.cat( + hw_trips_df_1_agency['unique_agency_id'].astype('str'), sep='_')) + hw_routes_df_1_agency['unique_route_id'] = ( + hw_routes_df_1_agency['route_id'].str.cat( + hw_routes_df_1_agency['unique_agency_id'].astype('str'), sep='_')) + + assert ua_gtfsfeeds_df.trips.equals(hw_trips_df_1_agency) + assert ua_gtfsfeeds_df.stop_times_int.equals(hw_stop_times_int_df_1_agency) + assert ua_gtfsfeeds_df.routes.equals(hw_routes_df_1_agency) + + _check_headway_result(ua_gtfsfeeds_df.headways, + hw_stop_times_int_df_1_agency, + expected_route_stop_ids, + expected_route_stop_headway_result_sample_1, + expected_route_stop_headway_result_sample_2) + + +def test_headways_invalid(ua_gtfsfeeds_df): + with pytest.raises(ValueError) as excinfo: + ua_gtfsfeeds_df.trips = pd.DataFrame() + ua_gtfsfeeds_df = headways.headways( + gtfsfeeds_df=ua_gtfsfeeds_df, + headway_timerange=['06:00:00', '12:00:00']) + expected_error = ( + 'One of the following gtfsfeeds_dfs objects: stop_times_int, ' + 'trips, or routes were found to be empty.') + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + ua_gtfsfeeds_df.trips = pd.DataFrame() + ua_gtfsfeeds_df = headways.headways( + gtfsfeeds_df=None, + headway_timerange=['06:00:00', '12:00:00']) + expected_error = 'gtfsfeeds_df cannot be None.' + assert expected_error in str(excinfo.value) + + +def test_add_unique_stop_route(): + data = { + 'unique_route_id': ['Route_A', 'Route_B'], + 'unique_stop_id': ['Stop_1', 'Stop_2'], + 'unique_stop_route': ['Stop_1,Route_A', 'Stop_2,Route_B'] + } + index = range(2) + df = pd.DataFrame(data, index) + + result = headways._add_unique_stop_route( + df[['unique_route_id', 'unique_stop_id']]) + assert result.equals(df) + + +def test_add_node_id_route(): + data = { + 'unique_route_id': ['Route_A', 'Route_B'], + 'unique_stop_id': ['Stop_1', 'Stop_2'], + 'node_id_route': ['Stop_1_Route_A', 'Stop_2_Route_B'] + } + index = range(2) + df = pd.DataFrame(data, index) + + result = headways._add_node_id_route( + df[['unique_route_id', 'unique_stop_id']]) + assert result.equals(df) From 6dbf809af60c833e9b8c1ec0a5b606f69298df2a Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 14:03:53 -0800 Subject: [PATCH 52/72] increase coverage adding tests for osm/load.py --- urbanaccess/tests/test_osm_load.py | 58 ++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 urbanaccess/tests/test_osm_load.py diff --git a/urbanaccess/tests/test_osm_load.py b/urbanaccess/tests/test_osm_load.py new file mode 100644 index 0000000..6b67dc3 --- /dev/null +++ b/urbanaccess/tests/test_osm_load.py @@ -0,0 +1,58 @@ +import pytest + +from urbanaccess.osm import load + + +def _check_osm_node_edge_format(nodes, edges): + # we cannot full know the expected results from OSM to test by given + # data can change at any time so tests must be broad on the resulting data + req_col_list = ['x', 'y', 'id'] + for col in req_col_list: + assert col in nodes.columns + assert nodes[col].isnull().values.any() == False # noqa + + req_col_list = ['distance', 'from', 'to'] + for col in req_col_list: + assert col in edges.columns + assert edges[col].isnull().values.any() == False # noqa + + +@pytest.fixture +def bbox1(): + # bbox where we do not expect low connectivity nodes + return (-122.2762870789, 37.8211879615, -122.2701716423, 37.8241329692) + + +@pytest.fixture +def bbox2(): + # bbox where we expect low connectivity nodes + return (-122.4261214345, 37.8073837621, -122.4198987096, 37.8286680163) + + +def test_ua_network_from_bbox_remove_lcn_False(bbox1): + nodes, edges = load.ua_network_from_bbox( + bbox=bbox1, network_type='walk', + timeout=180, memory=None, + max_query_area_size=50 * 1000 * 50 * 1000, + remove_lcn=False) + _check_osm_node_edge_format(nodes, edges) + + +def test_ua_network_from_bbox_remove_lcn_True_no_lcn(bbox1): + # we dont expect any low connectivity nodes to be removed + nodes, edges = load.ua_network_from_bbox( + bbox=bbox1, network_type='walk', + timeout=180, memory=None, + max_query_area_size=50 * 1000 * 50 * 1000, + remove_lcn=True) + _check_osm_node_edge_format(nodes, edges) + + +def test_ua_network_from_bbox_remove_lcn_True_yes_lcn(bbox2): + # expect low connectivity nodes to be removed + nodes, edges = load.ua_network_from_bbox( + bbox=bbox2, network_type='walk', + timeout=180, memory=None, + max_query_area_size=50 * 1000 * 50 * 1000, + remove_lcn=True) + _check_osm_node_edge_format(nodes, edges) From b50981c5345e06d870dde7a06b948d15a1c101be Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 14:04:43 -0800 Subject: [PATCH 53/72] increase coverage adding tests for utils.py --- urbanaccess/tests/test_utils.py | 427 ++++++++++++++++++++++++++++++++ 1 file changed, 427 insertions(+) create mode 100644 urbanaccess/tests/test_utils.py diff --git a/urbanaccess/tests/test_utils.py b/urbanaccess/tests/test_utils.py new file mode 100644 index 0000000..67231ee --- /dev/null +++ b/urbanaccess/tests/test_utils.py @@ -0,0 +1,427 @@ +import pytest +import os +import yaml +import logging as lg +import pandas as pd +import datetime as dt + +from urbanaccess import utils +from urbanaccess import config + + +@pytest.fixture +def df_test(): + data = { + 'col_1': [1, 2, 3], + 'col_2': ['one', 'two', 'three'], + 'col_3': [1.5, 2.5, 3.0], + } + index = range(3) + df = pd.DataFrame(data, index) + + return df + + +@pytest.fixture +def hdf_file(tmpdir, df_test): + hdf_file_name = 'test_hdf.h5' + hdf_file_path = os.path.join(tmpdir.strpath, hdf_file_name) + df_test.to_hdf(hdf_file_path, key='test_key', mode='a', format='table') + return tmpdir.strpath, hdf_file_name + + +@pytest.fixture +def dictionary_to_yaml(): + return { + 'logs_folder': 'logs', + 'log_file': True, + 'log_console': False, + 'log_name': 'urbanaccess', + 'log_filename': 'urbanaccess'} + + +@pytest.fixture +def dictionary_to_yaml_invalid(dictionary_to_yaml): + return [dictionary_to_yaml] + + +@pytest.fixture +def read_yaml_file(tmpdir, dictionary_to_yaml): + yaml_path = os.path.join(tmpdir.strpath, 'test_yaml.yaml') + with open(yaml_path, 'w') as f: + yaml.dump(dictionary_to_yaml, f, default_flow_style=False) + return tmpdir.strpath + + +@pytest.fixture +def read_yaml_file_invalid(tmpdir, dictionary_to_yaml_invalid): + yaml_path = os.path.join(tmpdir.strpath, 'test_yaml.yaml') + with open(yaml_path, 'w') as f: + yaml.dump(dictionary_to_yaml_invalid, f, + default_flow_style=False) + return tmpdir.strpath + + +def test_dict_to_yaml(tmpdir, dictionary_to_yaml): + utils._dict_to_yaml(dictionary=dictionary_to_yaml, + yaml_dir=tmpdir.strpath, + yaml_name='test_yaml.yaml', + overwrite=True) + + yaml_path = os.path.join(tmpdir.strpath, 'test_yaml.yaml') + with open(yaml_path, 'r') as f: + yaml_config = yaml.safe_load(f) + assert yaml_config == dictionary_to_yaml + + +def test_dict_to_yaml_invalid(tmpdir, dictionary_to_yaml, + dictionary_to_yaml_invalid): + with pytest.raises(ValueError) as excinfo: + utils._dict_to_yaml(dictionary=dictionary_to_yaml_invalid, + yaml_dir=tmpdir.strpath, + yaml_name='test_yaml.yaml', + overwrite=True) + expected_error = 'Data to convert to YAML must be a dictionary.' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + utils._dict_to_yaml(dictionary=dictionary_to_yaml, + yaml_dir=1, + yaml_name='test_yaml.yaml', + overwrite=True) + expected_error = 'yaml directory must be a string' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + utils._dict_to_yaml(dictionary=dictionary_to_yaml, + yaml_dir=tmpdir.strpath, + yaml_name=1, + overwrite=True) + expected_error = 'yaml file name must be a string' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + utils._dict_to_yaml(dictionary=dictionary_to_yaml, + yaml_dir=tmpdir.strpath, + yaml_name='test_yaml.yaml', + overwrite=True) + utils._dict_to_yaml(dictionary=dictionary_to_yaml, + yaml_dir=tmpdir.strpath, + yaml_name='test_yaml.yaml', + overwrite=False) + expected_error = ('test_yaml.yaml already exists. ' + 'Rename or set overwrite to True') + assert expected_error in str(excinfo.value) + + +def test_dict_to_yaml_create_dir(tmpdir, dictionary_to_yaml): + path_to_create = os.path.join(tmpdir.strpath, 'temp') + utils._dict_to_yaml(dictionary=dictionary_to_yaml, + yaml_dir=path_to_create, + yaml_name='test_yaml.yaml', + overwrite=True) + + yaml_path = os.path.join(path_to_create, 'test_yaml.yaml') + with open(yaml_path, 'r') as f: + yaml_config = yaml.safe_load(f) + assert yaml_config == dictionary_to_yaml + + +def test_yaml_to_dict(read_yaml_file, dictionary_to_yaml): + result_dict = utils._yaml_to_dict( + yaml_dir=read_yaml_file, yaml_name='test_yaml.yaml') + assert sorted(result_dict) == sorted(dictionary_to_yaml) + + +def test_yaml_to_dict_invalid(read_yaml_file, + read_yaml_file_invalid): + with pytest.raises(ValueError) as excinfo: + result_dict = utils._yaml_to_dict( + yaml_dir=1, yaml_name='test_yaml.yaml') + expected_error = 'yaml directory must be a string.' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + result_dict = utils._yaml_to_dict( + yaml_dir='test_dir', yaml_name='test_yaml.yaml') + expected_error = 'test_dir does not exist or was not found.' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + result_dict = utils._yaml_to_dict( + yaml_dir=read_yaml_file, yaml_name=1) + expected_error = 'yaml file name must be a string.' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + result_dict = utils._yaml_to_dict( + yaml_dir=read_yaml_file_invalid, + yaml_name='test_yaml.yaml') + expected_error = 'Yaml data must be a dictionary.' + + +def test_log_write(tmpdir): + # test that message is written to console + logs_folder = os.path.join(tmpdir.strpath, 'logs_test') + log_filename = 'urbanaccess_test' + log_name = 'urbanaccess_test_log' + todays_date = dt.datetime.today().strftime('%Y_%m_%d') + print_msg = 'test message' + # set settings to a different test log folder + config.settings.logs_folder = logs_folder + utils.log(message=print_msg, level=lg.INFO, + name=log_name, filename=log_filename) + utils.log(message=print_msg, level=lg.WARNING, + name=log_name, filename=log_filename) + utils.log(message=print_msg, level=lg.DEBUG, + name=log_name, filename=log_filename) + utils.log(message=print_msg, level=lg.ERROR, + name=log_name, filename=log_filename) + log_file_path = os.path.join( + logs_folder, '{}_{}.log'.format(log_filename, todays_date)) + + # test that log was created and has expected path and file name + assert os.path.exists(log_file_path) is True + # test that level, name, and log message exist in log file + expected_log_msg_line_1 = "INFO {} {}".format(log_name, print_msg) + expected_log_msg_line_2 = "WARNING {} {}".format(log_name, print_msg) + expected_log_msg_line_3 = "ERROR {} {}".format(log_name, print_msg) + with open(log_file_path) as f: + lines = f.readlines() + assert expected_log_msg_line_1 in lines[0] + assert expected_log_msg_line_2 in lines[1] + assert expected_log_msg_line_3 in lines[2] + + +def test_log_print(capsys): + # test that message is written to console + print_msg = 'test message' + utils.log(message=print_msg) + # check that expected print prints + captured = capsys.readouterr() + assert print_msg in captured.out + + +def test_log_console_true(): + # test that message is written to console + print_msg = 'test message' + config.settings.log_console = True + utils.log(message=print_msg) + + +def test_log_console_false(): + # test that message is written to console + print_msg = 'test message' + config.settings.log_console = False + utils.log(message=print_msg) + + +def test_get_logger(): + # test with defaults (params set to None) + logger = utils._get_logger() + assert isinstance(logger, lg.Logger) + + +def test_create_hdf5_general(tmpdir): + hdf_file_name = 'test_hdf.h5' + hdf_file_path = tmpdir.strpath + path = os.path.join(hdf_file_path, hdf_file_name) + utils.create_hdf5(dir=hdf_file_path, filename=hdf_file_name, + overwrite_hdf5=False) + # test file was created + assert os.path.exists(path) is True + # test it can be read + with pd.HDFStore(path) as store: + assert isinstance(store, pd.io.pytables.HDFStore) + + +def test_create_hdf5_overwrite_false(tmpdir): + hdf_file_name = 'test_hdf.h5' + hdf_file_path = tmpdir.strpath + path = os.path.join(hdf_file_path, hdf_file_name) + utils.create_hdf5(dir=hdf_file_path, filename=hdf_file_name, + overwrite_hdf5=False) + # write the same h5 again + utils.create_hdf5(dir=hdf_file_path, filename=hdf_file_name, + overwrite_hdf5=False) + # test file was created + assert os.path.exists(path) is True + # test it can be read + with pd.HDFStore(path) as store: + assert isinstance(store, pd.io.pytables.HDFStore) + + +def test_create_hdf5_overwrite_true(tmpdir): + hdf_file_name = 'test_hdf.h5' + hdf_file_path = tmpdir.strpath + path = os.path.join(hdf_file_path, hdf_file_name) + utils.create_hdf5(dir=hdf_file_path, filename=hdf_file_name, + overwrite_hdf5=False) + # write the same h5 again + utils.create_hdf5(dir=hdf_file_path, filename=hdf_file_name, + overwrite_hdf5=True) + # test file was created + assert os.path.exists(path) is True + # test it can be read + with pd.HDFStore(path) as store: + assert isinstance(store, pd.io.pytables.HDFStore) + + +def test_create_hdf5_w_defaults(): + path = os.path.join(config.settings.data_folder, 'urbanaccess.h5') + utils.create_hdf5(dir=None, filename=None, overwrite_hdf5=False) + # test file was created + assert os.path.exists(path) is True + # test it can be read + with pd.HDFStore(path) as store: + assert isinstance(store, pd.io.pytables.HDFStore) + # remove test data + os.remove(path) + + +def test_create_hdf5_invalid(tmpdir): + hdf_file_name = 'test_hdf.h5' + hdf_file_path = tmpdir.strpath + with pytest.raises(ValueError) as excinfo: + utils.create_hdf5(dir=1, filename=hdf_file_name, + overwrite_hdf5=False) + expected_error = 'Directory must be a string.' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + utils.create_hdf5(dir=hdf_file_path, filename=1, + overwrite_hdf5=False) + expected_error = 'Filename must be a string.' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + utils.create_hdf5(dir=hdf_file_path, filename='test_file', + overwrite_hdf5=False) + expected_error = 'HDF5 filename extension must be "h5".' + assert expected_error in str(excinfo.value) + + +def test_df_to_hdf5_general(tmpdir, df_test): + hdf_file_name = 'test_hdf.h5' + hdf_file_path = tmpdir.strpath + path = os.path.join(hdf_file_path, hdf_file_name) + utils.df_to_hdf5(data=df_test, key='test_key', overwrite_key=False, + dir=hdf_file_path, filename=hdf_file_name, + overwrite_hdf5=False) + # test file was created + assert os.path.exists(path) is True + with pd.HDFStore(path) as store: + result = store['test_key'] + # test df when read is as expected + assert result.equals(df_test) + + +def test_df_to_hdf5_overwrite_false(tmpdir, df_test): + hdf_file_name = 'test_hdf.h5' + hdf_file_path = tmpdir.strpath + path = os.path.join(hdf_file_path, hdf_file_name) + utils.df_to_hdf5(data=df_test, key='test_key', overwrite_key=False, + dir=hdf_file_path, filename=hdf_file_name, + overwrite_hdf5=False) + # write the same h5 again + utils.df_to_hdf5(data=df_test, key='test_key', overwrite_key=False, + dir=hdf_file_path, filename=hdf_file_name, + overwrite_hdf5=False) + assert os.path.exists(path) is True + with pd.HDFStore(path) as store: + result = store['test_key'] + # test df when read is as expected + assert result.equals(df_test) + + +def test_df_to_hdf5_overwrite_true(tmpdir, df_test): + hdf_file_name = 'test_hdf.h5' + hdf_file_path = tmpdir.strpath + path = os.path.join(hdf_file_path, hdf_file_name) + utils.df_to_hdf5(data=df_test, key='test_key', overwrite_key=False, + dir=hdf_file_path, filename=hdf_file_name, + overwrite_hdf5=False) + # write the same h5 again + utils.df_to_hdf5(data=df_test, key='test_key', overwrite_key=True, + dir=hdf_file_path, filename=hdf_file_name, + overwrite_hdf5=False) + assert os.path.exists(path) is True + with pd.HDFStore(path) as store: + result = store['test_key'] + # test df when read is as expected + assert result.equals(df_test) + + +def test_hdf5_to_df(df_test, hdf_file): + hdf_file_path, hdf_file_name = hdf_file + result = utils.hdf5_to_df(dir=hdf_file_path, filename=hdf_file_name, + key='test_key') + assert result.equals(df_test) + + +def test_hdf5_to_df_invalid(tmpdir, hdf_file): + hdf_file_path, hdf_file_name = hdf_file + with pytest.raises(ValueError) as excinfo: + result = utils.hdf5_to_df(dir=1, filename=hdf_file_name, + key='test_key') + expected_error = 'Directory must be a string.' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + result = utils.hdf5_to_df(dir=hdf_file_path, filename=1, + key='test_key') + expected_error = 'Filename must be a string.' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + result = utils.hdf5_to_df(dir=hdf_file_path, filename='test_file', + key='test_key') + expected_error = 'HDF5 filename extension must be "h5".' + assert expected_error in str(excinfo.value) + hdf_file_path, hdf_file_name + with pytest.raises(ValueError) as excinfo: + result = utils.hdf5_to_df(dir=hdf_file_path, + filename='invalid_file.h5', + key='test_key') + expected_error = 'Unable to find directory or file: {}.' + path = os.path.join(hdf_file_path, 'invalid_file.h5') + assert expected_error.format(path) in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + result = utils.hdf5_to_df(dir=hdf_file_path, + filename=hdf_file_name, + key='invalid_key') + expected_error = "Unable to find key: invalid_key. " \ + "Keys found: ['/test_key']." + assert expected_error in str(excinfo.value) + + +def test_add_unique_trip_id(): + data = { + 'trip_id': ['trip 1', 'trip 2'], + 'unique_agency_id': ['agency_a', 'agency_b'], + 'unique_trip_id': ['trip 1_agency_a', 'trip 2_agency_b'] + } + index = range(2) + df = pd.DataFrame(data, index) + + result = utils._add_unique_trip_id( + df[['trip_id', 'unique_agency_id']]) + assert result.equals(df) + + +def test_add_unique_route_id(): + data = { + 'route_id': ['route 1', 'route 2'], + 'unique_agency_id': ['agency_a', 'agency_b'], + 'unique_route_id': ['route 1_agency_a', 'route 2_agency_b'] + } + index = range(2) + df = pd.DataFrame(data, index) + + result = utils._add_unique_route_id( + df[['route_id', 'unique_agency_id']]) + assert result.equals(df) + + +def test_add_unique_stop_id(): + data = { + 'stop_id': ['stop 1', 'stop 2'], + 'unique_agency_id': ['agency_a', 'agency_b'], + 'unique_stop_id': ['stop 1_agency_a', 'stop 2_agency_b'] + } + index = range(2) + df = pd.DataFrame(data, index) + + result = utils._add_unique_stop_id( + df[['stop_id', 'unique_agency_id']]) + assert result.equals(df) From 2a9bbca4f67ff7c4b376400970a7ba2d6039008a Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 14:05:21 -0800 Subject: [PATCH 54/72] clean up temp test files --- urbanaccess/tests/test_plot.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/urbanaccess/tests/test_plot.py b/urbanaccess/tests/test_plot.py index 55f4cea..7dfab3a 100644 --- a/urbanaccess/tests/test_plot.py +++ b/urbanaccess/tests/test_plot.py @@ -413,6 +413,8 @@ def test_plot_save_w_path_and_filename(small_net, tmpdir, show_plot): if show_plot: imgplot = plt.imshow(img) plt.show() + # clean up test data + os.remove(file_name) def test_plot_save_w_default_name(small_net, show_plot): @@ -430,8 +432,8 @@ def test_plot_save_w_default_name(small_net, show_plot): show=False, close=False, save=True, filepath=None, dpi=300, ax=None) # check that file was created - file_name = os.path.join(settings.images_folder, - '{}.png'.format(settings.image_filename)) + file_name = os.path.join( + settings.images_folder, '{}.png'.format(settings.image_filename)) print('wrote test data to file: {}'.format(file_name)) assert os.path.isfile(file_name) # read and show image @@ -440,6 +442,8 @@ def test_plot_save_w_default_name(small_net, show_plot): if show_plot: imgplot = plt.imshow(img) plt.show() + # clean up test data + os.remove(file_name) def test_plot_save_w_filename_only(small_net, show_plot): @@ -466,6 +470,8 @@ def test_plot_save_w_filename_only(small_net, show_plot): if show_plot: imgplot = plt.imshow(img) plt.show() + # clean up test data + os.remove(file_name) def test_plot_invalid_params(small_net, transit_nodes_invalid_xy, show_plot): @@ -531,6 +537,10 @@ def test_plot_print_warn(small_net, capsys): # check that expected print prints captured = capsys.readouterr() assert 'Warning: Existing file' in captured.out + default_file = os.path.join( + settings.images_folder, '{}.png'.format(settings.image_filename)) + # clean up test data + os.remove(default_file) def test_col_colors_case_1(small_net, capsys): From fd7331cb9030d0a9486d27ad63063050c4e33330 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 14:06:10 -0800 Subject: [PATCH 55/72] updated tests for osm/network.py and increased coverage --- urbanaccess/tests/test_osm_network.py | 99 ++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 18 deletions(-) diff --git a/urbanaccess/tests/test_osm_network.py b/urbanaccess/tests/test_osm_network.py index cbcf516..eff0b24 100644 --- a/urbanaccess/tests/test_osm_network.py +++ b/urbanaccess/tests/test_osm_network.py @@ -1,23 +1,86 @@ import pytest +import pandas as pd -from urbanaccess.osm.load import ua_network_from_bbox +from urbanaccess.osm import network @pytest.fixture -def bbox1(): - return (-122.2762870789, 37.8211879615, -122.2701716423, 37.8241329692) - - -def test_column_names(bbox1): - nodes, edges = ua_network_from_bbox( - bbox=bbox1, network_type='walk', - timeout=180, memory=None, - max_query_area_size=50 * 1000 * 50 * 1000, - remove_lcn=False) - col_list = ['x', 'y', 'id'] - for col in col_list: - assert col in nodes.columns - - col_list = ['distance', 'from', 'to'] - for col in col_list: - assert col in edges.columns +def walk_nodes(): + data = { + 'id': [1, 2, 3, 4, 5, 6, 7, 8, 9], + 'x': [ + -122.265474, -122.272543, -122.273680, -122.262834, -122.269889, + -122.271170, -122.268333, -122.266974, -122.264433], + 'y': [ + 37.796897, 37.799683, 37.800206, 37.800964, 37.803884, + 37.804270, 37.809158, 37.808645, 37.807921], + # name is not expected in OSM nodes but is used here as placeholder + # for custom columns and as a reference for tests + 'name': [ + '1 8th & Oak', '2 8th & Franklin', '3 8th & Broadway', + '4 14th & Oak', '5 14th & Franklin', '6 14th & Broadway', + '7 Berkley & Broadway', '8 Berkley & Franklin', + '9 Berkley & Harrison']} + index = range(9) + df = pd.DataFrame(data, index) + df.set_index('id', inplace=True, drop=False) + + return df + + +@pytest.fixture +def walk_edges(): + data = { + 'from': [1, 2, 3, 6, 7, 8, 9, 4, 4, 5, 2, 5], + 'to': [2, 3, 6, 7, 8, 9, 4, 1, 5, 6, 5, 8], + 'name': ['8th', '8th', 'Broadway', 'Broadway', 'Berkley', 'Berkley', + 'Lakeside', 'Oak', '14th', '14th', 'Franklin', 'Franklin'], + 'highway': ['residential', 'residential', 'primary', 'primary', + 'primary', 'primary', 'residential', 'residential', + 'primary', 'primary', 'residential', 'residential'], + 'distance': [0.3, 0.3, 0.5, 0.5, 0.6, 0.6, 1, 0.8, 0.8, 0.8, 0.4, 0.4], + 'oneway': ['yes', 'yes', 'no', 'no', 'no', 'no', 'yes', 'yes', 'no', + 'no', 'yes', 'yes'] + } + index = range(12) + df = pd.DataFrame(data, index) + + # since this is a walk network we generate a bi-directed graph + twoway_df = df.copy() + twoway_df.rename(columns={'from': 'to', 'to': 'from'}, inplace=True) + + edge_df = pd.concat([df, twoway_df], axis=0, ignore_index=True) + edge_df.set_index(['from', 'to'], inplace=True, drop=False) + + return edge_df + + +@pytest.fixture +def expected_walk_edges_nodes(walk_edges, walk_nodes): + # build expected data + network_type = 'walk' + walk_edges['net_type'] = network_type + walk_nodes['net_type'] = network_type + walk_edges['weight'] = (walk_edges['distance'] / 1609.34) / 3 * 60 + return walk_nodes, walk_edges + + +def test_create_osm_net(walk_nodes, walk_edges, expected_walk_edges_nodes): + expected_walk_nodes, expected_walk_edges = expected_walk_edges_nodes + ua_network = network.create_osm_net( + osm_edges=walk_edges, osm_nodes=walk_nodes, + travel_speed_mph=3, network_type='walk') + assert ua_network.osm_edges.empty is False + assert ua_network.osm_edges.equals(expected_walk_edges) + assert ua_network.osm_nodes.empty is False + assert ua_network.osm_nodes.equals(expected_walk_nodes) + + +def test_create_osm_net_invalid(walk_nodes, walk_edges): + with pytest.raises(ValueError) as excinfo: + ua_network = network.create_osm_net( + osm_edges=walk_edges, osm_nodes=walk_nodes, + travel_speed_mph=3, network_type=None) + expected_error = ( + "'None' network_type passed is either not a string or is None.") + assert expected_error in str(excinfo.value) From b2075dc7aa1444da32ac766e1015f5ad31c778ce Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 14:07:50 -0800 Subject: [PATCH 56/72] updated tests for network.py to coverage headway usage and increased coverage --- urbanaccess/tests/conftest.py | 218 ++++++++++++++++ urbanaccess/tests/test_network.py | 420 ++++++++++++++++++++++++++++-- 2 files changed, 623 insertions(+), 15 deletions(-) diff --git a/urbanaccess/tests/conftest.py b/urbanaccess/tests/conftest.py index 3d48944..97f7865 100644 --- a/urbanaccess/tests/conftest.py +++ b/urbanaccess/tests/conftest.py @@ -3,6 +3,9 @@ import pandas as pd import numpy as np +from urbanaccess.tests.test_gtfs_headways import \ + _build_expected_hw_intermediate_stop_times_data + @pytest.fixture def agency_feed_1(): @@ -664,6 +667,221 @@ def stop_times_feed_4(): return df +@pytest.fixture +def hw_stops_df_1_agency(): + # intended for use in headway tests + data = { + 'stop_id': ['1', '2', '3', '4', '5', '6', + '7', '8', '9'], + 'stop_name': ['ave a', 'ave b', 'ave c', 'ave d', 'ave e', 'ave f', + '1st st', '2nd st', '3rd st'], + 'stop_lat': [37.797484, 37.774963, 37.803664, 37.80787, 37.828415, + 37.844601, 37.664174, 37.591208, 37.905628], + 'stop_lon': [-122.265609, -122.224274, -122.271604, -122.269029, + -122.267227, -122.251793, -122.444116, -122.017867, + -122.067423], + 'location_type': [1, 1, 1, 1, 1, 1, + 2, 2, 2], + 'wheelchair_boarding': [1, 0, 0, 0, 0, 0, + 1, 1, 1], + 'unique_agency_id': ['agency_a_city_a'] * 9 + } + index = range(9) + df = pd.DataFrame(data, index) + return df + + +@pytest.fixture +def hw_routes_df_1_agency(): + # intended for use in headway tests + data = { + 'agency_id': ['agency a city a'] * 2, + 'route_id': ['10-101', 'B1'], + 'route_long_name': ['Route 10-101', 'B1 Express'], + 'route_type': [3, 2], + 'unique_agency_id': ['agency_a_city_a'] * 2, + 'unique_feed_id': ['feed_1'] * 2} + index = range(2) + df = pd.DataFrame(data, index) + return df + + +@pytest.fixture +def hw_stop_times_int_df_1_agency(): + # intended for use in headway tests + + # only generate the data that is expected, leaving out: arrival_time, + # departure_time + + # outbound direction + data = { + 'trip_id': ['a1', 'a1', + 'a1', 'a1', + 'a1', 'a1', + 'B1', 'B1', + 'B1'], + 'stop_id': [1, 2, 3, 4, 5, 6, + 7, 8, 9], + 'stop_sequence': [1, 2, 3, 4, 5, 6, + 1, 2, 3], + 'unique_agency_id': ['agency_a_city_a'] * 9, + 'unique_feed_id': ['feed_1'] * 9, + 'route_type': [3, 3, 3, 3, 3, 3, + 2, 2, 2], + 'departure_time_sec': [ + 22500.0, 22620.0, 22860.0, 23100.0, 23580.0, 23940.0, + 22500.0, 22680.0, 22980.0], + # nan, 2, 4, 4, 8, 6 # min + # nan, 3, 5 # min + 'timediff': [ + np.nan, 120, 240, 240, 480, 360, + np.nan, 180, 300] + } + index = range(9) + direction_0 = pd.DataFrame(data, index) + + # inbound direction + data = { + 'trip_id': ['a2', 'a2', + 'a2', 'a2', + 'a2', 'a2', + 'B2', 'B2', + 'B2'], + 'stop_id': [6, 5, 4, 3, 2, 1, + 9, 8, 7], + 'stop_sequence': [1, 2, 3, 4, 5, 6, + 1, 2, 3], + 'unique_agency_id': ['agency_a_city_a'] * 9, + 'unique_feed_id': ['feed_1'] * 9, + 'route_type': [3, 3, 3, 3, 3, 3, + 2, 2, 2], + 'departure_time_sec': [ + 23940.0, 23580.0, 23100.0, 22860.0, 22620.0, 22500.0, + 22980.0, 22680.0, 22500.0], + # nan, 2, 4, 4, 8, 6 # min + # nan, 3, 5 # min + 'timediff': [ + np.nan, 360, 480, 240, 240, 120, + np.nan, 300, 180] + } + index = range(9) + direction_1 = pd.DataFrame(data, index) + + df = pd.concat([direction_1, direction_0], ignore_index=True) + + # make multiple trips that stop at the same stops but at offset times + # so we can replicate headways for each route + merged_dfs = df.copy() + trip_count = 5 + # set some trips to have even headways + even_headway = 1200 + # set some trips to have uneven headways but predicable + uneven_headway = {1: 900, 2: 1200, 3: 500, 4: 600, 5: 1800} + for trip_id in range(trip_count): + df_2 = df.copy() + trip_num = trip_id + 1 + # 20 min headways for (1200 sec) with even spacing + col = 'departure_time_sec' + trip = 'a1' + df_2.loc[df['trip_id'] == trip, col] = \ + df_2.loc[df['trip_id'] == trip, col] + even_headway + trip = 'a2' + df_2.loc[df['trip_id'] == trip, col] = \ + df_2.loc[df['trip_id'] == trip, col] + even_headway + # 15-30 min headways for (900-1200 sec) with uneven spacing and + trip = 'B1' + df_2.loc[df_2['trip_id'] == trip, col] = \ + df_2.loc[df['trip_id'] == trip, col] + uneven_headway[trip_num] + trip = 'B2' + df_2.loc[df_2['trip_id'] == trip, col] = \ + df_2.loc[df['trip_id'] == trip, col] + uneven_headway[trip_num] + df_2['trip_id'] = df_2['trip_id'] + '_{}'.format(trip_num) + + merged_dfs = merged_dfs.append(df_2, ignore_index=True) + + # add the other expected cols + merged_dfs['departure_time_sec_interpolate'] = merged_dfs[ + 'departure_time_sec'].astype('int') + merged_dfs['unique_stop_id'] = ( + merged_dfs['stop_id'].astype('str').str.cat( + merged_dfs['unique_agency_id'].astype('str'), sep='_')) + merged_dfs['unique_trip_id'] = merged_dfs['trip_id'].str.cat( + merged_dfs['unique_agency_id'].astype('str'), sep='_') + + return merged_dfs + + +@pytest.fixture +def hw_trips_df_1_agency(): + # intended for use in headway tests + data = { + 'route_id': ['10-101', '10-101', + 'B1', 'B1'], + 'trip_id': ['a1', 'a2', + 'B1', 'B2'], + 'service_id': ['weekday-1'] * 4, + 'direction_id': [1, 0, + 1, 0], + 'unique_agency_id': ['agency_a_city_a'] * 4, + 'unique_feed_id': ['feed_1'] * 4} + index = range(4) + df = pd.DataFrame(data, index) + + # expand trips to match what is in the test stop_times table + merged_dfs = df.copy() + trip_count = 5 + for trip_id in range(trip_count): + df_2 = df.copy() + trip_num = trip_id + 1 + df_2['trip_id'] = df_2['trip_id'] + '_{}'.format(trip_num) + merged_dfs = merged_dfs.append(df_2, ignore_index=True) + + return merged_dfs + + +@pytest.fixture() +def hw_headways_df_1_agency( + hw_trips_df_1_agency, hw_routes_df_1_agency, + hw_stop_times_int_df_1_agency): + # intended for use in headway tests + + # build expected headway table + int_stop_times = _build_expected_hw_intermediate_stop_times_data( + trips_df=hw_trips_df_1_agency, + routes_df=hw_routes_df_1_agency, + stop_times_df=hw_stop_times_int_df_1_agency) + + int_stop_times['unique_stop_route'] = ( + int_stop_times['unique_stop_id'].str.cat( + int_stop_times['unique_route_id'].astype('str'), sep=',')) + + stop_route_groups = int_stop_times.groupby('unique_stop_route') + + results = {} + col = 'departure_time_sec_interpolate' + for unique_stop_route, stop_route_group in stop_route_groups: + stop_route_group.sort_values([col], ascending=True, inplace=True) + next_veh_time = (stop_route_group[col].iloc[1:].values) + prev_veh_time = (stop_route_group[col].iloc[:-1].values) + stop_route_group_headways = (next_veh_time - prev_veh_time) / 60 + results[unique_stop_route] = (pd.Series(stop_route_group_headways) + .describe()) + + headway_by_routestop_df = pd.DataFrame(results).T + + headway_by_routestop_df = pd.merge( + headway_by_routestop_df, + int_stop_times[ + ['unique_stop_route', 'unique_stop_id', 'unique_route_id']], + how='left', left_index=True, right_on='unique_stop_route', sort=False) + headway_by_routestop_df.drop('unique_stop_route', axis=1, inplace=True) + headway_by_routestop_df['node_id_route'] = ( + headway_by_routestop_df['unique_stop_id'].str.cat( + headway_by_routestop_df['unique_route_id'].astype('str'), sep='_')) + + return headway_by_routestop_df + + @pytest.fixture() def agency_a_feed_on_disk_wo_calendar_dates( tmpdir, diff --git a/urbanaccess/tests/test_network.py b/urbanaccess/tests/test_network.py index 5b7baf2..228aa24 100644 --- a/urbanaccess/tests/test_network.py +++ b/urbanaccess/tests/test_network.py @@ -1,13 +1,19 @@ import pytest +import os +import numpy as np import pandas as pd from geopy import distance from sklearn.neighbors import KDTree from urbanaccess import network from urbanaccess.network import urbanaccess_network as ua_net +from urbanaccess.gtfs.gtfsfeeds_dataframe import urbanaccess_gtfs_df \ + as gtfsfeeds_df def _build_expected_nearest_osm_node_data(osm_nodes=None, transit_nodes=None): + # logic taken directly from urbanaccess\network_utils.py for + # data replication df1_matrix = osm_nodes[['x', 'y']].values df2_matrix = transit_nodes[['x', 'y']].values kdt = KDTree(df1_matrix) @@ -17,6 +23,8 @@ def _build_expected_nearest_osm_node_data(osm_nodes=None, transit_nodes=None): def _build_expected_connector_edges_data(osm_nodes=None, transit_nodes=None): + # logic taken directly from urbanaccess\network_utils.py for + # data replication travel_speed_mph = 3 net_connector_edges = [] for transit_node_id, row in transit_nodes.iterrows(): @@ -42,22 +50,42 @@ def _build_expected_connector_edges_data(osm_nodes=None, transit_nodes=None): def _build_expected_intermediate_transit_node_edge_data( transit_edges=None, transit_nodes=None): - transit_edges.rename( + # logic taken directly from urbanaccess\network.py for data replication + + # use copy to ensure we dont operate on in memory df + tr_edges = transit_edges.copy() + tr_nodes = transit_nodes.copy() + tr_edges.rename( columns={'node_id_from': 'from', 'node_id_to': 'to'}, inplace=True) - transit_nodes.reset_index(inplace=True, drop=False) - transit_nodes.rename(columns={'node_id': 'id'}, inplace=True) - return transit_edges, transit_nodes + tr_nodes.reset_index(inplace=True, drop=False) + tr_nodes.rename(columns={'node_id': 'id'}, inplace=True) + return tr_edges, tr_nodes + + +def _build_expected_intermediate_transit_node_edge_data_w_headways( + transit_edges=None, transit_nodes=None): + # logic taken directly from urbanaccess\network.py for data replication + + # use copy to ensure we dont operate on in memory df + tr_edges = transit_edges.copy() + tr_nodes = transit_nodes.copy() + tr_edges.rename( + columns={'node_id_route_from': 'from', 'node_id_route_to': 'to'}, + inplace=True) + tr_edges.drop(['node_id_from', 'node_id_to'], inplace=True, axis=1) + tr_nodes.reset_index(inplace=True, drop=False) + tr_nodes.rename(columns={'node_id_route': 'id'}, inplace=True) + return tr_edges, tr_nodes def _build_expected_node_edge_net_data( transit_edges=None, transit_nodes=None, walk_edges=None, walk_nodes=None, expected_connector_edges=None): - edges_df = pd.concat( - [transit_edges, walk_edges, expected_connector_edges], - axis=0, ignore_index=True) - nodes_df = pd.concat( - [transit_nodes, walk_nodes], - axis=0, ignore_index=True) + # logic taken directly from urbanaccess\network.py for data replication + edges_df = pd.concat([transit_edges, walk_edges, expected_connector_edges], + axis=0, ignore_index=True) + nodes_df = pd.concat([transit_nodes, walk_nodes], + axis=0, ignore_index=True) nodes_df['id_int'] = range(1, len(nodes_df) + 1) @@ -81,6 +109,55 @@ def _build_expected_node_edge_net_data( return edges_df_wnumericid, nodes_df +def _build_intermediate_transit_edges_w_routes(transit_edges=None): + # logic taken directly from urbanaccess\network.py for data replication + + # use copy to ensure we dont operate on in memory df + tr_edges = transit_edges.copy() + tr_edges.rename(columns={'from': 'node_id_from'}, inplace=True) + tr_edges.rename(columns={'to': 'node_id_to'}, inplace=True) + tr_edges['node_id_route_from'] = tr_edges['node_id_from'].str.cat( + tr_edges['unique_route_id'].astype('str'), sep='_') + tr_edges['node_id_route_to'] = tr_edges['node_id_to'].str.cat( + tr_edges['unique_route_id'].astype('str'), sep='_') + return tr_edges + + +def _build_expected_transit_nodes_w_routes_data( + stops_df=None, edges_w_routes=None): + # logic taken directly from urbanaccess\network.py for data replication + + # use copy to ensure we dont operate on in memory df + stops_df_copy = stops_df.copy() + stops_df_copy['unique_stop_id'] = stops_df_copy['stop_id'].str.cat( + stops_df_copy['unique_agency_id'].astype('str'), sep='_') + tmp1 = pd.merge(edges_w_routes[['node_id_from', 'node_id_route_from']], + stops_df_copy, + how='left', left_on='node_id_from', + right_on='unique_stop_id', sort=False, copy=False) + tmp1.rename(columns={'node_id_route_from': 'node_id_route', + 'stop_lon': 'x', + 'stop_lat': 'y'}, + inplace=True) + tmp2 = pd.merge(edges_w_routes[['node_id_to', 'node_id_route_to']], + stops_df_copy, + how='left', + left_on='node_id_to', + right_on='unique_stop_id', sort=False, copy=False) + tmp2.rename(columns={'node_id_route_to': 'node_id_route', + 'stop_lon': 'x', + 'stop_lat': 'y'}, + inplace=True) + transit_nodes_wroutes = pd.concat([tmp1, tmp2], axis=0) + transit_nodes_wroutes.drop_duplicates( + subset='node_id_route', keep='first', inplace=True) + transit_nodes_wroutes.drop( + columns=['node_id_to', 'node_id_to'], inplace=True) + transit_nodes_wroutes = transit_nodes_wroutes.set_index('node_id_route') + transit_nodes_wroutes['net_type'] = 'transit' + return transit_nodes_wroutes + + @pytest.fixture def transit_edges_1_agency(): # edges for outbound direction @@ -140,9 +217,8 @@ def transit_edges_1_agency(): df = pd.concat([direction_1, direction_0], ignore_index=True) # add ID - df['id'] = ( - df['unique_trip_id'].str.cat( - df['sequence'].astype('str'), sep='_')) + df['id'] = df['unique_trip_id'].str.cat( + df['sequence'].astype('str'), sep='_') return df @@ -179,6 +255,21 @@ def transit_nodes_1_agency(): return df +@pytest.fixture +def ua_gtfsfeeds_df( + hw_routes_df_1_agency, hw_trips_df_1_agency, + hw_stop_times_int_df_1_agency, + hw_stops_df_1_agency, hw_headways_df_1_agency): + # for testing only adding tables that are expected in tested functions, + # leaving out others not used in tests + gtfsfeeds_df.routes = hw_routes_df_1_agency.copy() + gtfsfeeds_df.trips = hw_trips_df_1_agency.copy() + gtfsfeeds_df.stop_times_int = hw_stop_times_int_df_1_agency.copy() + gtfsfeeds_df.stops = hw_stops_df_1_agency.copy() + gtfsfeeds_df.headways = hw_headways_df_1_agency.copy() + return gtfsfeeds_df + + @pytest.fixture def walk_nodes(): data = { @@ -252,8 +343,8 @@ def intermediate_transit_edges_nodes( _build_expected_intermediate_transit_node_edge_data( transit_edges_1_agency, transit_nodes_1_agency) # nodes are expected to have 'nearest_osm_node' column so generate the col - transit_nodes = _build_expected_nearest_osm_node_data \ - (walk_nodes, transit_nodes) + transit_nodes = _build_expected_nearest_osm_node_data( + walk_nodes, transit_nodes) return transit_edges, transit_nodes @@ -271,6 +362,94 @@ def expected_net_edges_nodes( return nodes_df, edges_df +@pytest.fixture +def hw_intermediate_transit_edges_w_routes(transit_edges_1_agency): + transit_edges_w_routes = _build_intermediate_transit_edges_w_routes( + transit_edges_1_agency) + return transit_edges_w_routes + + +@pytest.fixture +def hw_intermediate_transit_nodes_w_routes( + hw_intermediate_transit_edges_w_routes, hw_stops_df_1_agency): + transit_nodes_wroutes = _build_expected_transit_nodes_w_routes_data( + stops_df=hw_stops_df_1_agency, + edges_w_routes=hw_intermediate_transit_edges_w_routes) + return transit_nodes_wroutes + + +@pytest.fixture +def expected_connector_edges_w_headways( + hw_intermediate_transit_nodes_w_routes, walk_nodes): + # build expected connector edge df + transit_nodes = _build_expected_nearest_osm_node_data( + walk_nodes, hw_intermediate_transit_nodes_w_routes) + net_connector_edges = _build_expected_connector_edges_data( + walk_nodes, transit_nodes) + return net_connector_edges + + +@pytest.fixture +def expected_connector_edges_w_headways_w_impedance( + hw_headways_df_1_agency, expected_connector_edges_w_headways): + osm_to_transit_wheadway = pd.merge( + expected_connector_edges_w_headways, + hw_headways_df_1_agency[['mean', 'node_id_route']], + how='left', left_on=['to'], right_on=['node_id_route'], + sort=False, copy=False) + osm_to_transit_wheadway['weight_tmp'] = osm_to_transit_wheadway[ + 'weight'] + ( + osm_to_transit_wheadway[ + 'mean'] / 2.0) + osm_to_transit_wheadway['weight_tmp'].fillna( + osm_to_transit_wheadway['weight'], inplace=True) + osm_to_transit_wheadway.drop('weight', axis=1, inplace=True) + osm_to_transit_wheadway.rename( + columns={'weight_tmp': 'weight'}, inplace=True) + return osm_to_transit_wheadway + + +@pytest.fixture +def intermediate_transit_edges_nodes_w_headways( + hw_intermediate_transit_edges_w_routes, + hw_intermediate_transit_nodes_w_routes, walk_nodes): + # build intermediate data used downstream + tr_edges_w_routes = hw_intermediate_transit_edges_w_routes.copy() + tr_nodes_w_routes = hw_intermediate_transit_nodes_w_routes.copy() + + transit_edges, transit_nodes = \ + _build_expected_intermediate_transit_node_edge_data_w_headways( + tr_edges_w_routes, tr_nodes_w_routes) + # nodes are expected to have 'nearest_osm_node' column so generate the col + transit_nodes = _build_expected_nearest_osm_node_data( + walk_nodes, transit_nodes) + return transit_edges, transit_nodes + + +@pytest.fixture +def expected_net_edges_nodes_w_headways( + intermediate_transit_edges_nodes_w_headways, + walk_edges, walk_nodes, + expected_connector_edges_w_headways_w_impedance): + # build expected resulting integrated network edge and node tables + transit_edges, transit_nodes = intermediate_transit_edges_nodes_w_headways + + edges_df, nodes_df = _build_expected_node_edge_net_data( + transit_edges, transit_nodes, walk_edges, walk_nodes, + expected_connector_edges_w_headways_w_impedance) + return nodes_df, edges_df + + +@pytest.fixture +def expected_net_edges_nodes_h5(tmpdir, expected_net_edges_nodes): + nodes_df, edges_df = expected_net_edges_nodes + hdf_file_name = 'test_hdf.h5' + hdf_file_path = os.path.join(tmpdir.strpath, hdf_file_name) + edges_df.to_hdf(hdf_file_path, key='edges', mode='a', format='table') + nodes_df.to_hdf(hdf_file_path, key='nodes', mode='a', format='table') + return tmpdir.strpath, hdf_file_name + + @pytest.fixture def ua_net_object_w_transit_walk( transit_edges_1_agency, transit_nodes_1_agency, @@ -282,6 +461,14 @@ def ua_net_object_w_transit_walk( return ua_net +@pytest.fixture +def ua_net_object_expected_result(expected_net_edges_nodes): + nodes_df, edges_df = expected_net_edges_nodes + ua_net.net_edges = edges_df.copy() + ua_net.net_nodes = nodes_df.copy() + return ua_net + + def test_integrate_network_wo_headways( ua_net_object_w_transit_walk, intermediate_transit_edges_nodes, walk_edges, walk_nodes, expected_connector_edges, @@ -304,3 +491,206 @@ def test_integrate_network_wo_headways( expected_connector_edges) assert ua_net_object_result.net_edges.equals(expected_net_edges) assert ua_net_object_result.net_nodes.equals(expected_net_nodes) + + +def test_integrate_network_invalid_wo_headways( + ua_net_object_w_transit_walk): + with pytest.raises(ValueError) as excinfo: + ua_net_object_result = network.integrate_network( + urbanaccess_network=None, + headways=False, + urbanaccess_gtfsfeeds_df=None) + expected_error = 'urbanaccess_network is not specified' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + ua_net_object_result = network.integrate_network( + urbanaccess_network=ua_net_object_w_transit_walk, + headways='test string', + urbanaccess_gtfsfeeds_df=None) + assert 'headways must be bool type' in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + # set required table to be empty + ua_net_object_w_transit_walk.transit_edges = pd.DataFrame() + ua_net_object_result = network.integrate_network( + urbanaccess_network=ua_net_object_w_transit_walk, + headways=False, + urbanaccess_gtfsfeeds_df=None) + expected_error = ( + 'one of the network objects: transit_edges, transit_nodes, ' + 'osm_edges, or osm_nodes were found to be empty') + assert expected_error in str(excinfo.value) + + +def test_format_pandana_edges_nodes(intermediate_transit_edges_nodes): + # for testing purposes we will only use transit edge/node tables + transit_edges, transit_nodes = intermediate_transit_edges_nodes + expected_edge_id = transit_edges['id'].copy() + result_edge, result_node = network._format_pandana_edges_nodes( + edge_df=transit_edges, node_df=transit_nodes) + assert isinstance(result_edge, pd.core.frame.DataFrame) + assert isinstance(result_node, pd.core.frame.DataFrame) + assert result_edge.empty is False + assert result_node.empty is False + # expected node index values + expected_node_index = pd.Series( + data=[1, 2, 3, 4, 5, 6, 7, 8, 9], + index=range(1, 10), + name='id_int') + expected_edge_cols = ['to_int', 'from_int'] + + # check edge formatting + for col in expected_edge_cols: + assert col in result_edge.columns + assert all(result_edge[col].map(type) == int) is True + assert 'edge_id' in result_edge.columns + assert result_edge['edge_id'].equals(expected_edge_id) + + # check node formatting + assert result_node.index.name == 'id_int' + assert result_node.index.to_series().equals(expected_node_index) + assert isinstance(result_node['id'], object) + assert all(result_node['id'].map(type) != str) is False + assert 'nearest_osm_node' not in result_node.columns + + +def test_format_pandana_edges_nodes_unicode_format( + intermediate_transit_edges_nodes): + # for testing purposes we will only use transit edge/node tables + transit_edges, transit_nodes = intermediate_transit_edges_nodes + # set a string column to have characters that may generate encoding issues + char_val = 'Circulação , áéíóúüñ¿¡' + transit_edges['name'] = 'Circulação , áéíóúüñ¿¡' + result_edge, result_node = network._format_pandana_edges_nodes( + edge_df=transit_edges, node_df=transit_nodes) + assert result_edge.empty is False + assert result_edge['name'].iloc[0:1][0] == char_val + + +def test_save_network_wo_headways( + tmpdir, ua_net_object_expected_result, expected_net_edges_nodes): + expected_net_nodes, expected_net_edges = expected_net_edges_nodes + # change 'nan' to np.nan to match expected h5 tables + expected_net_edges = expected_net_edges.replace('nan', np.nan) + expected_net_nodes = expected_net_nodes.replace('nan', np.nan) + + filename = 'test_hdf.h5' + path = os.path.join(tmpdir.strpath, filename) + network.save_network( + urbanaccess_network=ua_net_object_expected_result, + filename=filename, + dir=tmpdir.strpath, + overwrite_key=False, overwrite_hdf5=False) + # test that file exists and read data make sure matches expected + assert os.path.exists(path) is True + with pd.HDFStore(path) as store: + edges = store['edges'] + assert edges.equals(expected_net_edges) + nodes = store['nodes'] + assert nodes.equals(expected_net_nodes) + + +def test_save_network_invalid(tmpdir, ua_net_object_expected_result): + # remove one of the required tables + ua_net_object_expected_result.net_edges = pd.DataFrame() + filename = 'test_hdf.h5' + with pytest.raises(ValueError) as excinfo: + network.save_network( + urbanaccess_network=ua_net_object_expected_result, + filename=filename, + dir=tmpdir.strpath, + overwrite_key=False, overwrite_hdf5=False) + expected_error = ('Either no urbanaccess_network specified or ' + 'net_edges or net_nodes are empty.') + assert expected_error in str(excinfo.value) + + +def test_load_network_wo_headways( + expected_net_edges_nodes_h5, expected_net_edges_nodes): + expected_net_nodes, expected_net_edges = expected_net_edges_nodes + hdf_file_path, hdf_file_name = expected_net_edges_nodes_h5 + + ua_net_object_result = network.load_network( + dir=hdf_file_path, filename=hdf_file_name) + # change 'nan' to np.nan to match expected h5 tables + expected_net_edges = expected_net_edges.replace('nan', np.nan) + expected_net_nodes = expected_net_nodes.replace('nan', np.nan) + # check that results match expected tables + assert ua_net_object_result.net_edges.equals(expected_net_edges) + assert ua_net_object_result.net_nodes.equals(expected_net_nodes) + + +def test_integrate_network_w_headways( + ua_gtfsfeeds_df, + ua_net_object_w_transit_walk, + intermediate_transit_edges_nodes_w_headways, + walk_edges, walk_nodes, + expected_connector_edges_w_headways_w_impedance, + expected_net_edges_nodes_w_headways): + expected_net_nodes, expected_net_edges = \ + expected_net_edges_nodes_w_headways + expected_transit_edges, expected_transit_nodes = \ + intermediate_transit_edges_nodes_w_headways + + ua_net_object_result = network.integrate_network( + urbanaccess_network=ua_net_object_w_transit_walk, + headways=True, + urbanaccess_gtfsfeeds_df=ua_gtfsfeeds_df, + headway_statistic='mean') + + # check that results match expected tables + assert ua_net_object_result.transit_edges.equals(expected_transit_edges) + assert ua_net_object_result.transit_nodes.equals(expected_transit_nodes) + assert ua_net_object_result.osm_edges.equals(walk_edges) + assert ua_net_object_result.osm_nodes.equals(walk_nodes) + assert ua_net_object_result.net_connector_edges.equals( + expected_connector_edges_w_headways_w_impedance) + assert ua_net_object_result.net_edges.equals(expected_net_edges) + assert ua_net_object_result.net_nodes.equals(expected_net_nodes) + + +def test_integrate_network_invalid_w_headways( + ua_gtfsfeeds_df, + ua_net_object_w_transit_walk): + with pytest.raises(ValueError) as excinfo: + ua_net_object_result = network.integrate_network( + urbanaccess_network=ua_net_object_w_transit_walk, + headways=True, + urbanaccess_gtfsfeeds_df=ua_gtfsfeeds_df, + headway_statistic='test') + expected_error = 'test is not a supported statistic or is not a string' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + # make headways invalid + ua_gtfsfeeds_df.headways = pd.DataFrame() + ua_net_object_result = network.integrate_network( + urbanaccess_network=ua_net_object_w_transit_walk, + headways=True, + urbanaccess_gtfsfeeds_df=ua_gtfsfeeds_df, + headway_statistic='mean') + expected_error = ('stops and or headway DataFrames were not found in the ' + 'urbanaccess_gtfsfeeds object. Please create these ' + 'DataFrames in order to use headways.') + assert expected_error in str(excinfo.value) + + +def test_route_id_to_node( + hw_stops_df_1_agency, hw_intermediate_transit_edges_w_routes, + hw_intermediate_transit_nodes_w_routes): + result = network._route_id_to_node( + stops_df=hw_stops_df_1_agency, + edges_w_routes=hw_intermediate_transit_edges_w_routes) + assert isinstance(result, pd.core.frame.DataFrame) + assert result.empty is False + assert result.equals(hw_intermediate_transit_nodes_w_routes) + + +def test_add_headway_impedance( + hw_headways_df_1_agency, expected_connector_edges_w_headways, + expected_connector_edges_w_headways_w_impedance): + result = network._add_headway_impedance( + ped_to_transit_edges_df=expected_connector_edges_w_headways, + headways_df=hw_headways_df_1_agency, + headway_statistic='mean') + assert isinstance(result, pd.core.frame.DataFrame) + assert result.empty is False + assert result.equals(expected_connector_edges_w_headways_w_impedance) From 730feeba4c6af565edbe0e2ce0409e4476d40349 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 14:10:23 -0800 Subject: [PATCH 57/72] updated general unit tests and increased coverage --- .gitignore | 1 + urbanaccess/tests/test_gtfs_network.py | 12 ++++ urbanaccess/tests/test_gtfs_utils_calendar.py | 4 +- urbanaccess/tests/test_gtfs_utils_format.py | 61 +++++++++++-------- .../tests/test_gtfs_utils_validation.py | 7 +++ urbanaccess/tests/test_gtfsfeeds.py | 50 +-------------- 6 files changed, 61 insertions(+), 74 deletions(-) diff --git a/.gitignore b/.gitignore index bf50b1f..d2cf20b 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ /demo/logs /demo/data /tests/images +/tests/data /tests/logs /tests/integration/data /tests/integration/logs diff --git a/urbanaccess/tests/test_gtfs_network.py b/urbanaccess/tests/test_gtfs_network.py index a1cf564..6f5d97e 100644 --- a/urbanaccess/tests/test_gtfs_network.py +++ b/urbanaccess/tests/test_gtfs_network.py @@ -9,6 +9,7 @@ import urbanaccess.gtfs.load as gtfs_load from urbanaccess.network import urbanaccess_network from urbanaccess.gtfs.gtfsfeeds_dataframe import urbanaccess_gtfs_df +from urbanaccess import config @pytest.fixture @@ -1801,6 +1802,17 @@ def test_edge_impedance_by_route_type_invalid_params( funicular=-0.5) # should return a result even if route type is not found in DataFrame assert result.empty is False + with pytest.raises(ValueError) as excinfo: + config._ROUTES_MODE_TYPE_LOOKUP.update({13: 'test mode'}) + result = gtfs_network.edge_impedance_by_route_type( + edge_route_type_impedance_df, + underground_rail='1', + intercity_rail=-0.5) + expected_error = ("ROUTES_MODE_TYPE_LOOKUP keys do not match keys in var_" + "mode_id_lookup. Keys must match.") + assert expected_error in str(excinfo.value) + # reset config removing the test key + del config._ROUTES_MODE_TYPE_LOOKUP[13] def test_save_processed_gtfs_data( diff --git a/urbanaccess/tests/test_gtfs_utils_calendar.py b/urbanaccess/tests/test_gtfs_utils_calendar.py index aaf2d69..302eb94 100644 --- a/urbanaccess/tests/test_gtfs_utils_calendar.py +++ b/urbanaccess/tests/test_gtfs_utils_calendar.py @@ -774,12 +774,12 @@ def test_cal_srv_id_selector_warn(capsys, 'parameters.') in captured.out -def test_unique_service_id( +def test_add_unique_service_id( calendar_dates_agency_a, calendar_agency_a): # remove the existing column from the test data calendar_agency_a.drop(columns=['unique_service_id'], inplace=True) df_list = [calendar_dates_agency_a, calendar_agency_a] - result = gtfs_utils_cal._unique_service_id(df_list) + result = gtfs_utils_cal._add_unique_service_id(df_list) assert isinstance(result, list) assert isinstance(result[0], pd.core.frame.DataFrame) assert isinstance(result[1], pd.core.frame.DataFrame) diff --git a/urbanaccess/tests/test_gtfs_utils_format.py b/urbanaccess/tests/test_gtfs_utils_format.py index 1315c65..d501b15 100644 --- a/urbanaccess/tests/test_gtfs_utils_format.py +++ b/urbanaccess/tests/test_gtfs_utils_format.py @@ -1221,6 +1221,19 @@ def test_timetoseconds_invalid_params(stop_times_feed_1): assert expected_error in str(excinfo.value) +def test_timetoseconds_negative_val_warn(capsys, stop_times_feed_1): + # create a negative time value + stop_times_feed_1['departure_time'].iloc[0] = '-3:20:20' + result = utils_format._timetoseconds( + stop_times_feed_1, time_cols=['departure_time']) + expected_print = ('Warning: Some stop times in departure_time column are ' + 'negative. Time should be positive. Suggest checking' + ' original GTFS feed stop_time file before proceeding') + # test that warning prints were printed + captured = capsys.readouterr() + assert expected_print in captured.out + + def test_timetoseconds_invalid_data(capsys, stop_times_feed_1): # add 2 records with invalid and large hr, min, sec values stop_times_feed_1['departure_time'].iloc[0] = '60:90:80' @@ -1451,7 +1464,7 @@ def test_append_route_type_invalid_param(stops_feed_1, stop_times_feed_1, assert expected_error in str(excinfo.value) -def test_add_unique_agencyid_case_1( +def test_add_unique_agency_id_case_1( agency_a_feed_on_disk_wo_agency, stops_feed_1, stop_times_feed_1, routes_feed_1, trips_feed_1, calendar_feed_1, calendar_dates_feed_1): # case 1: no agency.txt file so we expect 'unique_agency_id' will be @@ -1463,7 +1476,7 @@ def test_add_unique_agencyid_case_1( feed_path = agency_a_feed_on_disk_wo_agency expected_unique_agency_id = os.path.split(feed_path)[1] stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ - calendar_dates_df = utils_format._add_unique_agencyid( + calendar_dates_df = utils_format._add_unique_agency_id( agency_df=blank_agency, stops_df=stops_feed_1, routes_df=routes_feed_1, @@ -1492,7 +1505,7 @@ def test_add_unique_agencyid_case_1( calendar_dates_feed_1) -def test_add_unique_agencyid_case_2( +def test_add_unique_agency_id_case_2( agency_a_feed_on_disk_wo_calendar_dates, agency_feed_3, stops_feed_1, stop_times_feed_1, routes_feed_1, trips_feed_1, calendar_feed_1, calendar_dates_feed_1): @@ -1501,7 +1514,7 @@ def test_add_unique_agencyid_case_2( # using the agency_name in the agency.txt file feed_path = agency_a_feed_on_disk_wo_calendar_dates stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ - calendar_dates_df = utils_format._add_unique_agencyid( + calendar_dates_df = utils_format._add_unique_agency_id( agency_df=agency_feed_3, stops_df=stops_feed_1, routes_df=routes_feed_1, @@ -1529,7 +1542,7 @@ def test_add_unique_agencyid_case_2( calendar_dates_feed_1) -def test_add_unique_agencyid_case_3( +def test_add_unique_agency_id_case_3( agency_a_feed_on_disk_wo_calendar_dates, agency_feed_2, stops_feed_2, stop_times_feed_2, routes_feed_2, trips_feed_2, calendar_feed_2, calendar_dates_feed_2): @@ -1538,7 +1551,7 @@ def test_add_unique_agencyid_case_3( # the agency_id and agency_name in the agency.txt file feed_path = agency_a_feed_on_disk_wo_calendar_dates stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ - calendar_dates_df = utils_format._add_unique_agencyid( + calendar_dates_df = utils_format._add_unique_agency_id( agency_df=agency_feed_2, stops_df=stops_feed_2, routes_df=routes_feed_2, @@ -1567,7 +1580,7 @@ def test_add_unique_agencyid_case_3( calendar_dates_feed_2) -def test_add_unique_agencyid_case_4( +def test_add_unique_agency_id_case_4( agency_a_feed_on_disk_wo_calendar_dates, agency_feed_1, stops_feed_1, stop_times_feed_1, routes_feed_1, trips_feed_1, calendar_feed_1, calendar_dates_feed_1): @@ -1576,7 +1589,7 @@ def test_add_unique_agencyid_case_4( # the agency_name in the agency.txt file feed_path = agency_a_feed_on_disk_wo_calendar_dates stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ - calendar_dates_df = utils_format._add_unique_agencyid( + calendar_dates_df = utils_format._add_unique_agency_id( agency_df=agency_feed_1, stops_df=stops_feed_1, routes_df=routes_feed_1, @@ -1604,7 +1617,7 @@ def test_add_unique_agencyid_case_4( calendar_dates_feed_1) -def test_add_unique_agencyid_case_5( +def test_add_unique_agency_id_case_5( agency_a_feed_on_disk_wo_calendar_dates, agency_feed_1, stops_feed_1, stop_times_feed_1, routes_feed_1, trips_feed_1, calendar_feed_1): # case 5: same as case 4 but with no calendar_dates.txt @@ -1613,7 +1626,7 @@ def test_add_unique_agencyid_case_5( # workflow so replicate this df for this test blank_calendar_dates = pd.DataFrame() stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ - calendar_dates_df = utils_format._add_unique_agencyid( + calendar_dates_df = utils_format._add_unique_agency_id( agency_df=agency_feed_1, stops_df=stops_feed_1, routes_df=routes_feed_1, @@ -1640,7 +1653,7 @@ def test_add_unique_agencyid_case_5( assert calendar_dates_df.equals(blank_calendar_dates) -def test_add_unique_agencyid_case_6( +def test_add_unique_agency_id_case_6( agency_a_feed_on_disk_wo_calendar_dates, agency_feed_1, stops_feed_1, stop_times_feed_1, routes_feed_1, trips_feed_1, calendar_dates_feed_1): # case 6: same as case 4 but with no calendar.txt @@ -1649,7 +1662,7 @@ def test_add_unique_agencyid_case_6( # workflow so replicate this df for this test blank_calendar = pd.DataFrame() stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ - calendar_dates_df = utils_format._add_unique_agencyid( + calendar_dates_df = utils_format._add_unique_agency_id( agency_df=agency_feed_1, stops_df=stops_feed_1, routes_df=routes_feed_1, @@ -1678,7 +1691,7 @@ def test_add_unique_agencyid_case_6( assert calendar_df.equals(blank_calendar) -def test_add_unique_agencyid_multi_agency_id_mismatch_via_agency_txt( +def test_add_unique_agency_id_multi_agency_id_mismatch_via_agency_txt( agency_a_feed_on_disk_wo_calendar_dates, agency_feed_2, stops_feed_2, stop_times_feed_2, routes_feed_2, trips_feed_2, calendar_feed_2, calendar_dates_feed_2): @@ -1688,7 +1701,7 @@ def test_add_unique_agencyid_multi_agency_id_mismatch_via_agency_txt( # match existing records in routes.txt for test agency_feed_2['agency_id'].loc[1] = 'agency missing bus' stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ - calendar_dates_df = utils_format._add_unique_agencyid( + calendar_dates_df = utils_format._add_unique_agency_id( agency_df=agency_feed_2, stops_df=stops_feed_2, routes_df=routes_feed_2, @@ -1718,7 +1731,7 @@ def test_add_unique_agencyid_multi_agency_id_mismatch_via_agency_txt( calendar_dates_feed_2) -def test_add_unique_agencyid_multi_agency_id_mismatch_via_routes_txt( +def test_add_unique_agency_id_multi_agency_id_mismatch_via_routes_txt( agency_a_feed_on_disk_wo_calendar_dates, agency_feed_2, stops_feed_2, stop_times_feed_2, routes_feed_2, trips_feed_2, calendar_feed_2, calendar_dates_feed_2): @@ -1728,7 +1741,7 @@ def test_add_unique_agencyid_multi_agency_id_mismatch_via_routes_txt( # match existing records in agency.txt for test routes_feed_2['agency_id'].loc[0:1] = 'agency missing bus' stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ - calendar_dates_df = utils_format._add_unique_agencyid( + calendar_dates_df = utils_format._add_unique_agency_id( agency_df=agency_feed_2, stops_df=stops_feed_2, routes_df=routes_feed_2, @@ -1758,7 +1771,7 @@ def test_add_unique_agencyid_multi_agency_id_mismatch_via_routes_txt( calendar_dates_feed_2) -def test_add_unique_agencyid_value_errors( +def test_add_unique_agency_id_value_errors( agency_a_feed_on_disk_wo_calendar_dates, agency_a_feed_on_disk_wo_agency, agency_feed_1, stops_feed_1, stop_times_feed_1, routes_feed_1, trips_feed_1, calendar_feed_1, @@ -1768,7 +1781,7 @@ def test_add_unique_agencyid_value_errors( with pytest.raises(ValueError) as excinfo: # throw error if nulls_as_folder=False and no agency.txt file is found stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ - calendar_dates_df = utils_format._add_unique_agencyid( + calendar_dates_df = utils_format._add_unique_agency_id( agency_df=agency_feed_1, stops_df=stops_feed_1, routes_df=routes_feed_1, @@ -1792,7 +1805,7 @@ def test_add_unique_agencyid_value_errors( index = range(1) agency_df = pd.DataFrame(data, index) stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ - calendar_dates_df = utils_format._add_unique_agencyid( + calendar_dates_df = utils_format._add_unique_agency_id( agency_df=agency_df, stops_df=stops_feed_1, routes_df=routes_feed_1, @@ -1817,7 +1830,7 @@ def test_add_unique_agencyid_value_errors( index = range(1) agency_df = pd.DataFrame(data, index) stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ - calendar_dates_df = utils_format._add_unique_agencyid( + calendar_dates_df = utils_format._add_unique_agency_id( agency_df=agency_df, stops_df=stops_feed_1, routes_df=routes_feed_1, @@ -1841,7 +1854,7 @@ def test_add_unique_agencyid_value_errors( index = range(1) agency_df = pd.DataFrame(data, index) stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ - calendar_dates_df = utils_format._add_unique_agencyid( + calendar_dates_df = utils_format._add_unique_agency_id( agency_df=agency_df, stops_df=stops_feed_1, routes_df=routes_feed_1, @@ -1865,7 +1878,7 @@ def test_add_unique_agencyid_value_errors( index = range(1) agency_df = pd.DataFrame(data, index) stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ - calendar_dates_df = utils_format._add_unique_agencyid( + calendar_dates_df = utils_format._add_unique_agency_id( agency_df=agency_df, stops_df=stops_feed_1, routes_df=routes_feed_1, @@ -1889,7 +1902,7 @@ def test_add_unique_agencyid_value_errors( index = range(1) agency_df = pd.DataFrame(data, index) stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ - calendar_dates_df = utils_format._add_unique_agencyid( + calendar_dates_df = utils_format._add_unique_agency_id( agency_df=agency_df, stops_df=stops_feed_1, routes_df=routes_feed_1, @@ -1914,7 +1927,7 @@ def test_add_unique_agencyid_value_errors( index = range(2) agency_df = pd.DataFrame(data, index) stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ - calendar_dates_df = utils_format._add_unique_agencyid( + calendar_dates_df = utils_format._add_unique_agency_id( agency_df=agency_df, stops_df=stops_feed_1, routes_df=routes_feed_1, diff --git a/urbanaccess/tests/test_gtfs_utils_validation.py b/urbanaccess/tests/test_gtfs_utils_validation.py index e099f7f..c6d5e03 100644 --- a/urbanaccess/tests/test_gtfs_utils_validation.py +++ b/urbanaccess/tests/test_gtfs_utils_validation.py @@ -412,3 +412,10 @@ def test_check_time_range_format_invalid_params(): utils_v._check_time_range_format(['10:00:00', '07:00:00']) expected_error = ("['10:00:00', '07:00:00'] {}".format(msg)) assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + utils_v._check_time_range_format(None) + assert 'timerange cannot be None.' in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + utils_v._check_time_range_format(['07:00:000', '11:00:00']) + expected_error = ("['07:00:000', '11:00:00'] {}".format(msg)) + assert expected_error in str(excinfo.value) diff --git a/urbanaccess/tests/test_gtfsfeeds.py b/urbanaccess/tests/test_gtfsfeeds.py index 0646dfe..a76fb6a 100644 --- a/urbanaccess/tests/test_gtfsfeeds.py +++ b/urbanaccess/tests/test_gtfsfeeds.py @@ -323,7 +323,7 @@ def test_feed_object(): assert isinstance(feeds.to_dict(), dict) -def test_to_dict(feed_object): +def test_feed_to_dict(feed_object): assert isinstance(feed_object.to_dict(), dict) assert feed_object.to_dict() == { 'gtfs_feeds': { @@ -445,30 +445,6 @@ def test_to_yaml_feed(tmpdir, feed_dict3): feeds.remove_feed(remove_all=True) -def test_to_yaml_feed_invalid(tmpdir, feed_dict3): - with pytest.raises(ValueError) as excinfo: - feeds.add_feed(add_dict=feed_dict3) - feeds.to_yaml( - gtfsfeeddir=1, yamlname='gtfsfeeds.yaml', - overwrite=True) - expected_error = 'gtfsfeeddir must be a string' - assert expected_error in str(excinfo.value) - with pytest.raises(ValueError) as excinfo: - feeds.to_yaml( - gtfsfeeddir=tmpdir.strpath, yamlname=1, - overwrite=True) - expected_error = 'yamlname must be a string' - assert expected_error in str(excinfo.value) - with pytest.raises(ValueError) as excinfo: - feeds.to_yaml(tmpdir.strpath, overwrite=True) - feeds.to_yaml(tmpdir.strpath, overwrite=False) - expected_error = ('gtfsfeeds.yaml already exists. ' - 'Rename or set overwrite to True') - assert expected_error in str(excinfo.value) - # clear feeds from global memory - feeds.remove_feed(remove_all=True) - - def test_to_yaml_feed_create_dir(tmpdir, feed_dict3): path_to_create = os.path.join(tmpdir.strpath, 'temp') feeds.add_feed(add_dict=feed_dict3) @@ -502,27 +478,6 @@ def test_from_yaml_feed(feed_yaml): def test_from_yaml_feed_invalid( feed_yaml, feed_yaml_invalid_1, feed_yaml_invalid_2, feed_yaml_invalid_3, feed_yaml_invalid_4, feed_yaml_invalid_5): - with pytest.raises(ValueError) as excinfo: - feeds_from_yaml = feeds.from_yaml( - gtfsfeeddir=1, yamlname='gtfsfeeds.yaml') - expected_error = 'gtfsfeeddir must be a string' - assert expected_error in str(excinfo.value) - with pytest.raises(ValueError) as excinfo: - feeds_from_yaml = feeds.from_yaml( - gtfsfeeddir='test_dir', yamlname='test.yaml') - expected_error = 'test_dir does not exist or was not found' - assert expected_error in str(excinfo.value) - with pytest.raises(ValueError) as excinfo: - feeds_from_yaml = feeds.from_yaml( - gtfsfeeddir=feed_yaml, yamlname=1) - expected_error = 'yaml must be a string' - assert expected_error in str(excinfo.value) - with pytest.raises(ValueError) as excinfo: - feeds_from_yaml = feeds.from_yaml( - gtfsfeeddir=feed_yaml_invalid_1, - yamlname='gtfsfeeds_invalid_1.yaml') - expected_error = 'gtfsfeeds_invalid_1.yaml is not a dict' - assert expected_error in str(excinfo.value) with pytest.raises(ValueError) as excinfo: feeds_from_yaml = feeds.from_yaml( gtfsfeeddir=feed_yaml_invalid_2, @@ -547,8 +502,7 @@ def test_from_yaml_feed_invalid( yamlname='gtfsfeeds_invalid_5.yaml') expected_error = ( 'duplicate values were found in YAML file: {}. Feed URL values must ' - 'be unique.'.format(os.path.join( - feed_yaml_invalid_5, 'gtfsfeeds_invalid_5.yaml'))) + 'be unique.'.format('gtfsfeeds_invalid_5.yaml')) assert expected_error in str(excinfo.value) From 306b3b7a285dc7c8a525181e6aa075d77728881a Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 14:11:42 -0800 Subject: [PATCH 58/72] updated tests for gtfs/load.py and increased coverage --- urbanaccess/tests/test_gtfs_load.py | 140 ++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/urbanaccess/tests/test_gtfs_load.py b/urbanaccess/tests/test_gtfs_load.py index 492dbb4..87a28a3 100644 --- a/urbanaccess/tests/test_gtfs_load.py +++ b/urbanaccess/tests/test_gtfs_load.py @@ -229,3 +229,143 @@ def test_loadgtfsfeed_to_df_wo_agency( # check that df is not empty if key in expected_dfs: assert value.empty is False + + +def test_loadgtfsfeed_to_df_invalid_params( + agency_a_feed_on_disk_wo_req_file): + feed_dir = agency_a_feed_on_disk_wo_req_file + with pytest.raises(ValueError) as excinfo: + loaded_feeds = gtfs_load.gtfsfeed_to_df( + gtfsfeed_path=None, + validation=False, + verbose=True, + bbox=None, + remove_stops_outsidebbox=False, + append_definitions=False) + expected_error = "Directory: 'test\\gtfsfeed_text' does not exist." + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + non_existent_dir = os.path.join(feed_dir, 'test') + loaded_feeds = gtfs_load.gtfsfeed_to_df( + gtfsfeed_path=non_existent_dir, + validation=False, + verbose=True, + bbox=None, + remove_stops_outsidebbox=False, + append_definitions=False) + expected_error = "Directory: '{}' does not exist." + assert expected_error.format(non_existent_dir) in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + loaded_feeds = gtfs_load.gtfsfeed_to_df( + gtfsfeed_path=1, + validation=False, + verbose=True, + bbox=None, + remove_stops_outsidebbox=False, + append_definitions=False) + expected_error = 'gtfsfeed_path must be a string.' + assert expected_error in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + loaded_feeds = gtfs_load.gtfsfeed_to_df( + gtfsfeed_path=feed_dir, + validation=True, + verbose=True, + bbox=None, + remove_stops_outsidebbox=False, + append_definitions=False) + expected_error = ('Attempted to run validation but bbox, verbose, and or ' + 'remove_stops_outsidebbox were set to None. These ' + 'parameters must be specified for validation.') + assert expected_error in str(excinfo.value) + + +def test_loadgtfsfeed_to_df_w_validation_remove_stops_True_bbox_large( + agency_a_feed_on_disk_wo_calendar_dates, + expected_urbanaccess_gtfs_df_keys): + # test that all stops are retained given bbox size + feed_dir = agency_a_feed_on_disk_wo_calendar_dates + loaded_feeds = gtfs_load.gtfsfeed_to_df( + gtfsfeed_path=feed_dir, + validation=True, + verbose=True, + bbox=(-122.2850583493, 37.7947251937, -122.2644589841, 37.8131711247), + remove_stops_outsidebbox=True, + append_definitions=False) + assert isinstance(loaded_feeds, urbanaccess_gtfs_df) + urbanaccess_gtfs_df_info = vars(loaded_feeds) + expected_dfs = ['stops', 'routes', 'trips', 'stop_times', + 'calendar'] + assert expected_urbanaccess_gtfs_df_keys == sorted(list( + urbanaccess_gtfs_df_info.keys())) + for key, value in urbanaccess_gtfs_df_info.items(): + assert isinstance(value, pd.core.frame.DataFrame) + # check that df is not empty + if key in expected_dfs: + assert value.empty is False + + +def test_loadgtfsfeed_to_df_w_validation_remove_stops_True_bbox_small( + agency_a_feed_on_disk_wo_calendar_dates, + expected_urbanaccess_gtfs_df_keys): + # test that specific stops are removed given bbox size + feed_dir = agency_a_feed_on_disk_wo_calendar_dates + loaded_feeds = gtfs_load.gtfsfeed_to_df( + gtfsfeed_path=feed_dir, + validation=True, + verbose=True, + bbox=(-122.2702525556, 37.7992692881, -122.2625707089, 37.8036435628), + remove_stops_outsidebbox=True, + append_definitions=False) + assert isinstance(loaded_feeds, urbanaccess_gtfs_df) + urbanaccess_gtfs_df_info = vars(loaded_feeds) + expected_dfs_not_null = ['routes', 'trips', 'calendar'] + expected_dfs_null = ['stops', 'stop_times'] + assert expected_urbanaccess_gtfs_df_keys == sorted(list( + urbanaccess_gtfs_df_info.keys())) + for key, value in urbanaccess_gtfs_df_info.items(): + assert isinstance(value, pd.core.frame.DataFrame) + # check that df is not empty + if key in expected_dfs_not_null: + assert value.empty is False + for key, value in urbanaccess_gtfs_df_info.items(): + assert isinstance(value, pd.core.frame.DataFrame) + # check that df is empty to make sure bbox stop removal operation + # was successful + if key in expected_dfs_null: + assert value.empty is True + + +def test_loadgtfsfeed_to_df_w_append_definitions_True( + agency_a_feed_on_disk_wo_calendar_dates, + expected_urbanaccess_gtfs_df_keys): + feed_dir = agency_a_feed_on_disk_wo_calendar_dates + loaded_feeds = gtfs_load.gtfsfeed_to_df( + gtfsfeed_path=feed_dir, + validation=False, + verbose=True, + bbox=None, + remove_stops_outsidebbox=False, + append_definitions=True) + assert isinstance(loaded_feeds, urbanaccess_gtfs_df) + urbanaccess_gtfs_df_info = vars(loaded_feeds) + expected_dfs = ['stops', 'routes', 'trips', 'stop_times', + 'calendar'] + expected_attr_cols_dict = {'stops': ['wheelchair_boarding_desc', + 'location_type_desc'], + 'routes': ['route_type_desc'], + 'stop_times': ['pickup_type_desc', + 'drop_off_type_desc'], + 'trips': ['bikes_allowed_desc', + 'wheelchair_accessible_desc']} + + assert expected_urbanaccess_gtfs_df_keys == sorted(list( + urbanaccess_gtfs_df_info.keys())) + for table, df in urbanaccess_gtfs_df_info.items(): + assert isinstance(df, pd.core.frame.DataFrame) + # check that df is not empty + if table in expected_dfs: + assert df.empty is False + # check that attr col was added to table + if table in expected_attr_cols_dict.keys(): + for col in expected_attr_cols_dict[table]: + assert col in df.columns From fe25291c3b65c6b750be96592502c44e0c52a687 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Mon, 6 Dec 2021 14:34:52 -0800 Subject: [PATCH 59/72] minor pycodestyle formatting --- urbanaccess/gtfs/utils_format.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/urbanaccess/gtfs/utils_format.py b/urbanaccess/gtfs/utils_format.py index 5434f0c..5beca1b 100644 --- a/urbanaccess/gtfs/utils_format.py +++ b/urbanaccess/gtfs/utils_format.py @@ -394,9 +394,9 @@ def _stop_times_agencyid(stop_times_df, routes_df, trips_df, def _add_unique_agency_id(agency_df, stops_df, routes_df, - trips_df, stop_times_df, calendar_df, - calendar_dates_df, feed_folder, - nulls_as_folder=True): + trips_df, stop_times_df, calendar_df, + calendar_dates_df, feed_folder, + nulls_as_folder=True): """ Create an unique agency ID for all GTFS feed DataFrames to enable unique relational table keys. Pathways to create the unique agency ID are: From efcd50d582dc8e7a6116d9e3389e1f65043207ba Mon Sep 17 00:00:00 2001 From: jessicacamacho Date: Tue, 21 Jun 2022 10:31:21 -0300 Subject: [PATCH 60/72] Adds direction to headways calculation --- urbanaccess/gtfs/headways.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/urbanaccess/gtfs/headways.py b/urbanaccess/gtfs/headways.py index 04a06e5..2354f00 100644 --- a/urbanaccess/gtfs/headways.py +++ b/urbanaccess/gtfs/headways.py @@ -42,15 +42,26 @@ def _calc_headways_by_route_stop(df): results = {} col = 'departure_time_sec_interpolate' for unique_stop_route, stop_route_group in stop_route_groups: - stop_route_group.sort_values([col], ascending=True, inplace=True) - next_veh_time = (stop_route_group[col].iloc[1:].values) - prev_veh_time = (stop_route_group[col].iloc[:-1].values) - # compute headway with: - # (next veh arrival time - previous veh arrival time) / 60 seconds - # = headway in min - stop_route_group_headways = (next_veh_time - prev_veh_time) / 60 - results[unique_stop_route] = (pd.Series(stop_route_group_headways) - .describe()) + stop_route_group_headways = pd.DataFrame() + if 'direction_id' not in stop_route_group.columns: + stop_route_group.sort_values([col], ascending=True, inplace=True) + next_veh_time = (stop_route_group[col].iloc[1:].values) + prev_veh_time = (stop_route_group[col].iloc[:-1].values) + # compute headway with: + # (next veh arrival time - previous veh arrival time) / 60 seconds + # = headway in min + stop_route_group_headways = (next_veh_time - prev_veh_time) / 60 + results[unique_stop_route] = (pd.Series(stop_route_group_headways) + .describe()) + else: + for direction in stop_route_group.direction_id.unique(): + stop_route_group_dir = stop_route_group[stop_route_group['direction_id']==direction] + stop_route_group_dir.sort_values([col], ascending=True, inplace=True) + next_veh_time = (stop_route_group_dir[col].iloc[1:].values) + prev_veh_time = (stop_route_group_dir[col].iloc[:-1].values) + headways_dir = pd.DataFrame((next_veh_time - prev_veh_time) / 60, columns=['headway']) + stop_route_group_headways = stop_route_group_headways.append(headways_dir) + results[unique_stop_route] = (stop_route_group_headways['headway'].describe()) log('Route stop headway calculation complete. Took {:,.2f} seconds'.format( time.time() - start_time)) From 083c91bc8fc1c21110658976f45c1956f6e745fb Mon Sep 17 00:00:00 2001 From: jessicacamacho Date: Tue, 21 Jun 2022 10:31:43 -0300 Subject: [PATCH 61/72] Optimizes _route_type_to_edge() for memory --- urbanaccess/gtfs/network.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/urbanaccess/gtfs/network.py b/urbanaccess/gtfs/network.py index 48d0657..850634b 100644 --- a/urbanaccess/gtfs/network.py +++ b/urbanaccess/gtfs/network.py @@ -999,17 +999,11 @@ def _route_type_to_edge(transit_edge_df, stop_time_df): stop_time_df = _add_unique_trip_id(stop_time_df) # join route_id to the edge table - merged_df = pd.merge( - transit_edge_df, stop_time_df[['unique_trip_id', 'route_type']], - how='left', on='unique_trip_id', sort=False, copy=False) - merged_df.drop_duplicates( - subset='unique_trip_id', keep='first', inplace=True) - # need to get unique records here to have a one to one join - - # this serves as the look up table - # join the look up table created above to the table of interest - transit_edge_df_w_routetype = pd.merge( - transit_edge_df, merged_df[['route_type', 'unique_trip_id']], - how='left', on='unique_trip_id', sort=False, copy=False) + transit_edge_df = transit_edge_df.reset_index().set_index('unique_trip_id') + stop_time_df = stop_time_df[['unique_trip_id', 'route_type']].set_index('unique_trip_id') + stop_time_df = stop_time_df[~stop_time_df.index.duplicated(keep='first')] + merged_df = transit_edge_df.join(stop_time_df[['route_type']]) + transit_edge_df_w_routetype = merged_df.reset_index() log('Route type successfully joined to transit edges. ' 'Took {:,.2f} seconds.'.format(time.time() - start_time)) From 22b1e65d29e119834ba0ee802795d6ffec58f96e Mon Sep 17 00:00:00 2001 From: jessicacamacho Date: Tue, 21 Jun 2022 12:10:41 -0300 Subject: [PATCH 62/72] Fixes deprecation issue of append method --- urbanaccess/gtfs/headways.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/urbanaccess/gtfs/headways.py b/urbanaccess/gtfs/headways.py index 2354f00..eecc2d0 100644 --- a/urbanaccess/gtfs/headways.py +++ b/urbanaccess/gtfs/headways.py @@ -60,7 +60,7 @@ def _calc_headways_by_route_stop(df): next_veh_time = (stop_route_group_dir[col].iloc[1:].values) prev_veh_time = (stop_route_group_dir[col].iloc[:-1].values) headways_dir = pd.DataFrame((next_veh_time - prev_veh_time) / 60, columns=['headway']) - stop_route_group_headways = stop_route_group_headways.append(headways_dir) + stop_route_group_headways = pd.concat([stop_route_group_headways, headways_dir]) results[unique_stop_route] = (stop_route_group_headways['headway'].describe()) log('Route stop headway calculation complete. Took {:,.2f} seconds'.format( From acaad8d05e9750e09b4480dde7864d47b8a084ad Mon Sep 17 00:00:00 2001 From: sablanchard Date: Thu, 11 May 2023 15:49:10 -0700 Subject: [PATCH 63/72] add support for shapes.txt --- urbanaccess/config.py | 19 ++++-- urbanaccess/gtfs/load.py | 18 +++++- urbanaccess/gtfs/utils_format.py | 106 +++++++++++++++++++++++++++---- 3 files changed, 124 insertions(+), 19 deletions(-) diff --git a/urbanaccess/config.py b/urbanaccess/config.py index 5dd25de..dd003b3 100644 --- a/urbanaccess/config.py +++ b/urbanaccess/config.py @@ -270,7 +270,7 @@ def to_yaml(self, configdir='configs', yamlname='urbanaccess_config.yaml', _GTFS_TXT_FILE_TYPES = { 'required_files': ['stops.txt', 'routes.txt', 'trips.txt', 'stop_times.txt'], - 'optional_files': ['agency.txt'], + 'optional_files': ['agency.txt', 'shapes.txt'], 'calendar_files': ['calendar.txt', 'calendar_dates.txt']} _SUPPORTED_GTFS_TXT_FILES = [] @@ -291,16 +291,20 @@ def to_yaml(self, configdir='configs', yamlname='urbanaccess_config.yaml', 'remove_whitespace': ['stop_id'], 'min_required_cols': ['stop_id', 'stop_lat', 'stop_lon']}, 'routes': {'req_dtypes': {'route_id': object}, - 'opt_dtypes': None, + 'opt_dtypes': {'route_short_name': object, + 'route_long_name': object, + 'route_color': object, + 'route_text_color': object}, 'numeric_converter': None, 'remove_whitespace': ['route_id'], 'min_required_cols': ['route_id']}, 'trips': {'req_dtypes': {'trip_id': object, - 'service_id': str, + 'service_id': object, 'route_id': object}, 'opt_dtypes': {'shape_id': object}, 'numeric_converter': None, - 'remove_whitespace': ['trip_id', 'service_id', 'route_id'], + 'remove_whitespace': ['trip_id', 'service_id', 'route_id', + 'shape_id'], 'min_required_cols': ['trip_id', 'service_id', 'route_id']}, 'stop_times': {'req_dtypes': {'trip_id': object, @@ -312,6 +316,13 @@ def to_yaml(self, configdir='configs', yamlname='urbanaccess_config.yaml', 'remove_whitespace': ['trip_id', 'stop_id'], 'min_required_cols': ['trip_id', 'stop_id', 'departure_time', 'arrival_time']}, + 'shapes': {'req_dtypes': {'shape_id': object}, + 'opt_dtypes': None, + 'numeric_converter': ['shape_pt_lat', 'shape_pt_lon', + 'shape_pt_sequence'], + 'remove_whitespace': ['shape_id'], + 'min_required_cols': ['shape_id', 'shape_pt_lat', + 'shape_pt_lon', 'shape_pt_sequence']}, 'calendar': {'req_dtypes': {'service_id': object, 'start_date': object, 'end_date': object}, diff --git a/urbanaccess/gtfs/load.py b/urbanaccess/gtfs/load.py index 32a2405..3313d8e 100644 --- a/urbanaccess/gtfs/load.py +++ b/urbanaccess/gtfs/load.py @@ -210,6 +210,7 @@ def gtfsfeed_to_df(gtfsfeed_path=None, validation=False, verbose=True, gtfsfeeds_dfs.routes : pandas.DataFrame gtfsfeeds_dfs.trips : pandas.DataFrame gtfsfeeds_dfs.stop_times : pandas.DataFrame + gtfsfeeds_dfs.shapes : pandas.DataFrame gtfsfeeds_dfs.calendar : pandas.DataFrame gtfsfeeds_dfs.calendar_dates : pandas.DataFrame """ @@ -218,6 +219,7 @@ def gtfsfeed_to_df(gtfsfeed_path=None, validation=False, verbose=True, merged_routes_df = pd.DataFrame() merged_trips_df = pd.DataFrame() merged_stop_times_df = pd.DataFrame() + merged_shapes_df = pd.DataFrame() merged_calendar_df = pd.DataFrame() merged_calendar_dates_df = pd.DataFrame() @@ -336,25 +338,34 @@ def gtfsfeed_to_df(gtfsfeed_path=None, validation=False, verbose=True, textfile=textfile) else: agency_df = pd.DataFrame() + if textfile == 'shapes.txt': + if textfile in textfilelist: + shapes_df = utils_format._read_gtfs_file( + textfile_path=os.path.join(gtfsfeed_path, folder), + textfile=textfile) + else: + shapes_df = pd.DataFrame() - stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ + stops_df, routes_df, trips_df, stop_times_df, shapes_df, calendar_df, \ calendar_dates_df = utils_format._add_unique_agency_id( agency_df=agency_df, stops_df=stops_df, routes_df=routes_df, trips_df=trips_df, stop_times_df=stop_times_df, + shapes_df=shapes_df, calendar_df=calendar_df, calendar_dates_df=calendar_dates_df, feed_folder=os.path.join(gtfsfeed_path, folder), nulls_as_folder=True) - stops_df, routes_df, trips_df, stop_times_df, calendar_df, \ + stops_df, routes_df, trips_df, stop_times_df, shapes_df, calendar_df, \ calendar_dates_df = utils_format._add_unique_gtfsfeed_id( stops_df=stops_df, routes_df=routes_df, trips_df=trips_df, stop_times_df=stop_times_df, + shapes_df=shapes_df, calendar_df=calendar_df, calendar_dates_df=calendar_dates_df, feed_folder=folder, @@ -393,6 +404,8 @@ def gtfsfeed_to_df(gtfsfeed_path=None, validation=False, verbose=True, trips_df, ignore_index=True) merged_stop_times_df = merged_stop_times_df.append( stop_times_df, ignore_index=True) + merged_shapes_df = merged_shapes_df.append( + shapes_df, ignore_index=True) merged_calendar_df = merged_calendar_df.append( calendar_df, ignore_index=True) merged_calendar_dates_df = merged_calendar_dates_df.append( @@ -417,6 +430,7 @@ def gtfsfeed_to_df(gtfsfeed_path=None, validation=False, verbose=True, gtfsfeeds_dfs.routes = merged_routes_df gtfsfeeds_dfs.trips = merged_trips_df gtfsfeeds_dfs.stop_times = merged_stop_times_df + gtfsfeeds_dfs.shapes = merged_shapes_df gtfsfeeds_dfs.calendar = merged_calendar_df gtfsfeeds_dfs.calendar_dates = merged_calendar_dates_df diff --git a/urbanaccess/gtfs/utils_format.py b/urbanaccess/gtfs/utils_format.py index 5beca1b..11b15a3 100644 --- a/urbanaccess/gtfs/utils_format.py +++ b/urbanaccess/gtfs/utils_format.py @@ -77,12 +77,18 @@ def _read_gtfs_file(textfile_path, textfile): if df.empty: cal_file_warnings = {'calendar': 'calendar_dates.txt', 'calendar_dates': 'calendar.txt'} + shape_file_warnings = {'shapes': 'shapes.txt'} if file_name in cal_file_warnings.keys(): warning_msg = ( ' {} has no records. This could indicate that this feed ' 'is using {} instead of {}.txt for service_ids.') log(warning_msg.format(textfile, cal_file_warnings[file_name], file_name), level=lg.WARNING) + elif file_name in shape_file_warnings.keys(): + warning_msg = ( + ' {} has no records. This feed does not use {}.txt.') + log(warning_msg.format(textfile, shape_file_warnings[file_name], + file_name), level=lg.WARNING) else: raise ValueError('{} has no records. ' 'This file cannot be empty.'.format(textfile)) @@ -149,7 +155,6 @@ def _calendar_dates_agencyid(calendar_dates_df, routes_df, 'service_id', 'unique_agency_id']).size() group_counts = group.reset_index(level=1) # check if service_ids are associated with more than one agency - if any(group_counts.index.value_counts().values > 1): feed_name = os.path.split(feed_folder)[1] log('GTFS feed: {!s}, calendar_dates uses the same service_id across ' @@ -393,8 +398,63 @@ def _stop_times_agencyid(stop_times_df, routes_df, trips_df, return merged_df +def _shapes_agencyid(shapes_df, trips_df, routes_df, agency_df, feed_folder): + """ + Assign unique agency ID to shapes DataFrame + + Parameters + ---------- + shapes_df : pandas:DataFrame + shapes DataFrame + trips_df : pandas:DataFrame + trips DataFrame + routes_df : pandas:DataFrame + routes DataFrame + agency_df : pandas:DataFrame + agency DataFrame + feed_folder : str + name of GTFS feed folder + + Returns + ------- + merged_df : pandas.DataFrame + """ + tmp1 = pd.merge(routes_df, agency_df, how='left', on='agency_id', + sort=False, copy=False) + merged_df = pd.merge(trips_df[['shape_id', 'trip_id', 'route_id']], + tmp1, how='left', + on='route_id', sort=False, copy=False) + merged_df['unique_agency_id'] = _generate_unique_agency_id( + merged_df, 'agency_name') + + group = merged_df[['shape_id', 'unique_agency_id']].groupby([ + 'shape_id', 'unique_agency_id']).size() + group_counts = group.reset_index(level=1) + # check if shape_ids are associated with more than one agency + if any(group_counts.index.value_counts().values > 1): + feed_name = os.path.split(feed_folder)[1] + log('GTFS feed: {!s}, shapes uses the same shape_id across ' + 'multiple agency_ids. This feed shapes table will be ' + 'modified from its original format to provide shape_ids for ' + 'each agency using a one to many join.'.format(feed_name)) + + tmp = merged_df[['shape_id', 'unique_agency_id']].drop_duplicates( + ['shape_id', 'unique_agency_id'], inplace=False) + merged_df = tmp.merge(shapes_df, 'left', on='shape_id') + + else: + merged_df.drop_duplicates( + subset='shape_id', keep='first', inplace=True) + + merged_df = pd.merge(shapes_df, + merged_df[['unique_agency_id', 'shape_id']], + how='left', on='shape_id', + sort=False, copy=False) + return merged_df + + def _add_unique_agency_id(agency_df, stops_df, routes_df, - trips_df, stop_times_df, calendar_df, + trips_df, stop_times_df, shapes_df, calendar_df, calendar_dates_df, feed_folder, nulls_as_folder=True): """ @@ -430,6 +490,8 @@ def _add_unique_agency_id(agency_df, stops_df, routes_df, trips DataFrame stop_times_df : pandas:DataFrame stop times DataFrame + shapes_df : pandas:DataFrame + shapes DataFrame calendar_df : pandas:DataFrame calendar DataFrame calendar_dates_df : pandas:DataFrame @@ -441,7 +503,7 @@ def _add_unique_agency_id(agency_df, stops_df, routes_df, name will be used as the unique agency ID Returns ------- - stops_df, routes_df, trips_df, stop_times_df, calendar_df, + stops_df, routes_df, trips_df, stop_times_df, shapes_df, calendar_df, calendar_dates_df : pandas.DataFrame Returns all input GTFS DataFrames with a unique agency ID column and value for all tables and records. @@ -461,7 +523,8 @@ def _add_unique_agency_id(agency_df, stops_df, routes_df, 'stop_times': stop_times_df} optional_df_dict = {'calendar': calendar_df, - 'calendar_dates': calendar_dates_df} + 'calendar_dates': calendar_dates_df, + 'shapes': shapes_df} # if optional calendar or calendar_dates_df are not empty then add it # to the processing list for name, df in optional_df_dict.items(): @@ -561,6 +624,16 @@ def _add_unique_agency_id(agency_df, stops_df, routes_df, # below, apply the unique agency ID to all GTFS dfs via # merge operations + # if optional shapes_df is not empty then process it + if shapes_df.empty is False: + subset_trips_df_w_shpid = trips_df[ + ['trip_id', 'route_id', 'shape_id']] + shapes_replacement_df = _shapes_agencyid( + shapes_df=shapes_df, + routes_df=subset_routes_df, + trips_df=subset_trips_df_w_shpid, + agency_df=subset_agency_df, + feed_folder=feed_folder) # if optional calendar_dates_df is not empty then process it if calendar_dates_df.empty is False: @@ -609,8 +682,11 @@ def _add_unique_agency_id(agency_df, stops_df, routes_df, 'trips': trips_replacement_df, 'stop_times': stop_times_replacement_df} - # if optional calendar or calendar_dates_df are not empty then - # add it to the processing list + # if optional shapes, calendar, or calendar_dates_df are not + # empty then add it to the processing list + if shapes_df.empty is False: + df_dict.update( + {'shapes': shapes_replacement_df}) if calendar_df.empty is False: df_dict.update( {'calendar': calendar_replacement_df}) @@ -645,7 +721,8 @@ def _add_unique_agency_id(agency_df, stops_df, routes_df, # TODO: this dict may be redundant check if can refactor optional_df_dict = { 'calendar': calendar_df, - 'calendar_dates': calendar_dates_df} + 'calendar_dates': calendar_dates_df, + 'shapes': shapes_df} # if optional calendar or calendar_dates_df are empty then return # the original empty df with an empty agency ID column for name, df in optional_df_dict.items(): @@ -655,7 +732,7 @@ def _add_unique_agency_id(agency_df, stops_df, routes_df, # ensure returned items in list are always in the same expected order df_list = [df_dict['stops'], df_dict['routes'], df_dict['trips'], - df_dict['stop_times'], df_dict['calendar'], + df_dict['stop_times'], df_dict['shapes'], df_dict['calendar'], df_dict['calendar_dates']] log('Unique agency ID operation complete. ' @@ -664,8 +741,8 @@ def _add_unique_agency_id(agency_df, stops_df, routes_df, def _add_unique_gtfsfeed_id(stops_df, routes_df, trips_df, - stop_times_df, calendar_df, calendar_dates_df, - feed_folder, feed_number): + stop_times_df, shapes_df, calendar_df, + calendar_dates_df, feed_folder, feed_number): """ Create an unique GTFS feed specific ID for all GTFS feed DataFrames to enable tracking of specific feeds @@ -680,6 +757,8 @@ def _add_unique_gtfsfeed_id(stops_df, routes_df, trips_df, trips DataFrame stop_times_df : pandas:DataFrame stop times DataFrame + shapes_df : pandas:DataFrame + shapes DataFrame calendar_df : pandas:DataFrame calendar DataFrame calendar_dates_df : pandas:DataFrame @@ -690,7 +769,7 @@ def _add_unique_gtfsfeed_id(stops_df, routes_df, trips_df, current number iteration of GTFS feed being read in root directory Returns ------- - stops_df, routes_df, trips_df, stop_times_df, calendar_df, + stops_df, routes_df, trips_df, stop_times_df, shapes_df, calendar_df, calendar_dates_df : pandas.DataFrame """ start_time = time.time() @@ -701,7 +780,8 @@ def _add_unique_gtfsfeed_id(stops_df, routes_df, trips_df, 'stop_times': stop_times_df} optional_df_dict = {'calendar': calendar_df, - 'calendar_dates': calendar_dates_df} + 'calendar_dates': calendar_dates_df, + 'shapes': shapes_df} # if optional calendar or calendar_dates_df are not empty then add it # to the processing list for name, df in optional_df_dict.items(): @@ -725,7 +805,7 @@ def _add_unique_gtfsfeed_id(stops_df, routes_df, trips_df, # ensure returned items in list are always in the same expected order df_list = [df_dict['stops'], df_dict['routes'], df_dict['trips'], - df_dict['stop_times'], df_dict['calendar'], + df_dict['stop_times'], df_dict['shapes'], df_dict['calendar'], df_dict['calendar_dates']] log('Unique GTFS feed ID operation complete. ' From c9ed55ce350e1d36fd7111eb0eacfe93105d7455 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Thu, 11 May 2023 15:51:14 -0700 Subject: [PATCH 64/72] remove white space on departure and arrival time columns and update function to not fail if there are no lines to read --- urbanaccess/config.py | 3 ++- urbanaccess/gtfs/load.py | 36 +++++++++++++++++++----------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/urbanaccess/config.py b/urbanaccess/config.py index dd003b3..5c62b2f 100644 --- a/urbanaccess/config.py +++ b/urbanaccess/config.py @@ -313,7 +313,8 @@ def to_yaml(self, configdir='configs', yamlname='urbanaccess_config.yaml', 'arrival_time': object}, 'opt_dtypes': None, 'numeric_converter': None, - 'remove_whitespace': ['trip_id', 'stop_id'], + 'remove_whitespace': ['trip_id', 'stop_id', + 'departure_time', 'arrival_time'], 'min_required_cols': ['trip_id', 'stop_id', 'departure_time', 'arrival_time']}, 'shapes': {'req_dtypes': {'shape_id': object}, diff --git a/urbanaccess/gtfs/load.py b/urbanaccess/gtfs/load.py index 3313d8e..056e13d 100644 --- a/urbanaccess/gtfs/load.py +++ b/urbanaccess/gtfs/load.py @@ -143,23 +143,25 @@ def _txt_header_whitespace_check( file_path, encoding=txt_encoding) as f: lines = f.readlines() - line_wo_whitespace = re.sub(r'\s+', '', lines[0]) + '\n' - # only write the file if there are changes to be made - if lines[0] != line_wo_whitespace: - msg = 'Removing whitespace from header(s) in: {}...' - log(msg.format(file_path)) - lines[0] = line_wo_whitespace - # Write to file - if six.PY2: - with open( - file_path, 'w') as f: - f.writelines(lines) - else: - # write with default 'utf-8' encoding - with open( - file_path, 'w', - encoding=txt_encoding) as f: - f.writelines(lines) + if len(lines) != 0: + line_wo_whitespace = re.sub(r'\s+', '', lines[0]) + '\n' + # only write the file if there are changes to be made + if lines[0] != line_wo_whitespace: + msg = ('Removing whitespace from ' + 'header(s) in: {}...') + log(msg.format(file_path)) + lines[0] = line_wo_whitespace + # Write to file + if six.PY2: + with open( + file_path, 'w') as f: + f.writelines(lines) + else: + # write with default 'utf-8' encoding + with open( + file_path, 'w', + encoding=txt_encoding) as f: + f.writelines(lines) except Exception as e: msg = 'Unable to process: {}. Exception: {}' raise Exception(log(msg.format(file_path, e), From ac03ceae591efe3c7f98d06ccb9921ad40560357 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Thu, 11 May 2023 15:52:18 -0700 Subject: [PATCH 65/72] update how optional and required dtypes are dealt with --- urbanaccess/gtfs/utils_format.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/urbanaccess/gtfs/utils_format.py b/urbanaccess/gtfs/utils_format.py index 11b15a3..03d34f3 100644 --- a/urbanaccess/gtfs/utils_format.py +++ b/urbanaccess/gtfs/utils_format.py @@ -65,13 +65,19 @@ def _read_gtfs_file(textfile_path, textfile): '{} is missing required column(s): {}.'.format( textfile, missing_req_cols)) - # if optional dtype col exists include it in req_dtypes dict - if opt_dtypes is not None: - for col_name, dtype in opt_dtypes.items(): - if col_name in col_list: - req_dtypes.update({col_name: dtype}) + # build dtypes dict + if req_dtypes is not None or opt_dtypes is not None: + dtype_dict = {} + dtype_dict.update(req_dtypes) + # if optional dtype col exists include it in dtypes dict + if opt_dtypes is not None: + for col_name, dtype in opt_dtypes.items(): + if col_name in col_list: + dtype_dict.update({col_name: dtype}) + else: + dtype_dict = None - df = pd.read_csv(file_path, dtype=req_dtypes, low_memory=False) + df = pd.read_csv(file_path, dtype=dtype_dict, low_memory=False) # print warning or raise error when table is empty depending on the table if df.empty: From 92037262020b9395928c66b1eaee459dd12c3574 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Thu, 11 May 2023 15:52:48 -0700 Subject: [PATCH 66/72] update numeric col converter --- urbanaccess/gtfs/utils_format.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/urbanaccess/gtfs/utils_format.py b/urbanaccess/gtfs/utils_format.py index 03d34f3..23a44cb 100644 --- a/urbanaccess/gtfs/utils_format.py +++ b/urbanaccess/gtfs/utils_format.py @@ -115,7 +115,8 @@ def _read_gtfs_file(textfile_path, textfile): if numeric_converter is not None: for col in numeric_converter: - df[col] = pd.to_numeric(df[col]) + if col in df.columns: + df[col] = pd.to_numeric(df[col]) record_cnt = len(df) msg = (' Successfully read: {} with {:,} record(s). ' From 789ab180653f549476f84ef52bfdc8aa147506a8 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Thu, 11 May 2023 15:53:44 -0700 Subject: [PATCH 67/72] allow removal of column spaces to pass when expected columns dont exist --- urbanaccess/gtfs/utils_format.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/urbanaccess/gtfs/utils_format.py b/urbanaccess/gtfs/utils_format.py index 23a44cb..b7e86dc 100644 --- a/urbanaccess/gtfs/utils_format.py +++ b/urbanaccess/gtfs/utils_format.py @@ -1124,8 +1124,11 @@ def _remove_whitespace(df, textfile, col_list=None): # remove leading and trailing spaces in values for columns in list if col_list: + # rebuild col_list so that it doesnt fail on expected columns that were + # not present in the file + present_col_list = [col for col in col_list if col in after_cols] df_copy = df.copy() - for col in col_list: + for col in present_col_list: before_count = df_copy[col].str.len().sum() df_copy[col] = df_copy[col].str.rstrip().str.lstrip() after_count = df_copy[col].str.len().sum() From 8e3a8378059bc168598a81a6674e84ee990bc2c9 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Thu, 11 May 2023 15:54:09 -0700 Subject: [PATCH 68/72] fix negative time check --- urbanaccess/gtfs/utils_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/urbanaccess/gtfs/utils_format.py b/urbanaccess/gtfs/utils_format.py index b7e86dc..1953caa 100644 --- a/urbanaccess/gtfs/utils_format.py +++ b/urbanaccess/gtfs/utils_format.py @@ -884,7 +884,7 @@ def _timetoseconds(df, time_cols): series_df = col_series.to_frame(name=''.join([col, '_sec'])) # check if times are negative if so display warning - if (series_df > 1).any()[0]: + if (series_df < 0).any()[0]: log('Warning: Some stop times in {} column are negative. ' 'Time should be positive. Suggest checking original ' 'GTFS feed stop_time file before proceeding.'.format(col), From d8bca8fad0c89f0f4c7839de5ff6d8ade70c8855 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Thu, 11 May 2023 15:54:42 -0700 Subject: [PATCH 69/72] add exception catch when txt cannot be read --- urbanaccess/gtfs/utils_format.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/urbanaccess/gtfs/utils_format.py b/urbanaccess/gtfs/utils_format.py index 1953caa..8d5af44 100644 --- a/urbanaccess/gtfs/utils_format.py +++ b/urbanaccess/gtfs/utils_format.py @@ -1155,5 +1155,9 @@ def _list_raw_txt_columns(file): df : list list of columns in txt file """ - df = pd.read_csv(file) + try: + df = pd.read_csv(file) + except Exception as e: + msg = 'Unable to process: {}. Exception: {}' + raise ValueError(msg.format(file, e)) return list(df.columns) From 53db6a6b332cff578d2851eb61ccc4c639ddc0b6 Mon Sep 17 00:00:00 2001 From: sablanchard Date: Thu, 11 May 2023 15:55:25 -0700 Subject: [PATCH 70/72] minor doc conf update --- docs/source/conf.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index a073f17..a48aaaf 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,11 +28,11 @@ source_suffix = '.rst' master_doc = 'index' project = u'UrbanAccess' -author = u'UrbanSim Inc.' +author = u'UrbanSim Inc. and Samuel D. Blanchard' copyright = u'{}, {}'.format(datetime.now().year, author) version = u'0.2.2' -release = u'0.2.2' -language = None +release = version +language = 'en' # List of patterns to ignore when looking for source files. exclude_patterns = ['_build'] @@ -54,7 +54,7 @@ # Output file base name for HTML help builder. htmlhelp_basename = 'UrbanAccessdoc' html_show_sphinx = False -html_show_sourcelink = True +html_show_sourcelink = False # -- Options for LaTeX output --------------------------------------------- latex_elements = { From fae6f8bc8cba155612563ead5bed8e90399c42b6 Mon Sep 17 00:00:00 2001 From: knaaptime Date: Fri, 7 Feb 2025 13:29:01 -0800 Subject: [PATCH 71/72] pandas compat; working network integration --- urbanaccess/gtfs/load.py | 14 +++--- urbanaccess/gtfs/network.py | 8 ++-- urbanaccess/gtfs/utils_calendar.py | 2 +- urbanaccess/gtfsfeeds.py | 2 +- urbanaccess/network.py | 44 ++++++++++++------- urbanaccess/tests/test_gtfs_utils_calendar.py | 6 +-- 6 files changed, 42 insertions(+), 34 deletions(-) diff --git a/urbanaccess/gtfs/load.py b/urbanaccess/gtfs/load.py index 056e13d..d18d063 100644 --- a/urbanaccess/gtfs/load.py +++ b/urbanaccess/gtfs/load.py @@ -398,19 +398,19 @@ def gtfsfeed_to_df(gtfsfeed_path=None, validation=False, verbose=True, trips_df=trips_df[['trip_id', 'route_id']], info_to_append='route_type_to_stop_times') - merged_stops_df = merged_stops_df.append( + merged_stops_df = merged_stops_df._append( stops_df, ignore_index=True) - merged_routes_df = merged_routes_df.append( + merged_routes_df = merged_routes_df._append( routes_df, ignore_index=True) - merged_trips_df = merged_trips_df.append( + merged_trips_df = merged_trips_df._append( trips_df, ignore_index=True) - merged_stop_times_df = merged_stop_times_df.append( + merged_stop_times_df = merged_stop_times_df._append( stop_times_df, ignore_index=True) - merged_shapes_df = merged_shapes_df.append( + merged_shapes_df = merged_shapes_df._append( shapes_df, ignore_index=True) - merged_calendar_df = merged_calendar_df.append( + merged_calendar_df = merged_calendar_df._append( calendar_df, ignore_index=True) - merged_calendar_dates_df = merged_calendar_dates_df.append( + merged_calendar_dates_df = merged_calendar_dates_df._append( calendar_dates_df, ignore_index=True) # print break to visually separate each GTFS feed log diff --git a/urbanaccess/gtfs/network.py b/urbanaccess/gtfs/network.py index 850634b..cf83b8c 100644 --- a/urbanaccess/gtfs/network.py +++ b/urbanaccess/gtfs/network.py @@ -812,7 +812,7 @@ def _format_transit_net_edge(stop_times_df, time_aware=False): for trip, tmp_trip_df in stop_times_df.groupby(['unique_trip_id']): # if 'time_aware', also create arrival and departure time cols if time_aware: - edge_df = pd.DataFrame({ + edge_df = pd.DataFrame.from_dict({ "node_id_from": tmp_trip_df['unique_stop_id'].iloc[:-1].values, "node_id_to": tmp_trip_df['unique_stop_id'].iloc[1:].values, "weight": tmp_trip_df['timediff'].iloc[1:].values, @@ -827,9 +827,9 @@ def _format_transit_net_edge(stop_times_df, time_aware=False): # arrival_time at node_id_to stop "arrival_time": tmp_trip_df['arrival_time'].iloc[1:].values - }) + }, orient='index').T.ffill() else: - edge_df = pd.DataFrame({ + edge_df = pd.DataFrame.from_dict({ "node_id_from": tmp_trip_df['unique_stop_id'].iloc[:-1].values, "node_id_to": tmp_trip_df['unique_stop_id'].iloc[1:].values, "weight": tmp_trip_df['timediff'].iloc[1:].values, @@ -838,7 +838,7 @@ def _format_transit_net_edge(stop_times_df, time_aware=False): # set unique trip ID without edge order to join other data # later "unique_trip_id": trip - }) + }, orient='index').T.ffill().dropna() # Set current trip ID to edge ID column adding edge order at # end of string diff --git a/urbanaccess/gtfs/utils_calendar.py b/urbanaccess/gtfs/utils_calendar.py index 4361720..daaf868 100644 --- a/urbanaccess/gtfs/utils_calendar.py +++ b/urbanaccess/gtfs/utils_calendar.py @@ -155,7 +155,7 @@ def _cal_date_dt_conversion(df, date_cols): for col in date_cols: try: df[col] = pd.to_datetime( - df[col], format='%y%m%d', infer_datetime_format=True) + df[col], format='mixed') except ValueError: raise ValueError("Column: {} has values that are not in a " "supported date format. Expected format: " diff --git a/urbanaccess/gtfsfeeds.py b/urbanaccess/gtfsfeeds.py index fafd9f8..96b255c 100644 --- a/urbanaccess/gtfsfeeds.py +++ b/urbanaccess/gtfsfeeds.py @@ -325,7 +325,7 @@ def search(api='gtfsdataexch', search_text=None, search_field=None, search_result = feed_table[ feed_table[col].str.match( text, case=False, na=False)] - search_result_df = search_result_df.append(search_result) + search_result_df = search_result_df._append(search_result) search_result_df.drop_duplicates(inplace=True) log('Found {} records that matched {} inside {} columns:'.format( diff --git a/urbanaccess/network.py b/urbanaccess/network.py index 58a858a..a91872a 100644 --- a/urbanaccess/network.py +++ b/urbanaccess/network.py @@ -183,8 +183,12 @@ def integrate_network(urbanaccess_network, headways=False, urbanaccess_network.net_connector_edges], axis=0) urbanaccess_network.net_nodes = pd.concat( - [urbanaccess_network.transit_nodes, - urbanaccess_network.osm_nodes], axis=0) + [ + urbanaccess_network.transit_nodes, + urbanaccess_network.osm_nodes.assign(id=urbanaccess_network.osm_nodes.index), + ], + axis=0, + ) urbanaccess_network.net_edges, urbanaccess_network.net_nodes = \ _format_pandana_edges_nodes(edge_df=urbanaccess_network.net_edges, @@ -311,7 +315,7 @@ def _route_id_to_node(stops_df, edges_w_routes): log('routes successfully joined to transit nodes. ' 'Took {:,.2f} seconds'.format(time.time() - start_time)) - return transit_nodes_wroutes + return transit_nodes_wroutes.dropna(subset=['x', 'y']) def _format_pandana_edges_nodes(edge_df, node_df): @@ -338,30 +342,36 @@ def _format_pandana_edges_nodes(edge_df, node_df): # Pandana requires IDs that are integer: for nodes - make it the index, # for edges make it the from and to columns node_df['id_int'] = range(1, len(node_df) + 1) + node_df['id_int'] = node_df.id_int.astype(int) # TODO: reconsider the use of inplace or copy incoming dfs # to memory to avoid changing in memory tables edge_df.rename(columns={'id': 'edge_id'}, inplace=True) - tmp = pd.merge(edge_df, node_df[['id', 'id_int']], left_on='from', - right_on='id', sort=False, copy=False, how='left') - tmp['from_int'] = tmp['id_int'] - tmp.drop(['id_int', 'id'], axis=1, inplace=True) - edge_df_wnumericid = pd.merge(tmp, node_df[['id', 'id_int']], left_on='to', - right_on='id', sort=False, copy=False, - how='left') - edge_df_wnumericid['to_int'] = edge_df_wnumericid['id_int'] - edge_df_wnumericid.drop(['id_int', 'id'], axis=1, inplace=True) + id_to_int = node_df.set_index('id')['id_int'].astype(int) + edge_df = edge_df.merge(id_to_int.rename('from_int').astype(int), left_on='from', + right_index=True, how='left') + + #tmp = pd.merge(edge_df, node_df[['id', 'id_int']], left_on='from', + # right_on='id', sort=False, copy=False, how='left') + #tmp['from_int'] = tmp['id_int'] + #tmp.drop(['id_int', 'id'], axis=1, inplace=True) + edge_df = edge_df.merge(id_to_int.rename('to_int').astype(int), left_on='to', right_index=True, how='left') + + # edge_df_wnumericid = pd.merge(tmp, node_df[['id', 'id_int']], left_on='to', + # right_on='id', sort=False, copy=False, + # how='left') + #edge_df_wnumericid['to_int'] = edge_df_wnumericid['id_int'] + #edge_df_wnumericid.drop(['id_int', 'id'], axis=1, inplace=True) # turn mixed dtype cols into all same format - col_list = edge_df_wnumericid.select_dtypes(include=['object']).columns + col_list = edge_df.select_dtypes(include=["object"]).columns for col in col_list: try: - edge_df_wnumericid[col] = edge_df_wnumericid[col].astype(str) + edge_df[col] = edge_df[col].astype(str) # deal with edge cases where typically the name of a street is not # in an uniform string encoding such as names with accents except UnicodeEncodeError: log('Fixed unicode error in {} column'.format(col)) - edge_df_wnumericid[col] = edge_df_wnumericid[col].str.encode( - 'utf-8') + edge_df[col] = edge_df[col].str.encode("utf-8") node_df.set_index('id_int', drop=True, inplace=True) # turn mixed dtype col into all same format @@ -372,7 +382,7 @@ def _format_pandana_edges_nodes(edge_df, node_df): log('Edge and node tables formatted for Pandana with integer node IDs: ' 'id_int, to_int, and from_int. Took {:,.2f} seconds'.format( time.time() - start_time)) - return edge_df_wnumericid, node_df + return edge_df, node_df.dropna(subset=["x", "y"]) def save_network(urbanaccess_network, filename, diff --git a/urbanaccess/tests/test_gtfs_utils_calendar.py b/urbanaccess/tests/test_gtfs_utils_calendar.py index 302eb94..19c1248 100644 --- a/urbanaccess/tests/test_gtfs_utils_calendar.py +++ b/urbanaccess/tests/test_gtfs_utils_calendar.py @@ -159,8 +159,7 @@ def calendar_agency_a_datetime(calendar_agency_a): cols = ['start_date', 'end_date'] for col in cols: calendar_agency_a[col] = pd.to_datetime( - calendar_agency_a[col], format='%y%m%d', - infer_datetime_format=True) + calendar_agency_a[col], format='mixed') return calendar_agency_a @@ -168,8 +167,7 @@ def calendar_agency_a_datetime(calendar_agency_a): def calendar_dates_agency_a_datetime(calendar_dates_agency_a): col = 'date' calendar_dates_agency_a[col] = pd.to_datetime( - calendar_dates_agency_a[col], format='%y%m%d', - infer_datetime_format=True) + calendar_dates_agency_a[col], format='mixed') return calendar_dates_agency_a From ab506243f9351fdd8fb4ab770c832e82390dfcf3 Mon Sep 17 00:00:00 2001 From: knaaptime Date: Fri, 7 Feb 2025 15:09:08 -0800 Subject: [PATCH 72/72] use concat instead of private append --- urbanaccess/gtfs/load.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/urbanaccess/gtfs/load.py b/urbanaccess/gtfs/load.py index d18d063..ee2951a 100644 --- a/urbanaccess/gtfs/load.py +++ b/urbanaccess/gtfs/load.py @@ -398,20 +398,20 @@ def gtfsfeed_to_df(gtfsfeed_path=None, validation=False, verbose=True, trips_df=trips_df[['trip_id', 'route_id']], info_to_append='route_type_to_stop_times') - merged_stops_df = merged_stops_df._append( - stops_df, ignore_index=True) - merged_routes_df = merged_routes_df._append( - routes_df, ignore_index=True) - merged_trips_df = merged_trips_df._append( - trips_df, ignore_index=True) - merged_stop_times_df = merged_stop_times_df._append( - stop_times_df, ignore_index=True) - merged_shapes_df = merged_shapes_df._append( - shapes_df, ignore_index=True) - merged_calendar_df = merged_calendar_df._append( - calendar_df, ignore_index=True) - merged_calendar_dates_df = merged_calendar_dates_df._append( - calendar_dates_df, ignore_index=True) + merged_stops_df = pd.concat([merged_stops_df, + stops_df], ignore_index=True) + merged_routes_df = pd.concat([merged_routes_df, + routes_df], ignore_index=True) + merged_trips_df = pd.concat([merged_trips_df, + trips_df], ignore_index=True) + merged_stop_times_df = pd.concat([merged_stop_times_df, + stop_times_df], ignore_index=True) + merged_shapes_df = pd.concat([merged_shapes_df, + shapes_df], ignore_index=True) + merged_calendar_df = pd.concat([merged_calendar_df, + calendar_df], ignore_index=True) + merged_calendar_dates_df = pd.concat([merged_calendar_dates_df, + calendar_dates_df], ignore_index=True) # print break to visually separate each GTFS feed log log('--------------------------------')