From 1af465a6a04446d34ec295013dee738e745b7d29 Mon Sep 17 00:00:00 2001 From: Paul Armstrong Date: Wed, 11 Dec 2024 11:19:56 +1100 Subject: [PATCH] Python 3 and Cylc 8 conversion * Ran the Python 2 to 3 converter over the code * Fixed a number of py2 -> py3 issues around dividing two ints and getting an int vs getting a float in return * Added significant logging to assist with debugging * Put in some handling for getting None back and attempting to do a string comparison on it. * urllib.pathname2url -> urllib.request.pathname2url * Remove user_at_host from Rose Bush DAO as it's not in Cylc 8 * Nullified host, submit_method and submit_method_id as not in Cylc8 * Added exception handling to SQLite3 calls in RoseBushDAO * Coerced a lot of lists with list() --- bin/rose-app-run | 2 +- bin/rose-app-upgrade | 2 +- bin/rose-bush | 2 +- bin/rose-config | 2 +- bin/rose-config-diff | 2 +- bin/rose-config-dump | 2 +- bin/rose-config-edit | 2 +- bin/rose-date | 2 +- bin/rose-env-cat | 2 +- bin/rose-host-select | 2 +- bin/rose-macro | 2 +- bin/rose-metadata-check | 2 +- bin/rose-metadata-gen | 2 +- bin/rose-metadata-graph | 2 +- bin/rose-namelist-dump | 2 +- bin/rose-stem | 2 +- bin/rose-suite-clean | 2 +- bin/rose-suite-gcontrol | 2 +- bin/rose-suite-hook | 2 +- bin/rose-suite-log | 2 +- bin/rose-suite-restart | 2 +- bin/rose-suite-run | 2 +- bin/rose-suite-scan | 2 +- bin/rose-suite-shutdown | 2 +- bin/rose-task-env | 2 +- bin/rose-task-run | 2 +- bin/rose-test-battery | 4 +- bin/rosie | 171 +++++++++++++++++- bin/rosie-checkout | 2 +- bin/rosie-create | 2 +- bin/rosie-delete | 2 +- bin/rosie-disco | 2 +- bin/rosie-go | 2 +- bin/rosie-graph | 2 +- bin/rosie-hello | 2 +- bin/rosie-id | 2 +- bin/rosie-lookup | 2 +- bin/rosie-ls | 2 +- lib/python/isodatetime/data.py | 24 +-- lib/python/isodatetime/parser_spec.py | 2 +- lib/python/isodatetime/parsers.py | 22 +-- lib/python/isodatetime/tests.py | 108 +++++------ lib/python/rose/app_run.py | 2 +- lib/python/rose/apps/ana_builtin/grepper.py | 2 +- lib/python/rose/apps/rose_ana.py | 5 +- lib/python/rose/apps/rose_ana_v1.py | 2 +- lib/python/rose/apps/rose_arch.py | 12 +- .../rose_arch_compressions/rose_arch_gzip.py | 2 +- .../rose_arch_compressions/rose_arch_tar.py | 2 +- lib/python/rose/apps/rose_bunch.py | 10 +- lib/python/rose/apps/rose_prune.py | 4 +- lib/python/rose/bush.py | 98 ++++++---- lib/python/rose/bush_dao.py | 54 ++++-- lib/python/rose/c3.py | 6 +- lib/python/rose/config.py | 39 ++-- lib/python/rose/config_cli.py | 8 +- lib/python/rose/config_diff.py | 6 +- lib/python/rose/config_editor/__init__.py | 2 +- lib/python/rose/config_editor/data.py | 70 +++---- lib/python/rose/config_editor/data_helper.py | 22 +-- lib/python/rose/config_editor/keywidget.py | 2 +- lib/python/rose/config_editor/main.py | 34 ++-- lib/python/rose/config_editor/menu.py | 30 +-- lib/python/rose/config_editor/menuwidget.py | 10 +- .../rose/config_editor/nav_controller.py | 2 +- lib/python/rose/config_editor/nav_panel.py | 4 +- .../rose/config_editor/nav_panel_menu.py | 18 +- lib/python/rose/config_editor/ops/group.py | 6 +- lib/python/rose/config_editor/ops/section.py | 4 +- lib/python/rose/config_editor/ops/variable.py | 6 +- lib/python/rose/config_editor/page.py | 12 +- .../rose/config_editor/pagewidget/__init__.py | 2 +- .../config_editor/panelwidget/__init__.py | 4 +- .../config_editor/panelwidget/summary_data.py | 14 +- .../config_editor/plugin/um/widget/stash.py | 38 ++-- .../plugin/um/widget/stash_add.py | 16 +- lib/python/rose/config_editor/stack.py | 2 +- lib/python/rose/config_editor/updater.py | 36 ++-- .../rose/config_editor/upgrade_controller.py | 2 +- lib/python/rose/config_editor/util.py | 6 +- .../config_editor/valuewidget/__init__.py | 28 +-- .../valuewidget/array/logical.py | 4 +- .../config_editor/valuewidget/array/mixed.py | 4 +- .../valuewidget/array/python_list.py | 2 +- .../config_editor/valuewidget/array/row.py | 4 +- .../config_editor/valuewidget/boolradio.py | 4 +- .../config_editor/valuewidget/booltoggle.py | 12 +- .../rose/config_editor/valuewidget/choice.py | 2 +- .../rose/config_editor/valuewidget/intspin.py | 4 +- lib/python/rose/config_editor/variable.py | 6 +- lib/python/rose/config_editor/window.py | 2 +- lib/python/rose/config_processors/env.py | 2 +- .../rose/config_processors/fileinstall.py | 14 +- lib/python/rose/config_tree.py | 10 +- lib/python/rose/date.py | 18 +- lib/python/rose/formats/__init__.py | 2 +- lib/python/rose/gtk/choice.py | 2 +- lib/python/rose/gtk/dialog.py | 4 +- lib/python/rose/gtk/splash.py | 2 +- lib/python/rose/gtk/util.py | 10 +- lib/python/rose/host_select.py | 6 +- lib/python/rose/job_runner.py | 10 +- lib/python/rose/loc_handlers/namelist.py | 4 +- lib/python/rose/loc_handlers/svn.py | 4 +- lib/python/rose/macro.py | 44 ++--- lib/python/rose/macros/__init__.py | 12 +- lib/python/rose/macros/compulsory.py | 4 +- lib/python/rose/macros/duplicate.py | 4 +- lib/python/rose/macros/rule.py | 10 +- lib/python/rose/macros/trigger.py | 36 ++-- lib/python/rose/macros/value.py | 10 +- lib/python/rose/meta_type.py | 2 +- lib/python/rose/metadata_check.py | 6 +- lib/python/rose/metadata_gen.py | 4 +- lib/python/rose/metadata_graph.py | 8 +- lib/python/rose/namelist_dump.py | 4 +- lib/python/rose/reporter.py | 10 +- lib/python/rose/run.py | 6 +- lib/python/rose/scheme_handler.py | 2 +- lib/python/rose/section.py | 6 +- lib/python/rose/stem.py | 8 +- lib/python/rose/suite_clean.py | 4 +- lib/python/rose/suite_control.py | 2 +- lib/python/rose/suite_engine_proc.py | 8 +- lib/python/rose/suite_engine_procs/cylc.py | 18 +- lib/python/rose/suite_log.py | 2 +- lib/python/rose/suite_run.py | 4 +- lib/python/rose/task_env.py | 12 +- lib/python/rose/task_run.py | 2 +- lib/python/rose/upgrade.py | 6 +- lib/python/rose/variable.py | 8 +- lib/python/rose/ws.py | 2 +- lib/python/rosie/browser/history.py | 4 +- lib/python/rosie/browser/main.py | 6 +- lib/python/rosie/browser/result.py | 2 +- lib/python/rosie/browser/status.py | 4 +- lib/python/rosie/browser/util.py | 4 +- lib/python/rosie/suite_id.py | 6 +- lib/python/rosie/svn_post_commit.py | 14 +- lib/python/rosie/svn_pre_commit.py | 2 +- lib/python/rosie/vc.py | 12 +- lib/python/rosie/ws.py | 4 +- lib/python/rosie/ws_client.py | 14 +- lib/python/rosie/ws_client_auth.py | 4 +- lib/python/rosie/ws_client_cli.py | 10 +- sbin/rosa | 171 +++++++++++++++++- sbin/rosa-db-create | 2 +- sbin/rosa-rpmbuild | 2 +- sbin/rosa-svn-post-commit | 2 +- sbin/rosa-svn-pre-commit | 2 +- sbin/rosa-ws | 2 +- 151 files changed, 1038 insertions(+), 630 deletions(-) mode change 120000 => 100755 bin/rosie mode change 120000 => 100755 sbin/rosa diff --git a/bin/rose-app-run b/bin/rose-app-run index 9d0dbec436..a39e4305dc 100755 --- a/bin/rose-app-run +++ b/bin/rose-app-run @@ -79,4 +79,4 @@ # optional ROSE_FILE_INSTALL_ROOT # If specified, change to the specified directory to install files. #------------------------------------------------------------------------------- -exec python -m rose.app_run "$@" +exec python3 -m rose.app_run "$@" diff --git a/bin/rose-app-upgrade b/bin/rose-app-upgrade index d09f23176d..66b86576dc 100755 --- a/bin/rose-app-upgrade +++ b/bin/rose-app-upgrade @@ -47,4 +47,4 @@ # optional ROSE_META_PATH # Prepend $ROSE_META_PATH to the metadata search path. #------------------------------------------------------------------------------- -exec python -m rose.upgrade "$@" +exec python3 -m rose.upgrade "$@" diff --git a/bin/rose-bush b/bin/rose-bush index 6578287928..e73212a0d4 100755 --- a/bin/rose-bush +++ b/bin/rose-bush @@ -41,4 +41,4 @@ # --service-root, -R # (For start only.) Include web service name under root of URL. #------------------------------------------------------------------------------- -exec python -m rose.bush "$@" +exec python3 -m rose.bush "$@" diff --git a/bin/rose-config b/bin/rose-config index 42af4df817..ec9e2caa01 100755 --- a/bin/rose-config +++ b/bin/rose-config @@ -99,4 +99,4 @@ # optional ROSE_META_PATH # Prepend $ROSE_META_PATH to the metadata search path. #------------------------------------------------------------------------------- -exec python -m rose.config_cli "$@" +exec python3 -m rose.config_cli "$@" diff --git a/bin/rose-config-diff b/bin/rose-config-diff index 004e71f36b..7bc2334018 100755 --- a/bin/rose-config-diff +++ b/bin/rose-config-diff @@ -104,4 +104,4 @@ # instead of: # rose config-diff --ignore=namelist:bar --ignore=namelist:baz ... #------------------------------------------------------------------------------- -exec python -m rose.config_diff "$@" +exec python3 -m rose.config_diff "$@" diff --git a/bin/rose-config-dump b/bin/rose-config-dump index d4d23ed713..54382ff74e 100755 --- a/bin/rose-config-dump +++ b/bin/rose-config-dump @@ -47,4 +47,4 @@ # --quiet, -q # Decrement verbosity. Do not report modified files. #------------------------------------------------------------------------------- -exec python -m rose.config_dump "$@" +exec python3 -m rose.config_dump "$@" diff --git a/bin/rose-config-edit b/bin/rose-config-edit index 3059d3ee3f..6ce1a740f5 100755 --- a/bin/rose-config-edit +++ b/bin/rose-config-edit @@ -69,4 +69,4 @@ # optional ROSE_META_PATH # Prepend $ROSE_META_PATH to the metadata search path. #------------------------------------------------------------------------------- -exec python -m rose.config_editor.main "$@" +exec python3 -m rose.config_editor.main "$@" diff --git a/bin/rose-date b/bin/rose-date index 3ff6fa6abe..7114ec3232 100755 --- a/bin/rose-date +++ b/bin/rose-date @@ -169,4 +169,4 @@ # For example, for a duration P57DT12H: # 'y,m,d,h' -> '0,0,57,12' #------------------------------------------------------------------------------- -exec python -m rose.date "$@" +exec python3 -m rose.date "$@" diff --git a/bin/rose-env-cat b/bin/rose-env-cat index 6e037049f2..d65c03ed75 100755 --- a/bin/rose-env-cat +++ b/bin/rose-env-cat @@ -48,4 +48,4 @@ # variable with the value of STRING, (which can be an empty string), # instead of failing. #------------------------------------------------------------------------------- -exec python -m rose.env_cat "$@" +exec python3 -m rose.env_cat "$@" diff --git a/bin/rose-host-select b/bin/rose-host-select index 14ce2d0277..8273f5568c 100755 --- a/bin/rose-host-select +++ b/bin/rose-host-select @@ -83,4 +83,4 @@ # # Type "rose config rose-host-select" to print settings. #------------------------------------------------------------------------------- -exec python -m rose.host_select "$@" +exec python3 -m rose.host_select "$@" diff --git a/bin/rose-macro b/bin/rose-macro index 6de3ec2940..b9ef9065c0 100755 --- a/bin/rose-macro +++ b/bin/rose-macro @@ -74,4 +74,4 @@ # optional ROSE_META_PATH # Prepend $ROSE_META_PATH to the metadata search path. #------------------------------------------------------------------------------- -exec python -m rose.macro "$@" +exec python3 -m rose.macro "$@" diff --git a/bin/rose-metadata-check b/bin/rose-metadata-check index 9c952da0e5..b5619a4839 100755 --- a/bin/rose-metadata-check +++ b/bin/rose-metadata-check @@ -40,4 +40,4 @@ # metadata e.g.env=FOO or namelist:bar. If specified, only # this section will be checked. #------------------------------------------------------------------------------- -exec python -m rose.metadata_check "$@" +exec python3 -m rose.metadata_check "$@" diff --git a/bin/rose-metadata-gen b/bin/rose-metadata-gen index 3d275e94a0..1573e03cf8 100755 --- a/bin/rose-metadata-gen +++ b/bin/rose-metadata-gen @@ -45,4 +45,4 @@ # every setting e.g. "compulsory=true". If =VALUE is missing, # the property will be set to a null string in each setting. #------------------------------------------------------------------------------- -exec python -m rose.metadata_gen "$@" +exec python3 -m rose.metadata_gen "$@" diff --git a/bin/rose-metadata-graph b/bin/rose-metadata-graph index b61a5a29b3..76f5e10b8b 100755 --- a/bin/rose-metadata-graph +++ b/bin/rose-metadata-graph @@ -50,4 +50,4 @@ # optional ROSE_META_PATH # Prepend $ROSE_META_PATH to the metadata search path. #------------------------------------------------------------------------------- -exec python -m rose.metadata_graph "$@" +exec python3 -m rose.metadata_graph "$@" diff --git a/bin/rose-namelist-dump b/bin/rose-namelist-dump index d39b7ba89e..e111aae095 100755 --- a/bin/rose-namelist-dump +++ b/bin/rose-namelist-dump @@ -44,4 +44,4 @@ # -u, --upper # Shorthand for --case=upper. #------------------------------------------------------------------------------- -exec python -m rose.namelist_dump "$@" +exec python3 -m rose.namelist_dump "$@" diff --git a/bin/rose-stem b/bin/rose-stem index 9dae52b658..647476fc28 100755 --- a/bin/rose-stem +++ b/bin/rose-stem @@ -82,4 +82,4 @@ # is intended to specify the revision of 'fcm-make' config files. # #------------------------------------------------------------------------------- -exec python -m rose.stem "$@" +exec python3 -m rose.stem "$@" diff --git a/bin/rose-suite-clean b/bin/rose-suite-clean index 3dcee633a0..8bbeb93ad0 100755 --- a/bin/rose-suite-clean +++ b/bin/rose-suite-clean @@ -50,4 +50,4 @@ # Return the difference between the number of arguments and number of # successfully cleaned suites, i.e. 0 if all successful. #------------------------------------------------------------------------------- -exec python -m rose.suite_clean "$@" +exec python3 -m rose.suite_clean "$@" diff --git a/bin/rose-suite-gcontrol b/bin/rose-suite-gcontrol index 66e9da8476..2e7d810d6b 100755 --- a/bin/rose-suite-gcontrol +++ b/bin/rose-suite-gcontrol @@ -50,4 +50,4 @@ # --verbose, -v # Increment verbosity. #------------------------------------------------------------------------------- -exec python -m rose.suite_control gcontrol "$@" +exec python3 -m rose.suite_control gcontrol "$@" diff --git a/bin/rose-suite-hook b/bin/rose-suite-hook index 09131e02d5..125a63b497 100755 --- a/bin/rose-suite-hook +++ b/bin/rose-suite-hook @@ -50,4 +50,4 @@ # --verbose, -v # Increment verbosity. #------------------------------------------------------------------------------- -exec python -m rose.suite_hook "$@" +exec python3 -m rose.suite_hook "$@" diff --git a/bin/rose-suite-log b/bin/rose-suite-log index 2e5cbbe346..9ebd9ce518 100755 --- a/bin/rose-suite-log +++ b/bin/rose-suite-log @@ -64,4 +64,4 @@ # --view # Launch web browser to view suite log. #------------------------------------------------------------------------------- -exec python -m rose.suite_log "$@" +exec python3 -m rose.suite_log "$@" diff --git a/bin/rose-suite-restart b/bin/rose-suite-restart index 1bd2d7b921..df2da10161 100755 --- a/bin/rose-suite-restart +++ b/bin/rose-suite-restart @@ -58,4 +58,4 @@ # --verbose, -v # Increment verbosity. #------------------------------------------------------------------------------- -exec python -m rose.suite_restart "$@" +exec python3 -m rose.suite_restart "$@" diff --git a/bin/rose-suite-run b/bin/rose-suite-run index 0c933d8579..74080e7961 100755 --- a/bin/rose-suite-run +++ b/bin/rose-suite-run @@ -123,4 +123,4 @@ # "cylc help register" # "cylc help run" #------------------------------------------------------------------------------- -exec python -m rose.suite_run "$@" +exec python3 -m rose.suite_run "$@" diff --git a/bin/rose-suite-scan b/bin/rose-suite-scan index f7e5f05dfb..c564271698 100755 --- a/bin/rose-suite-scan +++ b/bin/rose-suite-scan @@ -35,4 +35,4 @@ # --verbose, -v # Increment verbosity. #------------------------------------------------------------------------------- -exec python -m rose.suite_scan "$@" +exec python3 -m rose.suite_scan "$@" diff --git a/bin/rose-suite-shutdown b/bin/rose-suite-shutdown index 14227e9dd5..7e3929ef59 100755 --- a/bin/rose-suite-shutdown +++ b/bin/rose-suite-shutdown @@ -56,4 +56,4 @@ # See the cylc documentation, cylc shutdown for options and details on # EXTRA-ARGS. #------------------------------------------------------------------------------- -exec python -m rose.suite_control shutdown "$@" +exec python3 -m rose.suite_control shutdown "$@" diff --git a/bin/rose-task-env b/bin/rose-task-env index 8a7129753c..8841f7953c 100755 --- a/bin/rose-task-env +++ b/bin/rose-task-env @@ -126,4 +126,4 @@ # env-script = "eval $(rose task-env)" # #------------------------------------------------------------------------------- -exec python -m rose.task_env "$@" +exec python3 -m rose.task_env "$@" diff --git a/bin/rose-task-run b/bin/rose-task-run index 0f1d22d7c7..310dc1e086 100755 --- a/bin/rose-task-run +++ b/bin/rose-task-run @@ -56,4 +56,4 @@ # rose app-run # rose task-env #------------------------------------------------------------------------------- -exec python -m rose.task_run "$@" +exec python3 -m rose.task_run "$@" diff --git a/bin/rose-test-battery b/bin/rose-test-battery index cd5a977646..8688dddd98 100755 --- a/bin/rose-test-battery +++ b/bin/rose-test-battery @@ -61,7 +61,7 @@ fi # Recompile *.pyc files to ensure we are running the current code. if [[ -w 'lib/python/' ]]; then find 'lib/python/' -name '*.pyc' -type 'f' -delete - python -mcompileall -q 'lib/python/' + python3 -mcompileall -q 'lib/python/' fi if PROVE_OPTIONS=$(rose config t prove-options); then exec prove $PROVE_OPTIONS -r "${@:-t}" @@ -69,7 +69,7 @@ else if [[ -f /proc/cpuinfo ]]; then NPROC=$(grep -ic processor /proc/cpuinfo) else - NPROC=$(python -c \ + NPROC=$(python3 -c \ 'import multiprocessing; print multiprocessing.cpu_count()') fi exec prove -j "$NPROC" -s -r "${@:-t}" diff --git a/bin/rosie b/bin/rosie deleted file mode 120000 index 426fcadcae..0000000000 --- a/bin/rosie +++ /dev/null @@ -1 +0,0 @@ -rose \ No newline at end of file diff --git a/bin/rosie b/bin/rosie new file mode 100755 index 0000000000..2bdceca2e5 --- /dev/null +++ b/bin/rosie @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------- +# (C) British Crown Copyright 2012-7 Met Office. +# +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +#------------------------------------------------------------------------------- +# NAME +# rose - environment for running scientific suites and applications +# rosie - revision/organisation of suites for identification/exploration +# rosa - rosie's admin utilities +# +# SYNOPSIS +# # "rose", "rosie" and "rosa" has the following interfaces: +# rose UTIL [ARGS ...] # Invoke a utility +# rose help # Print help and list available utilities +# rose help UTIL ... # Print help for UTIL ... +# rose version # Print version information +# +# DESCRIPTION +# Simple launcher for utilities in the "rose", "rosie" or "rosa" +# name-spaces. Launch a "rose", "rosie" or "rosa" utility with the given +# ARGS. +#------------------------------------------------------------------------------- +if ${ROSE_DEBUG:-false}; then + set -x +fi +. $(dirname $0)/../lib/bash/rose_init +rose_init + +# Print actual command of a command alias +get_alias() { + local NAME=$1 + local ALIAS=$(sed '/^#/d' $ROSE_HOME_BIN/$ROSE_NS-$NAME || true) + if [[ $(wc -l <<<"$ALIAS") == 1 ]] \ + && grep -q "^exec \$(dirname \$0)/$ROSE_NS-.* \"\$@\"\$" <<<"$ALIAS" + then + ALIAS=${ALIAS#"exec \$(dirname \$0)/$ROSE_NS-"} + ALIAS=${ALIAS%' "$@"'} + echo $ALIAS + fi +} + +# Print help for a given utility +help_util() { + local NAME=$1 + local COMMAND=$ROSE_HOME_BIN/$ROSE_NS-$NAME + if [[ ! -r $COMMAND ]]; then + echo "$1: utility not found." >&2 + return 1 + fi + local ALIAS=$(get_alias $NAME) + if [[ -n $ALIAS ]]; then + COMMAND=$ROSE_HOME_BIN/$ROSE_NS-$ALIAS + COMMAND=${COMMAND%% *} + fi + case $(head -1 -- $COMMAND) in + *bash*) + awk '{ + if (/^# NAME/) { + do {print substr($0, 3)} while (getline && !/^#----------/); + } + }' $COMMAND | ${PAGER:-less} + ;; + *python*) + $COMMAND --help | ${PAGER:-less} # FIXME: not too pretty at the moment + ;; + esac + return +} + +# Ensure that ITEM_STR is at the beginning of PATH_STR +path_lead() { + local PATH_STR=$1 + local ITEM_STR=$2 + if [[ -z ${PATH_STR:-} ]]; then + echo "$ITEM_STR" + elif [[ "$PATH_STR" != "$ITEM_STR" && "$PATH_STR" != $ITEM_STR:* ]]; then + while [[ "$PATH_STR" == *:$ITEM_STR ]]; do + PATH_STR=${PATH_STR%:$ITEM_STR} + done + while [[ "$PATH_STR" == *:$ITEM_STR:* ]]; do + local PATH_HEAD=${PATH_STR%:$ITEM_STR:*} + local PATH_TAIL=${PATH_STR##*:$ITEM_STR:} + PATH_STR="$PATH_HEAD:$PATH_TAIL" + done + echo "$ITEM_STR:$PATH_STR" + else + echo "$PATH_STR" + fi +} + + +# Print Rose version +function print_version() { + echo "Rose $ROSE_VERSION ($ROSE_HOME)" +} + +#------------------------------------------------------------------------------- +UTIL="help" +if (($# > 0)); then + UTIL=$1 + shift 1 +fi + +case $UTIL in +help|h|?|--help|-h) + if (($# == 0)); then + { + print_version + rose_usage + echo + echo "$ROSE_NS provides the following utilities:" + for U in $(cd $ROSE_HOME_BIN && ls $ROSE_NS-*); do + NAME=$(sed "s/^$ROSE_NS-\\(.*\\)\$/\1/" <<<$U) + ALIAS=$(get_alias $NAME) + if [[ -n $ALIAS ]]; then + echo " $NAME" + echo " (=$ALIAS)" + else + echo " $NAME" + sed '1,/^# DESCRIPTION$/d;{s/^# / /;q;}' \ + $ROSE_HOME_BIN/$U + fi + done + } | ${PAGER:-less} + exit 0 + fi + RC=0 + for U in "$@"; do + if [[ $U == 'help' || $U == 'version' ]]; then + continue + fi + help_util $U || RC=$? + done + exit $RC + :;; +version|--version|-V) + print_version + exit + :;; +esac + +COMMAND=$(dirname $0)/$ROSE_NS-$UTIL +if [[ ! -f $COMMAND || ! -x $COMMAND ]]; then + echo "$ROSE_NS: $UTIL: unknown utility. Abort." >&2 + echo "Type \"$ROSE_NS help\" for a list of utilities." >&2 + exit 1 +fi +if (($# > 0)) && [[ $1 == '--help' || $1 == '-h' ]]; then + help_util $UTIL + exit +fi +PATH=$(path_lead "${PATH:-}" "$ROSE_HOME_BIN") +PYTHONPATH=$(path_lead "${PYTHONPATH:-}" "$ROSE_HOME/lib/python") +ROSE_UTIL=$UTIL +export PATH PYTHONPATH ROSE_UTIL +exec $COMMAND "$@" diff --git a/bin/rosie-checkout b/bin/rosie-checkout index 1822762a52..be432e4ce8 100755 --- a/bin/rosie-checkout +++ b/bin/rosie-checkout @@ -38,4 +38,4 @@ # --verbose, -v # Increment verbosity. #------------------------------------------------------------------------------- -exec python -m rosie.vc checkout "$@" +exec python3 -m rosie.vc checkout "$@" diff --git a/bin/rosie-create b/bin/rosie-create index a310360b1e..d1a3b34110 100755 --- a/bin/rosie-create +++ b/bin/rosie-create @@ -69,4 +69,4 @@ # --verbose, -v # Increment verbosity. #------------------------------------------------------------------------------- -exec python -m rosie.vc create "$@" +exec python3 -m rosie.vc create "$@" diff --git a/bin/rosie-delete b/bin/rosie-delete index 111c233dcc..15b1ca74f7 100755 --- a/bin/rosie-delete +++ b/bin/rosie-delete @@ -49,4 +49,4 @@ # --verbose, -v # Increment verbosity. #------------------------------------------------------------------------------- -exec python -m rosie.vc delete "$@" +exec python3 -m rosie.vc delete "$@" diff --git a/bin/rosie-disco b/bin/rosie-disco index 3157c4bf6b..e03b9a6d2f 100755 --- a/bin/rosie-disco +++ b/bin/rosie-disco @@ -38,4 +38,4 @@ # --service-root, -R # (For start only.) Include web service name under root of URL. #------------------------------------------------------------------------------- -exec python -m rosie.ws "$@" +exec python3 -m rosie.ws "$@" diff --git a/bin/rosie-go b/bin/rosie-go index 4f125acc20..b463bfd954 100755 --- a/bin/rosie-go +++ b/bin/rosie-go @@ -65,5 +65,5 @@ # rosie lookup # rosie ls #------------------------------------------------------------------------------- -exec python -m rosie.browser.main "$@" +exec python3 -m rosie.browser.main "$@" diff --git a/bin/rosie-graph b/bin/rosie-graph index 31b2f772ac..3f4c0cc202 100755 --- a/bin/rosie-graph +++ b/bin/rosie-graph @@ -60,4 +60,4 @@ # A suite id to graph. If given, only the suites that are # connected to this id by copy history will be graphed. #------------------------------------------------------------------------------- -exec python -m rosie.graph "$@" +exec python3 -m rosie.graph "$@" diff --git a/bin/rosie-hello b/bin/rosie-hello index 33279130be..90598f766c 100755 --- a/bin/rosie-hello +++ b/bin/rosie-hello @@ -31,4 +31,4 @@ # Specify the name of one or more Rosie web service servers to use. # This option can be used multiple times. #------------------------------------------------------------------------------- -exec python -m rosie.ws_client_cli hello "$@" +exec python3 -m rosie.ws_client_cli hello "$@" diff --git a/bin/rosie-id b/bin/rosie-id index dfd3a99084..deb5133add 100755 --- a/bin/rosie-id +++ b/bin/rosie-id @@ -70,4 +70,4 @@ # --next # Print the next available suite ID in the repository #------------------------------------------------------------------------------- -exec python -m rosie.suite_id "$@" +exec python3 -m rosie.suite_id "$@" diff --git a/bin/rosie-lookup b/bin/rosie-lookup index b0863b1f0a..5fcc8e1c9e 100755 --- a/bin/rosie-lookup +++ b/bin/rosie-lookup @@ -82,4 +82,4 @@ # --verbose, -v # Display full info for each returned suite. #------------------------------------------------------------------------------- -exec python -m rosie.ws_client_cli lookup "$@" +exec python3 -m rosie.ws_client_cli lookup "$@" diff --git a/bin/rosie-ls b/bin/rosie-ls index c98bbddba8..179649d375 100755 --- a/bin/rosie-ls +++ b/bin/rosie-ls @@ -61,4 +61,4 @@ # --verbose, -v # Display full info for each returned suite. #------------------------------------------------------------------------------- -exec python -m rosie.ws_client_cli list_local_suites "$@" +exec python3 -m rosie.ws_client_cli list_local_suites "$@" diff --git a/lib/python/isodatetime/data.py b/lib/python/isodatetime/data.py index 57122fbb92..32fe30a883 100644 --- a/lib/python/isodatetime/data.py +++ b/lib/python/isodatetime/data.py @@ -557,7 +557,7 @@ def __cmp__(self, other): other_data = other.get_days_and_seconds() return cmp(my_data, other_data) - def __nonzero__(self): + def __bool__(self): for attr in ["years", "months", "weeks", "days", "hours", "minutes", "seconds"]: if getattr(self, attr, None): @@ -768,12 +768,12 @@ def __init__(self, expanded_year_digits=0, year=None, month_of_year=None, (time_zone_minute, "time_zone_minute", None, int) ) if (dump_format is not None and not - isinstance(dump_format, basestring)): + isinstance(dump_format, str)): raise BadInputError( BadInputError.TYPE, "dump_format", repr(dump_format), type(dump_format)) if (truncated_dump_format is not None and not - isinstance(truncated_dump_format, basestring)): + isinstance(truncated_dump_format, str)): raise BadInputError( BadInputError.TYPE, "truncated_dump_format", repr(truncated_dump_format), @@ -1600,7 +1600,7 @@ def _get_dump_format(self): if self.time_zone.hours == 0 and self.time_zone.minutes == 0: time_string += "Z" else: - time_string += u"+hh:mm" + time_string += "+hh:mm" return date_string + time_string def _get_truncated_dump_format(self): @@ -1672,7 +1672,7 @@ def _get_truncated_dump_format(self): if self.time_zone.hours == 0 and self.time_zone.minutes == 0: time_string += "Z" else: - time_string += u"+hh:mm" + time_string += "+hh:mm" if date_string == "YY": date_string = "-YY" time_string = time_string.replace(":", "") @@ -1773,7 +1773,7 @@ def _get_weeks_in_year(year, calendar_mode=None): cal_year_next, cal_ord_days_next = get_ordinal_date_week_date_start( year + 1) diff_days = cal_ord_days_next - cal_ord_days - for intervening_year in xrange(cal_year, cal_year_next): + for intervening_year in range(cal_year, cal_year_next): diff_days += get_days_in_year(intervening_year) return diff_days / CALENDAR.DAYS_IN_WEEK @@ -2089,7 +2089,7 @@ def _iter_months_days(is_leap_year, month_of_year, day_of_month, if in_reverse: if month_of_year is None: for month_num, days in reversed(source): - day_range = range(days, 0, -1) + day_range = list(range(days, 0, -1)) for day in day_range: results.append((month_num, day)) else: @@ -2097,23 +2097,23 @@ def _iter_months_days(is_leap_year, month_of_year, day_of_month, if month_num > month_of_year: continue elif month_num == month_of_year and day_of_month is not None: - day_range = range(day_of_month, 0, -1) + day_range = list(range(day_of_month, 0, -1)) else: - day_range = range(days, 0, -1) + day_range = list(range(days, 0, -1)) for day in day_range: results.append((month_num, day)) else: if month_of_year is None: for month_num, days in source: - day_range = range(1, days + 1) + day_range = list(range(1, days + 1)) for day in day_range: results.append((month_num, day)) else: for month_num, days in source[month_of_year - 1:]: if month_num == month_of_year and day_of_month is not None: - day_range = range(day_of_month, days + 1) + day_range = list(range(day_of_month, days + 1)) else: - day_range = range(1, days + 1) + day_range = list(range(1, days + 1)) for day in day_range: results.append((month_num, day)) return results diff --git a/lib/python/isodatetime/parser_spec.py b/lib/python/isodatetime/parser_spec.py index 353dff0028..f09f46ab39 100644 --- a/lib/python/isodatetime/parser_spec.py +++ b/lib/python/isodatetime/parser_spec.py @@ -330,7 +330,7 @@ def _translate_strftime_token(strftime_token, dump_mode=False, get_time_zone_translate_info() ) attr_names = STRFTIME_TRANSLATE_INFO[strftime_token] - if isinstance(attr_names, basestring): + if isinstance(attr_names, str): if dump_mode: return attr_names, [] return re.escape(attr_names), [] diff --git a/lib/python/isodatetime/parsers.py b/lib/python/isodatetime/parsers.py index d630f2a224..abba50e342 100644 --- a/lib/python/isodatetime/parsers.py +++ b/lib/python/isodatetime/parsers.py @@ -173,7 +173,7 @@ def _generate_regexes(self): self._date_regex_map.setdefault(format_type, {}) self._time_regex_map.setdefault(format_type, {}) self._time_zone_regex_map.setdefault(format_type, []) - for date_key in date_map[format_type].keys(): + for date_key in list(date_map[format_type].keys()): self._date_regex_map[format_type].setdefault(date_key, []) regex_list = self._date_regex_map[format_type][date_key] for date_expr in self.get_expressions( @@ -181,7 +181,7 @@ def _generate_regexes(self): date_regex = self.parse_date_expression_to_regex( date_expr) regex_list.append([re.compile(date_regex), date_expr]) - for time_key in time_map[format_type].keys(): + for time_key in list(time_map[format_type].keys()): self._time_regex_map[format_type].setdefault(time_key, []) regex_list = self._time_regex_map[format_type][time_key] for time_expr in self.get_expressions( @@ -275,13 +275,13 @@ def _create_timepoint_from_info(self, date_info, time_info, if date_info.pop("year_sign", "+") == "-": year *= -1 date_info["year"] = year - for key, value in date_info.items(): + for key, value in list(date_info.items()): try: date_info[key] = int(value) except (TypeError, ValueError): pass info.update(date_info) - for key, value in time_info.items(): + for key, value in list(time_info.items()): if key.endswith("_decimal"): value = "0." + value try: @@ -344,7 +344,7 @@ def _parse_from_custom_regex(self, regex, data_string, dump_format=None, if not result: raise StrptimeConversionError(source, data_string) info = result.groupdict() - for property_, value in info.items(): + for property_, value in list(info.items()): if property_ in data.PARSE_PROPERTY_TRANSLATORS: info.pop(property_) translator = data.PARSE_PROPERTY_TRANSLATORS[property_] @@ -361,7 +361,7 @@ def _parse_from_custom_regex(self, regex, data_string, dump_format=None, date_info = {} time_info = {} time_zone_info = {} - for key, value in info.items(): + for key, value in list(info.items()): if key in date_info_keys: date_info[key] = value elif key in time_info_keys: @@ -381,7 +381,7 @@ def get_date_info(self, date_string, bad_types=None): type_keys.remove(type_key) if not self.allow_truncated and "truncated" in type_keys: type_keys.remove("truncated") - for format_key, type_regex_map in self._date_regex_map.items(): + for format_key, type_regex_map in list(self._date_regex_map.items()): for type_key in type_keys: regex_list = type_regex_map[type_key] for regex, expr in regex_list: @@ -397,10 +397,10 @@ def get_time_info(self, time_string, bad_formats=None, bad_types=None): bad_formats = [] if bad_types is None: bad_types = [] - for format_key, type_regex_map in self._time_regex_map.items(): + for format_key, type_regex_map in list(self._time_regex_map.items()): if format_key in bad_formats: continue - for type_key, regex_list in type_regex_map.items(): + for type_key, regex_list in list(type_regex_map.items()): if type_key in bad_types: continue for regex, expr in regex_list: @@ -413,7 +413,7 @@ def get_time_zone_info(self, time_zone_string, bad_formats=None): """Return the properties from a time zone string.""" if bad_formats is None: bad_formats = [] - for format_key, regex_list in self._time_zone_regex_map.items(): + for format_key, regex_list in list(self._time_zone_regex_map.items()): if format_key in bad_formats: continue for regex, expr in regex_list: @@ -560,7 +560,7 @@ def parse(self, expression): if not result: continue result_map = result.groupdict() - for key, value in result_map.items(): + for key, value in list(result_map.items()): if value is None: result_map.pop(key) continue diff --git a/lib/python/isodatetime/tests.py b/lib/python/isodatetime/tests.py index 4c1aa15238..a64ef5fff7 100644 --- a/lib/python/isodatetime/tests.py +++ b/lib/python/isodatetime/tests.py @@ -48,7 +48,7 @@ def get_timeduration_tests(): ([], {"hours": 23, "minutes": 1446}, 169560) ] } - for method, method_tests in tests.items(): + for method, method_tests in list(tests.items()): for method_args, test_props, ctrl_results in method_tests: yield test_props, method, method_args, ctrl_results @@ -84,12 +84,12 @@ def get_timedurationparser_tests(): "P0004-078": {"years": 4, "days": 78}, "P0004-078T10,5": {"years": 4, "days": 78, "hours": 10.5}, "P00000020T133702": {"days": 20, "hours": 13, "minutes": 37, - "seconds": 02}, + "seconds": 0o2}, "-P3YT4H2M": {"years": -3, "hours": -4, "minutes": -2}, "-PT5M": {"minutes": -5}, "-P7Y": {"years": -7, "hours": 0} } - for expression, ctrl_result in test_expressions.items(): + for expression, ctrl_result in list(test_expressions.items()): ctrl_data = str(data.Duration(**ctrl_result)) yield expression, ctrl_data @@ -125,7 +125,7 @@ def get_timedurationdumper_tests(): "PT59S": {"seconds": 59, "standardize": True}, "PT1H4M56S": {"minutes": 10, "seconds": 3296, "standardize": True}, } - for expression, ctrl_result in test_expressions.items(): + for expression, ctrl_result in list(test_expressions.items()): yield expression, ctrl_result @@ -140,16 +140,16 @@ def get_timepoint_dumper_tests(): ("YYDDDThh:mm:ss", "44004T05:01:02"), ("WwwD", "W011"), ("CCDDDThh*ss-0600", "00003T23*02-0600"), - (u"+XCCYY-MM-DDThh:mm:ss-11:45", + ("+XCCYY-MM-DDThh:mm:ss-11:45", "+000044-01-03T17:16:02-11:45"), - (u"+XCCYYMM-DDThh-01:00", "+00004401-04T04-01:00"), - (u"+XCCYYMM-DDThh+13:00", "+00004401-04T18+13:00"), - (u"+XCCYYMM-DDThh-0100", "+00004401-04T04-0100"), - (u"+XCCYYMM-DDThh+1300", "+00004401-04T18+1300"), - (u"+XCCYYMMDDThh-0100", "+0000440104T04-0100"), - (u"+XCCYYMMDDThh+13", "+0000440104T18+13"), - (u"+XCCYYMMDDThh+hhmm", "+0000440104T05+0000"), - (u"+XCCYY-MM-DDThh:mm:ss+hh:mm", + ("+XCCYYMM-DDThh-01:00", "+00004401-04T04-01:00"), + ("+XCCYYMM-DDThh+13:00", "+00004401-04T18+13:00"), + ("+XCCYYMM-DDThh-0100", "+00004401-04T04-0100"), + ("+XCCYYMM-DDThh+1300", "+00004401-04T18+1300"), + ("+XCCYYMMDDThh-0100", "+0000440104T04-0100"), + ("+XCCYYMMDDThh+13", "+0000440104T18+13"), + ("+XCCYYMMDDThh+hhmm", "+0000440104T05+0000"), + ("+XCCYY-MM-DDThh:mm:ss+hh:mm", "+000044-01-04T05:01:02+00:00"), ("DD/MM/CCYY is a silly format", "04/01/0044 is a silly format"), ("ThhZ", "T05Z"), @@ -164,20 +164,20 @@ def get_timepoint_dumper_tests(): ("+XCCYYDDDThh:mm:ss", "+500200209T00:26:08"), ("WwwD", "W311"), ("+XCCDDDThh*ss-0600", "+5002209T02*08-0600"), - (u"+XCCYY-MM-DDThh:mm:ss-11:45", + ("+XCCYY-MM-DDThh:mm:ss-11:45", "+500200-07-27T21:11:08-11:45"), - (u"+XCCYYMM-DDThhmm-01:00", "+50020007-28T0756-01:00"), - (u"+XCCYYMM-DDThhmm+13:00", "+50020007-28T2156+13:00"), - (u"+XCCYYMM-DDThhmm-0100", "+50020007-28T0756-0100"), - (u"+XCCYYMM-DDThhmm+1300", "+50020007-28T2156+1300"), - (u"+XCCYYMMDDThhmm-0100", "+5002000728T0756-0100"), - (u"+XCCYYMMDDThhmm+13", "+5002000728T2156+13"), - (u"+XCCYYMMDDThh+hhmm", "+5002000728T00-0830"), - (u"+XCCYYWwwDThhmm+hh", "+500200W311T0026-08"), - (u"+XCCYYDDDThhmm+hh", "+500200209T0026-08"), - (u"+XCCYY-MM-DDThh:mm:ss+hh:mm", + ("+XCCYYMM-DDThhmm-01:00", "+50020007-28T0756-01:00"), + ("+XCCYYMM-DDThhmm+13:00", "+50020007-28T2156+13:00"), + ("+XCCYYMM-DDThhmm-0100", "+50020007-28T0756-0100"), + ("+XCCYYMM-DDThhmm+1300", "+50020007-28T2156+1300"), + ("+XCCYYMMDDThhmm-0100", "+5002000728T0756-0100"), + ("+XCCYYMMDDThhmm+13", "+5002000728T2156+13"), + ("+XCCYYMMDDThh+hhmm", "+5002000728T00-0830"), + ("+XCCYYWwwDThhmm+hh", "+500200W311T0026-08"), + ("+XCCYYDDDThhmm+hh", "+500200209T0026-08"), + ("+XCCYY-MM-DDThh:mm:ss+hh:mm", "+500200-07-28T00:26:08-08:30"), - (u"+XCCYY-MM-DDThh:mm:ssZ", "+500200-07-28T08:56:08Z"), + ("+XCCYY-MM-DDThh:mm:ssZ", "+500200-07-28T08:56:08Z"), ("DD/MM/+XCCYY is a silly format", "28/07/+500200 is a silly format"), ("ThhmmZ", "T0856Z"), @@ -190,20 +190,20 @@ def get_timepoint_dumper_tests(): ("+XCCYYDDDThh:mm:ss", "-000056318T05:01:00"), ("WwwD", "W461"), ("+XCCDDDThh*ss-0600", "-0000317T17*00-0600"), - (u"+XCCYY-MM-DDThh:mm:ss-11:45", + ("+XCCYY-MM-DDThh:mm:ss-11:45", "-000056-11-12T11:16:00-11:45"), - (u"+XCCYYMM-DDThhmm-01:00", "-00005611-12T2201-01:00"), - (u"+XCCYYMM-DDThhmm+13:00", "-00005611-13T1201+13:00"), - (u"+XCCYYMM-DDThhmm-0100", "-00005611-12T2201-0100"), - (u"+XCCYYMM-DDThhmm+1300", "-00005611-13T1201+1300"), - (u"+XCCYYMMDDThhmm-0100", "-0000561112T2201-0100"), - (u"+XCCYYMMDDThhmm+13", "-0000561113T1201+13"), - (u"+XCCYYMMDDThh+hhmm", "-0000561113T05+0600"), - (u"+XCCYYWwwDThhmm+hh", "-000056W461T0501+06"), - (u"+XCCYYDDDThhmm+hh", "-000056318T0501+06"), - (u"+XCCYY-MM-DDThh:mm:ss+hh:mm", + ("+XCCYYMM-DDThhmm-01:00", "-00005611-12T2201-01:00"), + ("+XCCYYMM-DDThhmm+13:00", "-00005611-13T1201+13:00"), + ("+XCCYYMM-DDThhmm-0100", "-00005611-12T2201-0100"), + ("+XCCYYMM-DDThhmm+1300", "-00005611-13T1201+1300"), + ("+XCCYYMMDDThhmm-0100", "-0000561112T2201-0100"), + ("+XCCYYMMDDThhmm+13", "-0000561113T1201+13"), + ("+XCCYYMMDDThh+hhmm", "-0000561113T05+0600"), + ("+XCCYYWwwDThhmm+hh", "-000056W461T0501+06"), + ("+XCCYYDDDThhmm+hh", "-000056318T0501+06"), + ("+XCCYY-MM-DDThh:mm:ss+hh:mm", "-000056-11-13T05:01:00+06:00"), - (u"+XCCYY-MM-DDThh:mm:ssZ", "-000056-11-12T23:01:00Z"), + ("+XCCYY-MM-DDThh:mm:ssZ", "-000056-11-12T23:01:00Z"), ("DD/MM/+XCCYY is a silly format", "13/11/-000056 is a silly format"), ("ThhmmZ", "T2301Z"), @@ -551,14 +551,14 @@ def get_timepointparser_tests(allow_only_basic=False, for date_key in date_format_tests: if not allow_truncated and date_key == "truncated": continue - for date_expr, info in date_format_tests[date_key].items(): + for date_expr, info in list(date_format_tests[date_key].items()): yield date_expr, info for date_key in date_combo_ok_keys: date_tests = date_format_tests[date_key] # Add a blank date for time-only testing. - for date_expr, info in date_tests.items(): + for date_expr, info in list(date_tests.items()): for time_key in time_combo_ok_keys: - time_items = time_format_tests[time_key].items() + time_items = list(time_format_tests[time_key].items()) for time_expr, time_info in time_items: combo_expr = ( date_expr + @@ -566,41 +566,41 @@ def get_timepointparser_tests(allow_only_basic=False, time_expr ) combo_info = {} - for key, value in info.items() + time_info.items(): + for key, value in list(info.items()) + list(time_info.items()): combo_info[key] = value yield combo_expr, combo_info if skip_time_zones: continue - time_zone_items = time_zone_format_tests.items() + time_zone_items = list(time_zone_format_tests.items()) for time_zone_expr, time_zone_info in time_zone_items: tz_expr = combo_expr + time_zone_expr tz_info = {} - for key, value in (combo_info.items() + - time_zone_info.items()): + for key, value in (list(combo_info.items()) + + list(time_zone_info.items())): tz_info[key] = value yield tz_expr, tz_info if not allow_truncated: continue for time_key in time_format_tests: time_tests = time_format_tests[time_key] - for time_expr, time_info in time_tests.items(): + for time_expr, time_info in list(time_tests.items()): combo_expr = ( time_designator + time_expr ) # Add truncated (no date). combo_info = {"truncated": True} - for key, value in time_info.items(): + for key, value in list(time_info.items()): combo_info[key] = value yield combo_expr, combo_info if skip_time_zones: continue - time_zone_items = time_zone_format_tests.items() + time_zone_items = list(time_zone_format_tests.items()) for time_zone_expr, time_zone_info in time_zone_items: tz_expr = combo_expr + time_zone_expr tz_info = {} - for key, value in (combo_info.items() + - time_zone_info.items()): + for key, value in (list(combo_info.items()) + + list(time_zone_info.items())): tz_info[key] = value yield tz_expr, tz_info @@ -929,7 +929,7 @@ def test_days_in_year_range(self): test_days = data.get_days_in_year_range( start_year, end_year) control_days = 0 - for year in xrange(start_year, end_year + 1): + for year in range(start_year, end_year + 1): control_days += data.get_days_in_year(year) self.assertEqual( control_days, test_days, "days in %s to %s" % ( @@ -978,7 +978,7 @@ def test_timeduration_add_week(self): def test_timepoint(self): """Test the time point data model (takes a while).""" pool = multiprocessing.Pool(processes=4) - pool.map_async(test_timepoint_at_year, range(1801, 2403)).get() + pool.map_async(test_timepoint_at_year, list(range(1801, 2403))).get() def test_timepoint_plus_float_time_duration_day_of_month_type(self): """Test (TimePoint + Duration).day_of_month is an int.""" @@ -1187,7 +1187,7 @@ def test_timepoint_strftime_strptime(self): """Test the strftime/strptime for date/time expressions.""" import datetime parser = parsers.TimePointParser() - parse_tokens = parser_spec.STRFTIME_TRANSLATE_INFO.keys() + parse_tokens = list(parser_spec.STRFTIME_TRANSLATE_INFO.keys()) parse_tokens.remove("%z") # Don't test datetime's tz handling. format_string = "" for i, token in enumerate(parse_tokens): @@ -1202,10 +1202,10 @@ def test_timepoint_strftime_strptime(self): format_string += "++(" strftime_string = format_string strptime_strings = [format_string] - for key in parser_spec.STRPTIME_EXCLUSIVE_GROUP_INFO.keys(): + for key in list(parser_spec.STRPTIME_EXCLUSIVE_GROUP_INFO.keys()): strptime_strings[-1] = strptime_strings[-1].replace(key, "") strptime_strings.append(format_string) - for values in parser_spec.STRPTIME_EXCLUSIVE_GROUP_INFO.values(): + for values in list(parser_spec.STRPTIME_EXCLUSIVE_GROUP_INFO.values()): for value in values: strptime_strings[-1] = strptime_strings[-1].replace(value, "") ctrl_date = datetime.datetime(2002, 3, 1, 12, 30, 2) diff --git a/lib/python/rose/app_run.py b/lib/python/rose/app_run.py index 7f5342c3d1..3b407f2768 100644 --- a/lib/python/rose/app_run.py +++ b/lib/python/rose/app_run.py @@ -372,7 +372,7 @@ def _prep(self, conf_tree, opts): # Free format files not defined in the configuration file file_section_prefix = self.config_pm.get_handler("file").PREFIX - for rel_path, conf_dir in conf_tree.files.items(): + for rel_path, conf_dir in list(conf_tree.files.items()): if not rel_path.startswith("file" + os.sep): continue name = rel_path[len("file" + os.sep):] diff --git a/lib/python/rose/apps/ana_builtin/grepper.py b/lib/python/rose/apps/ana_builtin/grepper.py index 2915b0f6e8..4f53d789d1 100644 --- a/lib/python/rose/apps/ana_builtin/grepper.py +++ b/lib/python/rose/apps/ana_builtin/grepper.py @@ -264,7 +264,7 @@ def run_analysis(self): matched_groups = self.search_for_matches() # Check that the number of matchings found is equal in all files - group_lens = [len(groups) for groups in matched_groups.values()] + group_lens = [len(groups) for groups in list(matched_groups.values())] for igroup, group_len in enumerate(group_lens[1:]): if group_len != group_lens[0]: msg = ("File ({0}) matches pattern {1} times, but File ({2}) " diff --git a/lib/python/rose/apps/rose_ana.py b/lib/python/rose/apps/rose_ana.py index 61b0e6748e..3bf23fb6f0 100644 --- a/lib/python/rose/apps/rose_ana.py +++ b/lib/python/rose/apps/rose_ana.py @@ -151,14 +151,13 @@ def buffer_to_db(self, reporter=None): self.statement_buffer = [] -class AnalysisTask(object): +class AnalysisTask(object, metaclass=abc.ABCMeta): """ Base class for an analysis task; all custom user tasks should inherit from this class and override the "run_analysis" method to perform whatever analysis is required. """ - __metaclass__ = abc.ABCMeta def __init__(self, parent_app, task_options): """ @@ -384,7 +383,7 @@ def _get_global_ana_config(self): user_config = ( self.rose_conf.get_value(["rose-ana"])) if user_config is not None: - for name, obj in user_config.items(): + for name, obj in list(user_config.items()): if obj.state == "": self.ana_config[name] = obj.value diff --git a/lib/python/rose/apps/rose_ana_v1.py b/lib/python/rose/apps/rose_ana_v1.py index 0d38f73b3a..ef231bc958 100644 --- a/lib/python/rose/apps/rose_ana_v1.py +++ b/lib/python/rose/apps/rose_ana_v1.py @@ -391,7 +391,7 @@ def load_tasks(self): """ tasks = [] - for task in self.config.value.keys(): + for task in list(self.config.value.keys()): if task is "env": continue if task.startswith("file:"): diff --git a/lib/python/rose/apps/rose_arch.py b/lib/python/rose/apps/rose_arch.py index 7d78d9ba90..f61fbee600 100644 --- a/lib/python/rose/apps/rose_arch.py +++ b/lib/python/rose/apps/rose_arch.py @@ -74,7 +74,7 @@ def __str__(self): t_info, ret_code_str) if target.status != target.ST_OLD: - for source in sorted(target.sources.values(), + for source in sorted(list(target.sources.values()), lambda s1, s2: cmp(s1.name, s2.name)): ret += "\n%s\t%s (%s)" % ( target.status, source.name, source.orig_name) @@ -267,7 +267,7 @@ def _run_target_setup( exc) else: rename_parser = None - for source in target.sources.values(): + for source in list(target.sources.values()): dict_ = { "cycle": os.getenv("ROSE_TASK_CYCLE_TIME"), "name": source.name} @@ -312,12 +312,12 @@ def _run_target_update(cls, dao, app_runner, compress_manager, target): # Rename/edit sources target.status = target.ST_BAD rename_required = False - for source in target.sources.values(): + for source in list(target.sources.values()): if source.name != source.orig_name: rename_required = True break if rename_required or target.source_edit_format: - for source in target.sources.values(): + for source in list(target.sources.values()): source.path = os.path.join(work_dir, source.name) app_runner.fs_util.makedirs( os.path.dirname(source.path)) @@ -340,7 +340,7 @@ def _run_target_update(cls, dao, app_runner, compress_manager, target): if target.work_source_path: sources = [target.work_source_path] else: - for source in target.sources.values(): + for source in list(target.sources.values()): sources.append(source.path) command = target.command_format % { "sources": app_runner.popen.list_to_shell_str(sources), @@ -509,7 +509,7 @@ def insert(self, target): conn.execute(t_stmt, t_stmt_args) sh_stmt = r"INSERT INTO " + self.T_SOURCES + " VALUES (?, ?, ?)" sh_stmt_args = [target.name] - for checksum, source in target.sources.items(): + for checksum, source in list(target.sources.items()): conn.execute(sh_stmt, sh_stmt_args + [source.name, checksum]) conn.commit() diff --git a/lib/python/rose/apps/rose_arch_compressions/rose_arch_gzip.py b/lib/python/rose/apps/rose_arch_compressions/rose_arch_gzip.py index b08773c26a..05a62162fa 100644 --- a/lib/python/rose/apps/rose_arch_compressions/rose_arch_gzip.py +++ b/lib/python/rose/apps/rose_arch_compressions/rose_arch_gzip.py @@ -39,7 +39,7 @@ def compress_sources(self, target, work_dir): Use work_dir to dump results. """ - for source in target.sources.values(): + for source in list(target.sources.values()): if source.path.endswith("." + target.compress_scheme): continue # assume already done name_gz = source.name + "." + target.compress_scheme diff --git a/lib/python/rose/apps/rose_arch_compressions/rose_arch_tar.py b/lib/python/rose/apps/rose_arch_compressions/rose_arch_tar.py index c5d82ea0e4..122c15f2cc 100644 --- a/lib/python/rose/apps/rose_arch_compressions/rose_arch_tar.py +++ b/lib/python/rose/apps/rose_arch_compressions/rose_arch_tar.py @@ -41,7 +41,7 @@ def compress_sources(self, target, work_dir): Use work_dir to dump results. """ - sources = target.sources.values() + sources = list(target.sources.values()) scheme = target.compress_scheme if (len(sources) == 1 and sources[0].path.endswith("." + target.compress_scheme)): diff --git a/lib/python/rose/apps/rose_bunch.py b/lib/python/rose/apps/rose_bunch.py index 922468ae38..6bdcb12d1f 100644 --- a/lib/python/rose/apps/rose_bunch.py +++ b/lib/python/rose/apps/rose_bunch.py @@ -150,7 +150,7 @@ def run(self, app_runner, conf_tree, opts, args, uuid, work_files): self.incremental = rose.env.env_var_process(self.incremental) multi_args = conf_tree.node.get_value([self.ARGS_SECTION], {}) - for key, val in multi_args.items(): + for key, val in list(multi_args.items()): multi_args[key].value = rose.env.env_var_process(val.value) self.command_format = rose.env.env_var_process( @@ -168,7 +168,7 @@ def run(self, app_runner, conf_tree, opts, args, uuid, work_files): if instances: try: - instances = range(int(rose.env.env_var_process(instances))) + instances = list(range(int(rose.env.env_var_process(instances)))) except ValueError: raise ConfigValueError([self.BUNCH_SECTION, "command-instances"], @@ -182,7 +182,7 @@ def run(self, app_runner, conf_tree, opts, args, uuid, work_files): else: item, val = sorted(multi_args.items())[0] arglength = len(shlex.split(val.value)) - self.invocation_names = range(0, arglength) + self.invocation_names = list(range(0, arglength)) else: arglength = len(self.invocation_names) @@ -235,7 +235,7 @@ def run(self, app_runner, conf_tree, opts, args, uuid, work_files): abort = False while procs or (commands and not abort): - for key, proc in procs.items(): + for key, proc in list(procs.items()): if proc.poll() is not None: procs.pop(key) if proc.returncode: @@ -460,7 +460,7 @@ def record_config(self, config, clear_db=False): args = [] - for key, value in res.items(): + for key, value in list(res.items()): args.append((key, value)) i_stmt = ("INSERT OR REPLACE INTO " + self.TABLE_CONFIG + diff --git a/lib/python/rose/apps/rose_prune.py b/lib/python/rose/apps/rose_prune.py index 8765994375..3817286879 100644 --- a/lib/python/rose/apps/rose_prune.py +++ b/lib/python/rose/apps/rose_prune.py @@ -219,7 +219,7 @@ def _get_prune_globs(self, app_runner, conf_tree): if nodes is None: return [], set() cycle_formats = {} - for key, node in nodes.items(): + for key, node in list(nodes.items()): if node.is_ignored(): continue if key.startswith("cycle-format{") and key.endswith("}"): @@ -252,7 +252,7 @@ def _get_prune_globs(self, app_runner, conf_tree): cycle_set.add(cycle) if cycle_args: cycle_strs = {"cycle": cycle} - for key, cycle_format in cycle_formats.items(): + for key, cycle_format in list(cycle_formats.items()): if self._get_cycling_mode() == "integer": cycle_strs[key] = cycle_format % int(cycle) else: # date time cycling diff --git a/lib/python/rose/bush.py b/lib/python/rose/bush.py index 92fd3a0427..df965f92e3 100644 --- a/lib/python/rose/bush.py +++ b/lib/python/rose/bush.py @@ -23,6 +23,7 @@ from fnmatch import fnmatch from glob import glob import jinja2 +import math import mimetypes import os import re @@ -37,8 +38,9 @@ from tempfile import NamedTemporaryFile from time import gmtime, strftime import traceback -import urllib +import urllib.request, urllib.parse, urllib.error +cherrypy.config.update({'environment': 'production'}) class RoseBushService(object): @@ -59,6 +61,7 @@ class RoseBushService(object): VIEW_SIZE_MAX = 10 * 1024 * 1024 # 10MB def __init__(self, *args, **kwargs): + cherrypy.log("bush.init()") self.exposed = True self.bush_dao = RoseBushDAO() rose_conf = ResourceLocator.default().get_conf() @@ -79,11 +82,13 @@ def __init__(self, *args, **kwargs): @classmethod def url2hyperlink(cls, text): """Turn http or https link into a hyperlink.""" + cherrypy.log("bush.url2hyperlink()") return cls.REC_URL.sub(r'\g<1>', text) @cherrypy.expose def index(self, form=None): """Display a page to input user ID and suite ID.""" + cherrypy.log("bush.index()") data = { "logo": self.logo, "title": self.title, @@ -101,6 +106,7 @@ def index(self, form=None): @cherrypy.expose def broadcast_states(self, user, suite, form=None): """List current broadcasts of a running or completed suite.""" + cherrypy.log("bush.broadcast_states()") data = { "logo": self.logo, "title": self.title, @@ -132,6 +138,7 @@ def broadcast_states(self, user, suite, form=None): @cherrypy.expose def broadcast_events(self, user, suite, form=None): """List broadcasts history of a running or completed suite.""" + cherrypy.log("bush.broadcast_events()") data = { "logo": self.logo, "title": self.title, @@ -163,6 +170,7 @@ def cycles( self, user, suite, page=1, order=None, per_page=None, no_fuzzy_time="0", form=None): """List cycles of a running or completed suite.""" + cherrypy.log("bush.cycles()") conf = ResourceLocator.default().get_conf() per_page_default = int(conf.get_value( ["rose-bush", "cycles-per-page"], self.CYCLES_PER_PAGE)) @@ -200,7 +208,7 @@ def cycles( self.bush_dao.get_suite_cycles_summary( user, suite, order, per_page, (page - 1) * per_page)) if per_page: - data["n_pages"] = data["of_n_entries"] / per_page + data["n_pages"] = math.ceil(data["of_n_entries"] / per_page) if data["of_n_entries"] % per_page != 0: data["n_pages"] += 1 else: @@ -257,6 +265,7 @@ def taskjobs( return a JSON data structure. """ + cherrypy.log("bush.taskjobs()") conf = ResourceLocator.default().get_conf() per_page_default = int(conf.get_value( ["rose-bush", "jobs-per-page"], self.JOBS_PER_PAGE)) @@ -329,7 +338,7 @@ def taskjobs( data["entries"] = entries data["of_n_entries"] = of_n_entries if per_page: - data["n_pages"] = of_n_entries / per_page + data["n_pages"] = math.ceil(of_n_entries / per_page) if of_n_entries % per_page != 0: data["n_pages"] += 1 else: @@ -351,12 +360,13 @@ def jobs(self, user, suite, page=1, cycles=None, tasks=None, Convert "no_status" to "task_status" argument of self.taskjobs. """ + cherrypy.log("bush.jobs()") task_status = None if no_status: task_status = [] if not isinstance(no_status, list): no_status = [no_status] - for key, values in self.bush_dao.TASK_STATUS_GROUPS.items(): + for key, values in list(self.bush_dao.TASK_STATUS_GROUPS.items()): if key not in no_status: task_status += values return self.taskjobs( @@ -373,6 +383,7 @@ def suites(self, user, names=None, page=1, order=None, per_page=None, return a JSON data structure. """ + cherrypy.log("bush.suites()") user_suite_dir_root = self._get_user_suite_dir_root(user) conf = ResourceLocator.default().get_conf() per_page_default = int(conf.get_value( @@ -433,17 +444,26 @@ def suites(self, user, names=None, page=1, order=None, per_page=None, except OSError: continue - if order == "name_asc": - data["entries"].sort(key=lambda entry: entry["name"]) - elif order == "name_desc": - data["entries"].sort(key=lambda entry: entry["name"], reverse=True) - elif order == "time_asc": - data["entries"].sort(self._sort_summary_entries, reverse=True) - else: # order == "time_desc" - data["entries"].sort(self._sort_summary_entries) + try: + if order == "name_asc": + data["entries"].sort(key=lambda entry: entry["name"]) + elif order == "name_desc": + data["entries"].sort(key=lambda entry: entry["name"], + reverse=True) + elif order == "time_asc": + data["entries"].sort(key=lambda x: x["name"], reverse=True) + data["entries"].sort(key=lambda x: x.get("last_activity_time"), + reverse=True) + else: # order == "time_desc" + data["entries"].sort(key=lambda x: x["name"]) + data["entries"].sort(key=lambda x: x.get("last_activity_time")) + except TypeError: + # Empty entries create a NoneType vs str comparison + pass + data["of_n_entries"] = len(data["entries"]) if per_page: - data["n_pages"] = data["of_n_entries"] / per_page + data["n_pages"] = math.ceil(data["of_n_entries"] / per_page) if data["of_n_entries"] % per_page != 0: data["n_pages"] += 1 offset = (page - 1) * per_page @@ -456,7 +476,7 @@ def suites(self, user, names=None, page=1, order=None, per_page=None, rose_suite_info = os.path.join(user_suite_dir, "rose-suite.info") try: info_root = rose.config.load(rose_suite_info) - for key, node in info_root.value.items(): + for key, node in list(info_root.value.items()): if (node.is_ignored() or not isinstance(node.value, str)): continue @@ -471,6 +491,7 @@ def suites(self, user, names=None, page=1, order=None, per_page=None, def get_file(self, user, suite, path, path_in_tar=None, mode=None): """Returns file information / content or a cherrypy response.""" + cherrypy.log("bush.get_file()") f_name = self._get_user_suite_dir(user, suite, path) conf = ResourceLocator.default().get_conf() view_size_max = int(conf.get_value( @@ -480,6 +501,8 @@ def get_file(self, user, suite, path, path_in_tar=None, mode=None): try: tar_info = tar_f.getmember(path_in_tar) except KeyError: + cherrypy.log.error("Unable to find " + path_in_tar + + "in tar file") raise cherrypy.HTTPError(404) f_size = tar_info.size handle = tar_f.extractfile(path_in_tar) @@ -487,7 +510,7 @@ def get_file(self, user, suite, path, path_in_tar=None, mode=None): mime = self.MIME_TEXT_PLAIN else: mime = mimetypes.guess_type( - urllib.pathname2url(path_in_tar))[0] + urllib.request.pathname2url(path_in_tar))[0] handle.seek(0) if (mode == "download" or f_size > view_size_max or @@ -511,7 +534,7 @@ def get_file(self, user, suite, path, path_in_tar=None, mode=None): if open(f_name).read(2) == "#!": mime = self.MIME_TEXT_PLAIN else: - mime = mimetypes.guess_type(urllib.pathname2url(f_name))[0] + mime = mimetypes.guess_type(urllib.request.pathname2url(f_name))[0] if not mime: mime = self.MIME_TEXT_PLAIN if (mode == "download" or @@ -524,7 +547,7 @@ def get_file(self, user, suite, path, path_in_tar=None, mode=None): try: if mode in [None, "text"]: text = jinja2.escape(text) - lines = [unicode(line) for line in text.splitlines()] + lines = [str(line) for line in text.splitlines()] except UnicodeDecodeError: if path_in_tar: handle.seek(0) @@ -561,18 +584,22 @@ def get_file(self, user, suite, path, path_in_tar=None, mode=None): def get_last_activity_time(self, user, suite): """Returns last activity time for a suite based on database stat""" + cherrypy.log("bush.get_last_activity_time()") for name in [os.path.join("log", "db"), "cylc-suite.db"]: fname = os.path.join(self._get_user_suite_dir(user, suite), name) try: return strftime( "%Y-%m-%dT%H:%M:%SZ", gmtime(os.stat(fname).st_mtime)) - except OSError: + except OSError as e: + cherrypy.log.error("bush.get_last_activity_time().error: " + + str(e)) continue @cherrypy.expose def viewsearch(self, user, suite, path=None, path_in_tar=None, mode=None, search_string=None, search_mode=None): """Search a text log file.""" + cherrypy.log("bush.viewsearch()") # get file or serve raw data file_output = self.get_file( user, suite, path, path_in_tar=path_in_tar, mode=mode) @@ -614,7 +641,7 @@ def viewsearch(self, user, suite, path=None, path_in_tar=None, mode=None, # no search is being performed, client is requesting the whole # page if mode in [None, "text"]: - line_numbers = range(1, len(lines) + 1) + line_numbers = list(range(1, len(lines) + 1)) else: line_numbers = [] lines = [[line] for line in lines] @@ -629,6 +656,7 @@ def viewsearch(self, user, suite, path=None, path_in_tar=None, mode=None, def view(self, user, suite, path, path_in_tar=None, mode=None, no_fuzzy_time="0"): """View a text log file.""" + cherrypy.log("bush.view()") # get file or serve raw data file_output = self.get_file( user, suite, path, path_in_tar=path_in_tar, mode=mode) @@ -665,6 +693,7 @@ def view(self, user, suite, path, path_in_tar=None, mode=None, def _get_suite_logs_info(self, user, suite): """Return a dict with suite logs and Rosie suite info.""" + cherrypy.log("bush._get_suite_logs_info()") data = {"info": {}, "files": {}} user_suite_dir = self._get_user_suite_dir(user, suite) @@ -673,7 +702,7 @@ def _get_suite_logs_info(self, user, suite): if os.path.isfile(info_name): try: info_root = rose.config.load(info_name) - for key, node in info_root.value.items(): + for key, node in list(info_root.value.items()): if node.is_ignored() or not isinstance(node.value, str): continue data["info"][key] = node.value @@ -717,44 +746,43 @@ def _check_dir_access(cls, path): Return path on success. """ + cherrypy.log("bush._check_dir_access()") if not os.path.exists(path): + cherrypy.log.error( + "bush._check_dir_access: Path does not exist: " + path) raise cherrypy.HTTPError(404) if not os.access(path, os.R_OK): + cherrypy.log.error( + "bush._check_dir_access: Path is not accessible: " + path) + raise cherrypy.HTTPError(403) return path @staticmethod def _get_user_home(user): - """Return, e.g. ~/cylc-run/ for a cylc suite. - - N.B. os.path.expanduser does not fail if ~user is invalid. - - """ + """Return, the path for a user cylc suite.""" + cherrypy.log("bush._get_user_home()") try: return pwd.getpwnam(user).pw_dir except KeyError: raise cherrypy.HTTPError(404) def _get_user_suite_dir_root(self, user): - """Return, e.g. ~user/cylc-run/ for a cylc suite.""" + """Return /cylc-run/ for a cylc suite.""" + cherrypy.log("bush._get_user_suite_dir_root()") return self._check_dir_access(os.path.join( self._get_user_home(user), self.bush_dao.SUITE_DIR_REL_ROOT)) def _get_user_suite_dir(self, user, suite, *paths): - """Return, e.g. ~user/cylc-run/suite/... for a cylc suite.""" - return self._check_dir_access(os.path.join( + """Return, /cylc-run/suite/... for a cylc suite.""" + cherrypy.log("bush._get_user_suite_dir()") + result = self._check_dir_access(os.path.join( self._get_user_home(user), self.bush_dao.SUITE_DIR_REL_ROOT, suite, *paths)) - - @staticmethod - def _sort_summary_entries(suite1, suite2): - """Sort suites by last_activity_time.""" - return (cmp(suite2.get("last_activity_time"), - suite1.get("last_activity_time")) or - cmp(suite1["name"], suite2["name"])) + return result if __name__ == "__main__": diff --git a/lib/python/rose/bush_dao.py b/lib/python/rose/bush_dao.py index c43cdfc48f..3804e13f8c 100644 --- a/lib/python/rose/bush_dao.py +++ b/lib/python/rose/bush_dao.py @@ -110,6 +110,7 @@ class RoseBushDAO(object): def __init__(self): self.daos = {} + cherrypy.log("BushDAO.init") def get_suite_broadcast_states(self, user_name, suite_name): """Return broadcast states of a suite. @@ -117,6 +118,7 @@ def get_suite_broadcast_states(self, user_name, suite_name): [[point, name, key, value], ...] """ + cherrypy.log("bush_dao.get_suite_broadcast_states()") # Check if "broadcast_states" table is available or not if not self._db_has_table(user_name, suite_name, "broadcast_states"): return @@ -136,6 +138,7 @@ def get_suite_broadcast_events(self, user_name, suite_name): [[time, change, point, name, key, value], ...] """ + cherrypy.log("bush_dao.get_suite_broadcast_events()") # Check if "broadcast_events" table is available or not if not self._db_has_table(user_name, suite_name, "broadcast_events"): return {} @@ -157,6 +160,7 @@ def get_suite_dir_rel(suite_name, *paths): paths -- if specified, are added to the end of the path. """ + cherrypy.log("bush_dao.get_suite_dir_rel()") return CylcProcessor.get_suite_dir_rel(suite_name, *paths) def get_suite_job_entries( @@ -197,6 +201,7 @@ def get_suite_job_entries( "err": {...}, ...}} """ + cherrypy.log("bush_dao.get_suite_job_entries()") where_expr, where_args = self._get_suite_job_entries_where( cycles, tasks, task_status, job_status) @@ -222,8 +227,7 @@ def get_suite_job_entries( " task_states.submit_num AS submit_num_max," + " task_states.status AS task_status," + " time_submit, submit_status," + - " time_run, time_run_exit, run_signal, run_status," + - " user_at_host, batch_sys_name, batch_sys_job_id" + + " time_run, time_run_exit, run_signal, run_status" + " FROM task_jobs JOIN task_states USING (cycle, name)" + where_expr + " ORDER BY " + @@ -237,8 +241,7 @@ def get_suite_job_entries( ( cycle, name, submit_num, submit_num_max, task_status, time_submit, submit_status, - time_run, time_run_exit, run_signal, run_status, - user_at_host, batch_sys_name, batch_sys_job_id + time_run, time_run_exit, run_signal, run_status ) = row[1:] entry = { "cycle": cycle, @@ -250,9 +253,9 @@ def get_suite_job_entries( "submit_status": submit_status, "run_signal": run_signal, "run_status": run_status, - "host": user_at_host, - "submit_method": batch_sys_name, - "submit_method_id": batch_sys_job_id, + "host": "", + "submit_method": "", + "submit_method_id": "", "logs": {}, "seq_logs_indexes": {}} entries.append(entry) @@ -268,6 +271,7 @@ def _get_suite_job_entries_where( Get query's "WHERE" expression and its arguments. """ + cherrypy.log("bush_dao._get_suite_job_entries_where()") where_exprs = [] where_args = [] if cycles: @@ -383,7 +387,7 @@ def _get_job_logs(self, user_name, suite_name, entries, entry_of): # Sequential logs for entry in entries: - for filename, filename_items in entry["logs"].items(): + for filename, filename_items in list(entry["logs"].items()): seq_log_match = self.REC_SEQ_LOG.match(filename) if not seq_log_match: continue @@ -393,7 +397,7 @@ def _get_job_logs(self, user_name, suite_name, entries, entry_of): if seq_key not in entry["seq_logs_indexes"]: entry["seq_logs_indexes"][seq_key] = {} entry["seq_logs_indexes"][seq_key][index_str] = filename - for seq_key, indexes in entry["seq_logs_indexes"].items(): + for seq_key, indexes in list(entry["seq_logs_indexes"].items()): # Only one item, not a sequence if len(indexes) <= 1: entry["seq_logs_indexes"].pop(seq_key) @@ -401,12 +405,12 @@ def _get_job_logs(self, user_name, suite_name, entries, entry_of): # the template can sort them as numbers try: int_indexes = {} - for index_str, filename in indexes.items(): + for index_str, filename in list(indexes.items()): int_indexes[int(index_str)] = filename entry["seq_logs_indexes"][seq_key] = int_indexes except ValueError: pass - for filename, log_dict in entry["logs"].items(): + for filename, log_dict in list(entry["logs"].items()): # Unset seq_key for singular items if log_dict["seq_key"] not in entry["seq_logs_indexes"]: log_dict["seq_key"] = None @@ -493,6 +497,7 @@ def get_suite_cycles_summary( and of_n_entries is the total number of entries. """ + cherrypy.log("bush_dao.get_suite_cycles_summary()") of_n_entries = 0 stmt = ("SELECT COUNT(DISTINCT cycle) FROM task_states WHERE " + "submit_num > 0") @@ -519,12 +524,15 @@ def get_suite_cycles_summary( try: for item in os.listdir(os.path.join(user_suite_dir, "log")): if item.startswith("job-") and item.endswith(".tar.gz"): + cherrypy.log( + "bush_dao.get_suite_cycles_summary:.log_cycles " + + item[4:-7]) targzip_log_cycles.append(item[4:-7]) except OSError: pass states_stmt = {} - for key, names in self.TASK_STATUS_GROUPS.items(): + for key, names in list(self.TASK_STATUS_GROUPS.items()): states_stmt[key] = " OR ".join( ["status=='%s'" % (name) for name in names]) stmt = ( @@ -614,6 +622,7 @@ def get_suite_state_summary(self, user_name, suite_name): * server: host:port of server, if available """ + cherrypy.log("bush_dao.get_suite_state_summary()") ret = { "is_running": False, "is_failed": False, @@ -621,6 +630,17 @@ def get_suite_state_summary(self, user_name, suite_name): dao = self._db_init(user_name, suite_name) if not os.access(dao.db_f_name, os.F_OK | os.R_OK): return ret + except FileNotFoundError: + cherrypy.log( + "CylcSuiteDAO.connect: File does not exist: " + + str(port_file_path)) + return ret + except multiprocessing.pool.MaybeEncodingError: + cherrypy.log( + "CylcSuiteDAO.connect: File encoding error: " + + str(port_file_path)) + return ret + port_file_path = os.path.expanduser( os.path.join( @@ -664,17 +684,24 @@ def parse_job_log_rel_path(cls, f_name): def _db_close(self, user_name, suite_name): """Close a named database connection.""" + cherrypy.log("bush_dao._db_close()") key = (user_name, suite_name) if self.daos.get(key) is not None: self.daos[key].close() def _db_exec(self, user_name, suite_name, stmt, stmt_args=None): """Execute a query on a named database connection.""" + cherrypy.log("bush_dao._db_exec()") daos = self._db_init(user_name, suite_name) - return daos.execute(stmt, stmt_args) + cherrypy.log("bush_dao._db_exec stmt: " + str(stmt) + " stmt_args: " + + str(stmt_args)) + result = daos.execute(stmt, stmt_args) + cherrypy.log("bush_dao._db_exec.result: " + str(result)) + return result def _db_has_table(self, user_name, suite_name, table_name): """Return True if table_name exists in the suite database.""" + cherrypy.log("bush_dao._db_has_table()") cursor = self._db_exec( user_name, suite_name, "SELECT name FROM sqlite_master WHERE name==?", [table_name]) @@ -682,6 +709,7 @@ def _db_has_table(self, user_name, suite_name, table_name): def _db_init(self, user_name, suite_name): """Initialise a named database connection.""" + cherrypy.log("bush_dao._db_init()") key = (user_name, suite_name) if key not in self.daos: prefix = "~" diff --git a/lib/python/rose/c3.py b/lib/python/rose/c3.py index 67392fa93a..55c71a305a 100644 --- a/lib/python/rose/c3.py +++ b/lib/python/rose/c3.py @@ -124,15 +124,15 @@ def get_base_names(self, name): def ok(self, key, cond): self.test_num += 1 if cond: - print "ok %d - %s" % (self.test_num, key) + print("ok %d - %s" % (self.test_num, key)) else: - print "not ok %d - %s" % (self.test_num, key) + print("not ok %d - %s" % (self.test_num, key)) def test(self, key, actual, expect): self.ok(key, actual == expect) def run(self): - print self.test_plan + print(self.test_plan) # Test good cases self.base_names_of["O"] = [] diff --git a/lib/python/rose/config.py b/lib/python/rose/config.py index e68aba4f3b..530f4827b1 100644 --- a/lib/python/rose/config.py +++ b/lib/python/rose/config.py @@ -165,22 +165,25 @@ class ConfigNode(object): """ - __slots__ = ["STATE_NORMAL", "STATE_USER_IGNORED", - "STATE_SYST_IGNORED", "value", "state", "comments"] - - STATE_NORMAL = "" - """The default state of a ConfigNode.""" - STATE_USER_IGNORED = "!" - """ConfigNode state if it has been specifically ignored in the config.""" - STATE_SYST_IGNORED = "!!" - """ConfigNode state if a metadata opperation has logically ignored the - config.""" - - def __init__(self, value=None, state=STATE_NORMAL, comments=None): + __slots__ = ("STATE_NORMAL", "STATE_USER_IGNORED", + "STATE_SYST_IGNORED", "value", "state", "comments") + + def __init__(self, value=None, state=None, comments=None): if value is None: value = {} if comments is None: comments = [] + + if state is None: + self.STATE_NORMAL = "" + else: + self.STATE_NORMAL = state + """The default state of a ConfigNode.""" + self.STATE_USER_IGNORED = "!" + """ConfigNode state if it has been specifically ignored in the config.""" + self.STATE_SYST_IGNORED = "!!" + """ConfigNode state if a metadata opperation has logically ignored the + config.""" self.value = value self.state = state self.comments = comments @@ -206,7 +209,7 @@ def __delitem__(self, key): def __iter__(self): if isinstance(self.value, dict): - for key in self.value.keys(): + for key in list(self.value.keys()): yield key def __eq__(self, other): @@ -284,7 +287,7 @@ def walk(self, keys=None, no_ignore=False): while stack: node_keys, node = stack.pop(0) if isinstance(node.value, dict): - for key in node.value.keys(): + for key in list(node.value.keys()): child_keys = node_keys + [key] subnode = self.get(child_keys, no_ignore) if subnode is not None: @@ -1074,7 +1077,7 @@ def dump(self, root, target=sys.stdout, sort_sections=None, for comment in root.comments: handle.write(self._comment_format(comment)) blank = "\n" - root_keys = root.value.keys() + root_keys = list(root.value.keys()) root_keys.sort(sort_sections) root_option_keys = [] section_keys = [] @@ -1102,7 +1105,7 @@ def dump(self, root, target=sys.stdout, sort_sections=None, "state": section_node.state, "key": section_key, "close": CHAR_SECTION_CLOSE}) - keys = section_node.value.keys() + keys = list(section_node.value.keys()) keys.sort(sort_option_items) for key in keys: value = section_node.value[key] @@ -1565,8 +1568,8 @@ def sort_element(elem_1, elem_2): def sort_settings(setting_1, setting_2): """Sort sections and options, by numeric element if possible.""" - if (not isinstance(setting_1, basestring) or - not isinstance(setting_2, basestring)): + if (not isinstance(setting_1, str) or + not isinstance(setting_2, str)): return cmp(setting_1, setting_2) match_1 = REC_SETTING_ELEMENT.match(setting_1) match_2 = REC_SETTING_ELEMENT.match(setting_2) diff --git a/lib/python/rose/config_cli.py b/lib/python/rose/config_cli.py index 368f50610c..28c51be330 100644 --- a/lib/python/rose/config_cli.py +++ b/lib/python/rose/config_cli.py @@ -155,7 +155,7 @@ def main(): conf_dump(ConfigNode().set(args, node.value), concat_mode=True) sys.exit() - keys = node.value.keys() + keys = list(node.value.keys()) keys.sort() for key in keys: node_of_key = node.get([key], opts.no_ignore) @@ -164,10 +164,10 @@ def main(): state = node_of_key.state string = "%s%s=%s" % (state, key, value) lines = string.splitlines() - print lines[0] + print(lines[0]) i_equal = len(state + key) + 1 for line in lines[1:]: - print " " * i_equal + line + print(" " * i_equal + line) sys.exit() if node is None: @@ -181,7 +181,7 @@ def main(): if opts.print_conf_mode: conf_dump(ConfigNode().set(args, value), concat_mode=True) else: - print value + print(value) sys.exit() diff --git a/lib/python/rose/config_diff.py b/lib/python/rose/config_diff.py index ba9d27ffdb..a23c0fb01c 100644 --- a/lib/python/rose/config_diff.py +++ b/lib/python/rose/config_diff.py @@ -23,7 +23,7 @@ import re import os import shlex -import StringIO +import io import sys import tempfile @@ -122,7 +122,7 @@ def format_metadata_as_text(metadata, only_these_options=None): if value is None: continue id_node.set([property_], value=value) - string_file = StringIO.StringIO() + string_file = io.StringIO() rose.config.dump(id_node, target=string_file) return string_file.getvalue() @@ -235,7 +235,7 @@ def load_override_config(): ["rose-config-diff"]) if conf is None: return - for key, node in conf.value.items(): + for key, node in list(conf.value.items()): if node.is_ignored(): continue try: diff --git a/lib/python/rose/config_editor/__init__.py b/lib/python/rose/config_editor/__init__.py index 1a66cf3383..1ac816f415 100644 --- a/lib/python/rose/config_editor/__init__.py +++ b/lib/python/rose/config_editor/__init__.py @@ -742,7 +742,7 @@ def load_override_config(sections, my_globals=None): conf = ResourceLocator.default().get_conf().get([section]) if conf is None: continue - for key, node in conf.value.items(): + for key, node in list(conf.value.items()): if node.is_ignored(): continue try: diff --git a/lib/python/rose/config_editor/data.py b/lib/python/rose/config_editor/data.py index 189a719ad5..af5b9b8001 100644 --- a/lib/python/rose/config_editor/data.py +++ b/lib/python/rose/config_editor/data.py @@ -71,10 +71,10 @@ def foreach(self, save=False, skip_latent=False): else: real = self.now latent = self.latent - for section, variables in real.items(): + for section, variables in list(real.items()): yield section, variables if not skip_latent: - for section, variables in latent.items(): + for section, variables in list(latent.items()): yield section, variables def get_all(self, save=False, skip_latent=False, skip_real=False): @@ -87,9 +87,9 @@ def get_all(self, save=False, skip_latent=False, skip_real=False): latent = self.latent all_vars = [] if not skip_real: - all_vars += list(itertools.chain(*real.values())) + all_vars += list(itertools.chain(*list(real.values()))) if not skip_latent: - all_vars += list(itertools.chain(*latent.values())) + all_vars += list(itertools.chain(*list(latent.values()))) return all_vars def get_var(self, section, option, save=False, skip_latent=False): @@ -129,9 +129,9 @@ def get_all(self, save=False, skip_latent=False, skip_real=False): latent = self.latent all_sections = [] if not skip_real: - all_sections += real.values() + all_sections += list(real.values()) if not skip_latent: - all_sections += latent.values() + all_sections += list(latent.values()) return all_sections def get_sect(self, section, save=False, skip_latent=False): @@ -220,9 +220,9 @@ def load(self, top_level_directory, config_obj_dict, elif not config_obj_dict: self.load_top_config(None) else: - self.top_level_name = config_obj_dict.keys()[0] + self.top_level_name = list(config_obj_dict.keys())[0] self.top_level_directory = None - for name, obj in config_obj_dict.items(): + for name, obj in list(config_obj_dict.items()): config_type = config_obj_type_dict.get(name) self.load_config(config_name=name, config=obj, config_type=config_type) @@ -472,8 +472,8 @@ def load_sections_from_config(self, config_name, save=False): else: config = self.config[config_name].config meta_config = self.config[config_name].meta - items = config.value.items() - for section, node in config.value.items(): + items = list(config.value.items()) + for section, node in list(config.value.items()): if not isinstance(node.value, dict): if "" in sect_map: sect_map[""].options.append(section) @@ -486,7 +486,7 @@ def load_sections_from_config(self, config_name, save=False): continue meta_data = self.helper.get_metadata_for_config_id(section, config_name) - options = node.value.keys() + options = list(node.value.keys()) sect_map.update({section: rose.section.Section(section, options, meta_data)}) sect_map[section].comments = list(node.comments) @@ -511,7 +511,7 @@ def load_sections_from_config(self, config_name, save=False): config_name) sect_map.update({"": rose.section.Section("", [], meta_data)}) real_sect_ids.append("") - for setting_id, sect_node in meta_config.value.items(): + for setting_id, sect_node in list(meta_config.value.items()): if sect_node.is_ignored() or isinstance(sect_node.value, str): continue section, option = self.util.get_section_option_from_id(setting_id) @@ -520,7 +520,7 @@ def load_sections_from_config(self, config_name, save=False): continue ignored_reason = {} meta_data = {} - for prop_opt, opt_node in sect_node.value.items(): + for prop_opt, opt_node in list(sect_node.value.items()): if opt_node.is_ignored(): continue meta_data.update({prop_opt: opt_node.value}) @@ -627,7 +627,7 @@ def load_vars_from_config(self, config_name, only_this_section=None, comments=cfg_comments ) ) - id_node_stack = meta_config.value.items() + id_node_stack = list(meta_config.value.items()) while id_node_stack: setting_id, sect_node = id_node_stack.pop(0) if sect_node.is_ignored() or isinstance(sect_node.value, str): @@ -668,7 +668,7 @@ def load_vars_from_config(self, config_name, only_this_section=None, rose.variable.IGNORED_BY_SECTION: rose.config_editor.IGNORED_STATUS_CONFIG} meta_data = {} - for prop_opt, opt_node in sect_node.value.items(): + for prop_opt, opt_node in list(sect_node.value.items()): if opt_node.is_ignored(): continue meta_data.update({prop_opt: opt_node.value}) @@ -760,7 +760,7 @@ def dump_to_internal_config(self, config_name, only_this_ns=None): enabled_state = rose.config.ConfigNode.STATE_NORMAL sections_to_be_dumped = [] if only_this_ns is None: - allowed_sections = set(sect_map.keys() + var_map.keys()) + allowed_sections = set(list(sect_map.keys()) + list(var_map.keys())) else: allowed_sections = self.helper.get_sections_from_namespace( only_this_ns) @@ -817,7 +817,7 @@ def load_meta_path(self, config=None, directory=None): return rose.macro.load_meta_path(config, directory) def clear_meta_lookups(self, config_name): - for ns in self.namespace_meta_lookup.keys(): + for ns in list(self.namespace_meta_lookup.keys()): if (ns.startswith(config_name) and self.util.split_full_ns(self, ns)[0] == config_name): self.namespace_meta_lookup.pop(ns) @@ -842,7 +842,7 @@ def load_meta_config_tree(self, config=None, directory=None, def load_meta_files(self, config_tree): """Load the file paths of files within the metadata directory.""" meta_files = [] - for rel_path, conf_dir in config_tree.files.items(): + for rel_path, conf_dir in list(config_tree.files.items()): meta_files.append(os.path.join(conf_dir, rel_path)) return meta_files @@ -894,7 +894,7 @@ def load_ignored_data(self, config_name): syst_ignored_state = rose.config.ConfigNode.STATE_SYST_IGNORED # Deliberately reset state information in the macro config. for keylist, node in config.walk(): - if len(keylist) == 1 and node.value.keys(): + if len(keylist) == 1 and list(node.value.keys()): # Setting non-empty section info would overwrite options. continue config_for_macro.set(keylist, copy.deepcopy(node.value)) @@ -911,14 +911,14 @@ def load_ignored_data(self, config_name): config_for_macro, meta_config) self.trigger_id_value_lookup.setdefault(config_name, {}) var_id_map = {} - for variables in var_map.values(): + for variables in list(var_map.values()): for variable in variables: var_id_map.update({variable.metadata['id']: variable}) latent_var_id_map = {} - for variables in latent_var_map.values(): + for variables in list(latent_var_map.values()): for variable in variables: latent_var_id_map.update({variable.metadata['id']: variable}) - trig_ids = self.trigger[config_name].trigger_family_lookup.keys() + trig_ids = list(self.trigger[config_name].trigger_family_lookup.keys()) while trig_ids: var_id = trig_ids.pop() var = var_id_map.get(var_id) @@ -945,7 +945,7 @@ def load_ignored_data(self, config_name): id_node_map.update(latent_var_id_map) ignored_dict = self.trigger[config_name].ignored_dict enabled_dict = self.trigger[config_name].enabled_dict - for setting_id, node_inst in id_node_map.items(): + for setting_id, node_inst in list(id_node_map.items()): is_latent = False section, option = self.util.get_section_option_from_id(setting_id) is_section = (option is None) @@ -972,13 +972,13 @@ def load_ignored_data(self, config_name): continue comp_val = node_inst.metadata.get(rose.META_PROP_COMPULSORY) node_is_compulsory = comp_val == rose.META_PROP_VALUE_TRUE - ignored_reasons = node_inst.ignored_reason.keys() + ignored_reasons = list(node_inst.ignored_reason.keys()) if trig_cfg_state == syst_ignored_state: # It should be trigger-ignored. # Doc table: * -> I_t info = ignored_dict.get(setting_id) if rose.variable.IGNORED_BY_SYSTEM not in ignored_reasons: - help_str = ", ".join(info.values()) + help_str = ", ".join(list(info.values())) if rose.variable.IGNORED_BY_USER in ignored_reasons: # It is user-ignored but should be trigger-ignored. # Doc table: I_u -> I_t @@ -1057,13 +1057,13 @@ def load_file_metadata(self, config_name, section_name=None): config = self.config[config_name].config meta_config = self.config[config_name].meta file_sections = [] - for section, sect_node in config.value.items(): + for section, sect_node in list(config.value.items()): if not isinstance(sect_node.value, dict): continue if not sect_node.is_ignored() and section.startswith("file:"): file_sections.append(section) duplicate_file_sections = [] - for meta_id, sect_node in meta_config.value.items(): + for meta_id, sect_node in list(meta_config.value.items()): section, option = self.util.get_section_option_from_id(meta_id) if option is None: if not isinstance(sect_node.value, dict): @@ -1081,7 +1081,7 @@ def load_file_metadata(self, config_name, section_name=None): if base_section in duplicate_file_sections: file_sections.remove(section) file_ids = [] - for setting_id, sect_node in meta_config.value.items(): + for setting_id, sect_node in list(meta_config.value.items()): # The following 'wildcard-esque' id is an exception. # Wildcards are not supported in Rose metadata. if not sect_node.is_ignored() and setting_id.startswith("file:*="): @@ -1093,7 +1093,7 @@ def load_file_metadata(self, config_name, section_name=None): section.replace("file:", "", 1)) for file_entry in file_ids: sect_node = meta_config.get([file_entry]) - for meta_prop, opt_node in sect_node.value.items(): + for meta_prop, opt_node in list(sect_node.value.items()): if opt_node.is_ignored(): continue prop_val = opt_node.value @@ -1143,7 +1143,7 @@ def load_metadata_for_namespaces(self, config_name): """Load namespace metadata, e.g. namespace titles.""" config_data = self.config[config_name] meta_config = config_data.meta - for setting_id, sect_node in meta_config.value.items(): + for setting_id, sect_node in list(meta_config.value.items()): if sect_node.is_ignored(): continue section, option = self.util.get_section_option_from_id( @@ -1174,7 +1174,7 @@ def load_metadata_for_namespaces(self, config_name): namespace = config_name self.namespace_meta_lookup.setdefault(namespace, {}) ns_metadata = self.namespace_meta_lookup[namespace] - for option, opt_node in sect_node.value.items(): + for option, opt_node in list(sect_node.value.items()): if opt_node.is_ignored(): continue value = meta_config[setting_id][option].value @@ -1220,7 +1220,7 @@ def load_metadata_for_namespaces(self, config_name): # Note: rogue-variable section metadata can be overridden. metadata = self.helper.get_metadata_for_config_id(ns_section, config_name) - for key, value in metadata.items(): + for key, value in list(metadata.items()): if (ns_section not in default_ns_sections.get(ns, []) and key in [rose.META_PROP_TITLE, rose.META_PROP_SORT_KEY, rose.META_PROP_DESCRIPTION]): @@ -1236,7 +1236,7 @@ def load_metadata_for_namespaces(self, config_name): else: ns_metadata.setdefault(key, value) self.load_namespace_has_sub_data(config_name) - for config_name in self.config.keys(): + for config_name in list(self.config.keys()): icon_path = self.helper.get_icon_path_for_config(config_name) self.namespace_meta_lookup.setdefault(config_name, {}) self.namespace_meta_lookup[config_name].setdefault( @@ -1265,7 +1265,7 @@ def load_namespace_has_sub_data(self, config_name=None): ns_hierarchy.setdefault(parent_ns, []) ns_hierarchy[parent_ns].append(ns) if config_name is None: - configs = self.config.keys() + configs = list(self.config.keys()) else: configs = [config_name] # File root pages have summary data for files. @@ -1275,7 +1275,7 @@ def load_namespace_has_sub_data(self, config_name=None): self.namespace_meta_lookup[file_root_ns].setdefault( "has_sub_data", True) # Duplicate root pages have summary data for their members. - for ns, prop_map in self.namespace_meta_lookup.items(): + for ns, prop_map in list(self.namespace_meta_lookup.items()): if config_name is not None and not ns.startswith(config_name): continue if (rose.META_PROP_DUPLICATE in prop_map and diff --git a/lib/python/rose/config_editor/data_helper.py b/lib/python/rose/config_editor/data_helper.py index 7b1e958d9f..540819b528 100644 --- a/lib/python/rose/config_editor/data_helper.py +++ b/lib/python/rose/config_editor/data_helper.py @@ -151,11 +151,11 @@ def get_section_data_for_namespace(self, ns): config_name = self.util.split_full_ns(self.data, ns)[0] config_data = self.data.config[config_name] real_sections = [] - for section, sect_data in config_data.sections.now.items(): + for section, sect_data in list(config_data.sections.now.items()): if section in allowed_sections: real_sections.append(sect_data) latent_sections = [] - for section, sect_data in config_data.sections.latent.items(): + for section, sect_data in list(config_data.sections.latent.items()): if section in allowed_sections: latent_sections.append(sect_data) return real_sections, latent_sections @@ -165,11 +165,11 @@ def get_sub_data_for_namespace(self, ns, from_saved=False): sub_data = {"sections": {}, "variables": {}} config_name = self.util.split_full_ns(self.data, ns)[0] config_data = self.data.config[config_name] - for sect, sect_data in config_data.sections.now.items(): + for sect, sect_data in list(config_data.sections.now.items()): sect_ns = sect_data.metadata["full_ns"] if sect_ns.startswith(ns): sub_data["sections"].update({sect: sect_data}) - for sect, variables in config_data.vars.now.items(): + for sect, variables in list(config_data.vars.now.items()): for variable in variables: if variable.metadata["full_ns"].startswith(ns): sub_data["variables"].setdefault(sect, []) @@ -272,7 +272,7 @@ def get_ns_is_default(self, namespace): def get_all_namespaces(self, only_this_config=None): """Return all unique namespaces.""" - nses = self.data.namespace_meta_lookup.keys() + nses = list(self.data.namespace_meta_lookup.keys()) if only_this_config is not None: nses = [n for n in nses if n.startswith(only_this_config)] return nses @@ -283,12 +283,12 @@ def get_missing_sections(self, config_name=None): if config_name is not None: config_names = [config_name] else: - config_names = self.data.config.keys() + config_names = list(self.data.config.keys()) for config_name in config_names: section_store = self.data.config[config_name].sections miss_sections = [] - real_sections = section_store.now.keys() - for section in section_store.latent.keys(): + real_sections = list(section_store.now.keys()) + for section in list(section_store.latent.keys()): if section not in real_sections: miss_sections.append(section) for section in self.data.config[config_name].vars.latent: @@ -373,7 +373,7 @@ def get_ignored_sections(self, namespace, get_enabled=False): config_name = self.util.split_full_ns(self.data, namespace)[0] config_data = self.data.config[config_name] if namespace == config_name: - sections = config_data.sections.now.keys() + sections = list(config_data.sections.now.keys()) else: sections = self.get_sections_from_namespace(namespace) return_sections = [] @@ -393,7 +393,7 @@ def get_latent_sections(self, namespace): config_name = self.util.split_full_ns(self.data, namespace)[0] config_data = self.data.config[config_name] if namespace == config_name: - sections = config_data.sections.now.keys() + sections = list(config_data.sections.now.keys()) else: sections = self.get_sections_from_namespace(namespace) return_sections = [] @@ -453,7 +453,7 @@ def get_ns_ignored_status(self, namespace): object_statuses = default_section_statuses else: object_statuses = variable_statuses - status_counts = object_statuses.items() + status_counts = list(object_statuses.items()) status_counts.sort(lambda x, y: cmp(x[1], y[1])) if not status_counts: cache[namespace] = status diff --git a/lib/python/rose/config_editor/keywidget.py b/lib/python/rose/config_editor/keywidget.py index 6b1bfdf022..25cad73e75 100644 --- a/lib/python/rose/config_editor/keywidget.py +++ b/lib/python/rose/config_editor/keywidget.py @@ -96,7 +96,7 @@ def __init__(self, variable, var_ops, launch_help_func, update_func, event_box.connect('button-press-event', self.handle_launch_help) self.update_comment_display() self.entry.show() - for key, value in self.show_modes.items(): + for key, value in list(self.show_modes.items()): if key not in [rose.config_editor.SHOW_MODE_CUSTOM_DESCRIPTION, rose.config_editor.SHOW_MODE_CUSTOM_HELP, rose.config_editor.SHOW_MODE_CUSTOM_TITLE]: diff --git a/lib/python/rose/config_editor/main.py b/lib/python/rose/config_editor/main.py index 7a51390c03..51f6ef8b49 100644 --- a/lib/python/rose/config_editor/main.py +++ b/lib/python/rose/config_editor/main.py @@ -406,12 +406,12 @@ def generate_toolbar(self): self.toolbar.set_widget_sensitive( rose.config_editor.TOOLBAR_SUITE_GCONTROL, any(c.config_type == rose.TOP_CONFIG_NAME - for c in self.data.config.values())) + for c in list(self.data.config.values()))) self.toolbar.set_widget_sensitive( rose.config_editor.TOOLBAR_VIEW_OUTPUT, any(c.config_type == rose.TOP_CONFIG_NAME - for c in self.data.config.values())) + for c in list(self.data.config.values()))) def generate_menubar(self): """Link in the menu functionality and accelerators.""" @@ -608,7 +608,7 @@ def generate_menubar(self): )) self.main_handle.load_macro_menu(self.menubar) if not any(c.config_type == rose.TOP_CONFIG_NAME - for c in self.data.config.values()): + for c in list(self.data.config.values())): self.menubar.uimanager.get_widget( "/TopMenuBar/Tools/Run Suite").set_sensitive(False) self.update_bar_widgets() @@ -664,7 +664,7 @@ def generate_status_bar(self): def handle_load_all(self, *args): """Handle a request to load all preview configurations.""" load_these = [] - for item in self.data.config.keys(): + for item in list(self.data.config.keys()): if self.data.config[item].is_preview: load_these.append(item) load_these.sort() @@ -781,7 +781,7 @@ def make_page(self, namespace_name): if no_mod_name != no_num_name: # There's a modifier in the section name. ok_names.append(no_num_name) - for section, variables in config_data.vars.now.items(): + for section, variables in list(config_data.vars.now.items()): if not section.startswith(rose.SUB_CONFIG_FILE_DIR): continue for variable in variables: @@ -1160,7 +1160,7 @@ def save_to_file(self, only_config_name=None, check_on_save=False): """Dump the component configurations in memory to disk.""" if only_config_name is None: config_names = [] - for config_name in self.data.config.keys(): + for config_name in list(self.data.config.keys()): if not self.data.config[config_name].is_preview: config_names.append(config_name) else: @@ -1245,19 +1245,19 @@ def save_to_file(self, only_config_name=None, check_on_save=False): config_data.save_config = new_save_config config_vars.save.clear() config_vars.latent_save.clear() - for section, variables in config_vars.now.items(): + for section, variables in list(config_vars.now.items()): config_vars.save.update({section: []}) for variable in variables: config_vars.save[section].append(variable.copy()) - for section, variables in config_vars.latent.items(): + for section, variables in list(config_vars.latent.items()): config_vars.latent_save.update({section: []}) for variable in variables: config_vars.latent_save[section].append(variable.copy()) config_sections.save.clear() config_sections.latent_save.clear() - for section, data in config_sections.now.items(): + for section, data in list(config_sections.now.items()): config_sections.save.update({section: data.copy()}) - for section, data in config_sections.latent.items(): + for section, data in list(config_sections.latent.items()): config_sections.latent_save.update({section: data.copy()}) self.data.saved_config_names = set(self.data.config.keys()) # Update open pages. @@ -1271,7 +1271,7 @@ def save_to_file(self, only_config_name=None, check_on_save=False): def output_config_objects(self, only_config_name=None): """Return a dict of config name - object pairs from this session.""" if only_config_name is None: - config_names = self.data.config.keys() + config_names = list(self.data.config.keys()) else: config_names = [only_config_name] return_dict = {} @@ -1345,7 +1345,7 @@ def _remove_config(self, config_name, meta=None): page_window = self.tab_windows[tab_nses.index(name)] page_window.destroy() self.group_ops.remove_sections(config_name, - config_data.sections.now.keys()) + list(config_data.sections.now.keys())) if dirpath is not None: try: shutil.rmtree(dirpath) @@ -1378,7 +1378,7 @@ def _get_menu_widget(self, suffix): def _has_preview_apps(self): """Return whether any configurations are currently just previews.""" - for item in self.data.config.keys(): + for item in list(self.data.config.keys()): if self.data.config[item].is_preview: return True else: @@ -1443,7 +1443,7 @@ def refresh_metadata(self, metadata_off=False, only_this_config=None): self._get_menu_widget('/Reload metadata').set_sensitive( not self.metadata_off) if only_this_config is None: - configs = self.data.config.keys() + configs = list(self.data.config.keys()) else: configs = [only_this_config] for config_name in configs: @@ -1654,7 +1654,7 @@ def get_found_page_and_id(self, expression, start_page): ns_cmp = lambda x, y: (y == current_ns) - (x == current_ns) name_cmp = lambda x, y: (y == current_name) - (x == current_name) id_cmp = lambda v, w: cmp(v.metadata['id'], w.metadata['id']) - config_keys = self.data.config.keys() + config_keys = list(self.data.config.keys()) config_keys.sort() config_keys.sort(name_cmp) for config_name in config_keys: @@ -1673,7 +1673,7 @@ def get_found_page_and_id(self, expression, start_page): if reg_find(variable.name) or reg_find(variable.value): found_ns_vars.setdefault(ns, []) found_ns_vars[ns].append(variable) - ns_list = found_ns_vars.keys() + ns_list = list(found_ns_vars.keys()) ns_list.sort() ns_list.sort(ns_cmp) for ns in ns_list: @@ -1916,7 +1916,7 @@ def spawn_window(config_directory_path=None, debug_mode=False, try: ctrl.view_page(namespace) except Exception: - print >> sys.stderr, 'could not open ' + namespace + print('could not open ' + namespace, file=sys.stderr) # expand namespace in nav_panel path = ctrl.nav_panel.get_path_from_names(namespace.split('/')) if path: diff --git a/lib/python/rose/config_editor/menu.py b/lib/python/rose/config_editor/menu.py index 62974f5798..4be0552d49 100644 --- a/lib/python/rose/config_editor/menu.py +++ b/lib/python/rose/config_editor/menu.py @@ -277,7 +277,7 @@ def set_accelerators(self, accel_dict): key_list = [] mod_list = [] action_list = [] - for key_press, accel_func in accel_dict.items(): + for key_press, accel_func in list(accel_dict.items()): key, mod = gtk.accelerator_parse(key_press) self.accelerators.lookup[str(key) + str(mod)] = accel_func self.accelerators.connect_group( @@ -411,7 +411,7 @@ def destroy(self, *args): def check_all_extra(self): """Check fail-if, warn-if, and run all validator macros.""" - for config_name in self.data.config.keys(): + for config_name in list(self.data.config.keys()): if not self.data.config[config_name].is_preview: self.update_config(config_name) num_errors = self.check_fail_rules(configs_updated=True) @@ -430,7 +430,7 @@ def check_all_extra(self): def check_fail_rules(self, configs_updated=False): """Check the fail-if and warn-if conditions of the configurations.""" if not configs_updated: - for config_name in self.data.config.keys(): + for config_name in list(self.data.config.keys()): if not self.data.config[config_name].is_preview: self.update_config(config_name) macro = rose.macros.rule.FailureRuleChecker() @@ -490,7 +490,7 @@ def load_page_menu(self, menubar, add_menuitem, current_page): def load_macro_menu(self, menubar): """Refresh the menu dealing with custom macro launches.""" menubar.clear_macros() - config_keys = self.data.config.keys() + config_keys = list(self.data.config.keys()) config_keys.sort() tuple_sorter = lambda x, y: cmp(x[0], y[0]) for config_name in config_keys: @@ -520,9 +520,9 @@ def inspect_custom_macro(self, macro_meth): def handle_graph(self): """Handle a graph metadata request.""" config_sect_dict = {} - for config_name in self.data.config.keys(): + for config_name in list(self.data.config.keys()): config_data = self.data.config[config_name] - config_sect_dict[config_name] = config_data.sections.now.keys() + config_sect_dict[config_name] = list(config_data.sections.now.keys()) config_sect_dict[config_name].sort(rose.config.sort_settings) config_name, section = self.mainwindow.launch_graph_dialog( config_sect_dict) @@ -537,7 +537,7 @@ def handle_graph(self): def check_entry_value(self, entry_widget, dialog, entries, labels, optionals): is_valid = True - for k, entry in entries.items(): + for k, entry in list(entries.items()): this_is_valid = True try: new_val = ast.literal_eval(entry.get_text()) @@ -555,7 +555,7 @@ def check_entry_value(self, entry_widget, dialog, entries, return def handle_macro_entry_activate(self, entry_widget, dialog, entries): - for k, entry in entries.items(): + for k, entry in list(entries.items()): try: ast.literal_eval(entry.get_text()) except (ValueError, EOFError, SyntaxError): @@ -581,10 +581,10 @@ def override_macro_defaults(self, optionals, methname): None) dialog.set_markup('Specify overrides for macro arguments:') dialog.set_title(methname) - table = gtk.Table(len(optionals.items()), 2, False) + table = gtk.Table(len(list(optionals.items())), 2, False) dialog.vbox.add(table) - for i in range(len(optionals.items())): - k, v = optionals.items()[i] + for i in range(len(list(optionals.items()))): + k, v = list(optionals.items())[i] label = gtk.Label(str(k) + ":") entry = gtk.Entry() if isinstance(v, str): @@ -608,7 +608,7 @@ def override_macro_defaults(self, optionals, methname): dialog.destroy() else: res = {} - for k, box in entries.items(): + for k, box in list(entries.items()): res[k] = ast.literal_eval(box.get_text()) dialog.destroy() return res @@ -878,7 +878,7 @@ def _report_macro_validation(self, config_name, macro_name, num_errors): def handle_upgrade(self, only_this_config_name=None): """Run the upgrade manager for this suite.""" config_dict = {} - for config_name in self.data.config.keys(): + for config_name in list(self.data.config.keys()): config_data = self.data.config[config_name] if config_data.is_preview: continue @@ -997,7 +997,7 @@ def run_suite(self, args=None, **kwargs): """Run the suite, if possible.""" if not isinstance(args, list): args = [] - for key, value in kwargs.items(): + for key, value in list(kwargs.items()): args.extend([key, value]) rose.gtk.run.run_suite(*args) return False @@ -1005,7 +1005,7 @@ def run_suite(self, args=None, **kwargs): def transform_default(self, only_this_config=None): """Run the Rose built-in transformer macros.""" if (only_this_config is not None and - only_this_config in self.data.config.keys()): + only_this_config in list(self.data.config.keys())): config_keys = [only_this_config] text = rose.config_editor.DIALOG_LABEL_AUTOFIX else: diff --git a/lib/python/rose/config_editor/menuwidget.py b/lib/python/rose/config_editor/menuwidget.py index 686cfe935e..f4cc7ec0b4 100644 --- a/lib/python/rose/config_editor/menuwidget.py +++ b/lib/python/rose/config_editor/menuwidget.py @@ -90,7 +90,7 @@ def load_contents(self): option_ui_middle = ( option_ui_middle.replace("", '')) var_type = variable.metadata.get(rose.META_PROP_TYPE, '') - var_values = variable.metadata.get(rose.META_PROP_VALUES, range(2)) + var_values = variable.metadata.get(rose.META_PROP_VALUES, list(range(2))) error_types = rose.config_editor.WARNING_TYPES_IGNORE if (set(error_types) & set(variable.error.keys()) or set(error_types) & set(variable.warning.keys()) or @@ -181,12 +181,12 @@ def _set_hover_over(self, variable): hover_string = 'Variable options' if variable.warning: hover_string = rose.config_editor.VAR_MENU_TIP_WARNING - for warn, warn_info in variable.warning.items(): + for warn, warn_info in list(variable.warning.items()): hover_string += "(" + warn + "): " + warn_info + '\n' hover_string = hover_string.rstrip('\n') if variable.error: hover_string = rose.config_editor.VAR_MENU_TIP_ERROR - for err, err_info in variable.error.items(): + for err, err_info in list(variable.error.items()): hover_string += "(" + err + "): " + err_info + '\n' hover_string = hover_string.rstrip('\n') if self.is_ghost: @@ -211,8 +211,8 @@ def _popup_option_menu(self, option_ui, actions, button, time): lambda b: self.trigger_remove()) edit_item = uimanager.get_widget('/Options/Edit') edit_item.connect("activate", self.launch_edit) - errors = self.my_variable.error.keys() - warnings = self.my_variable.warning.keys() + errors = list(self.my_variable.error.keys()) + warnings = list(self.my_variable.warning.keys()) ns = self.my_variable.metadata["full_ns"] search_function = lambda i: self.var_ops.search_for_var(ns, i) dialog_func = rose.gtk.dialog.run_hyperlink_dialog diff --git a/lib/python/rose/config_editor/nav_controller.py b/lib/python/rose/config_editor/nav_controller.py index 7b5b3737fb..39d95be136 100644 --- a/lib/python/rose/config_editor/nav_controller.py +++ b/lib/python/rose/config_editor/nav_controller.py @@ -65,7 +65,7 @@ def reload_namespace_tree(self, only_this_namespace=None, rose.config_editor.SHOW_MODE_LATENT] # Reload the information into the tree. if only_this_config is None: - configs = self.data.config.keys() + configs = list(self.data.config.keys()) configs.sort(rose.config.sort_settings) configs.sort( lambda x, y: cmp( diff --git a/lib/python/rose/config_editor/nav_panel.py b/lib/python/rose/config_editor/nav_panel.py index cabff48437..263cd08b4e 100644 --- a/lib/python/rose/config_editor/nav_panel.py +++ b/lib/python/rose/config_editor/nav_panel.py @@ -188,7 +188,7 @@ def load_tree_stack(self, row, namespace_subtree): self.visible_iter_map = {} if row is None: self.data_store.clear() - initials = namespace_subtree.items() + initials = list(namespace_subtree.items()) initials.sort(self.sort_tree_items) stack = [] if row is None: @@ -219,7 +219,7 @@ def load_tree_stack(self, row, namespace_subtree): new_keylist = keylist + [key] name_iter_map["/".join(new_keylist)] = new_row if type(value) is dict: - newer_initials = value.items() + newer_initials = list(value.items()) newer_initials.sort(self.sort_tree_items) for x in newer_initials: stack.append([new_row] + [list(new_keylist)] + list(x)) diff --git a/lib/python/rose/config_editor/nav_panel_menu.py b/lib/python/rose/config_editor/nav_panel_menu.py index b18e598e32..7a4f89c640 100644 --- a/lib/python/rose/config_editor/nav_panel_menu.py +++ b/lib/python/rose/config_editor/nav_panel_menu.py @@ -70,8 +70,8 @@ def add_dialog(self, base_ns): else: help_str = sections[0] help_str = help_str.split(':', 1)[0] - for config_section in (config_data.sections.now.keys() + - config_data.sections.latent.keys()): + for config_section in (list(config_data.sections.now.keys()) + + list(config_data.sections.latent.keys())): if config_section.startswith(help_str + ":"): help_str = help_str + ":" else: @@ -94,7 +94,7 @@ def ask_is_preview(self, base_ns): config_data = self.data.config[config_name] return config_data.is_preview except KeyError: - print config_name + print(config_name) return False def copy_request(self, base_ns, new_section=None, skip_update=False): @@ -112,7 +112,7 @@ def copy_request(self, base_ns, new_section=None, skip_update=False): def create_request(self): """Handle a create configuration request.""" if not any(v.config_type == rose.TOP_CONFIG_NAME - for v in self.data.config.values()): + for v in list(self.data.config.values())): text = rose.config_editor.WARNING_APP_CONFIG_CREATE title = rose.config_editor.WARNING_APP_CONFIG_CREATE_TITLE rose.gtk.dialog.run_dialog(rose.gtk.dialog.DIALOG_TYPE_ERROR, @@ -129,7 +129,7 @@ def create_request(self): def ignore_request(self, base_ns, is_ignored): """Handle an ignore or enable section request.""" - config_names = self.data.config.keys() + config_names = list(self.data.config.keys()) if base_ns is not None and '/' in base_ns: config_name, subsp = self.util.split_full_ns(self.data, base_ns) prefer_name_sections = { @@ -233,7 +233,7 @@ def graph_request(self, namespace): def remove_request(self, base_ns): """Handle a delete section request.""" - config_names = self.data.config.keys() + config_names = list(self.data.config.keys()) if base_ns is not None and '/' in base_ns: config_name, subsp = self.util.split_full_ns(self.data, base_ns) prefer_name_sections = { @@ -245,7 +245,7 @@ def remove_request(self, base_ns): sorter = rose.config.sort_settings for config_name in config_names: config_data = self.data.config[config_name] - config_sect_dict[config_name] = config_data.sections.now.keys() + config_sect_dict[config_name] = list(config_data.sections.now.keys()) config_sect_dict[config_name].sort(rose.config.sort_settings) if config_name in prefer_name_sections: prefer_name_sections[config_name].sort( @@ -282,7 +282,7 @@ def rename_dialog(self, base_ns): config_sect_dict = {} for config_name in self.data.config: config_data = self.data.config[config_name] - config_sect_dict[config_name] = config_data.sections.now.keys() + config_sect_dict[config_name] = list(config_data.sections.now.keys()) config_sect_dict[config_name].sort(rose.config.sort_settings) if config_name in prefer_name_sections: prefer_name_sections[config_name].sort( @@ -343,7 +343,7 @@ def popup_panel_menu(self, base_ns, event): if self.data.config[config_name].is_preview: return False cloneable = self.is_ns_duplicate(namespace) - is_top = (namespace in self.data.config.keys()) + is_top = (namespace in list(self.data.config.keys())) is_fixable = bool(self.get_ns_errors(namespace)) has_content = self.data.helper.is_ns_content(namespace) is_unsaved = self.data.helper.get_config_has_unsaved_changes( diff --git a/lib/python/rose/config_editor/ops/group.py b/lib/python/rose/config_editor/ops/group.py index 6b03b2c7e4..c99c48afa3 100644 --- a/lib/python/rose/config_editor/ops/group.py +++ b/lib/python/rose/config_editor/ops/group.py @@ -259,7 +259,7 @@ def add_section_with_options(self, config_name, new_section_name, (var.metadata.get(rose.META_PROP_COMPULSORY) == rose.META_PROP_VALUE_TRUE)): self.var_ops.add_var(var, skip_update=True) - for opt_name, value in opt_map.items(): + for opt_name, value in list(opt_map.items()): var_id = self.util.get_id_from_section_option( new_section_name, opt_name) metadata = self.data.helper.get_metadata_for_config_id( @@ -287,8 +287,8 @@ def copy_section(self, config_name, section, new_section=None, section_base = re.sub('(.*)\(\w+\)$', r"\1", section) existing_sections = [] clone_vars = [] - existing_sections = config_data.vars.now.keys() - existing_sections.extend(config_data.sections.now.keys()) + existing_sections = list(config_data.vars.now.keys()) + existing_sections.extend(list(config_data.sections.now.keys())) for variable in config_data.vars.now.get(section, []): clone_vars.append(variable.copy()) if new_section is None: diff --git a/lib/python/rose/config_editor/ops/section.py b/lib/python/rose/config_editor/ops/section.py index 9d99d340c1..9c28be39cf 100644 --- a/lib/python/rose/config_editor/ops/section.py +++ b/lib/python/rose/config_editor/ops/section.py @@ -165,7 +165,7 @@ def ignore_section(self, config_name, section, is_ignored, # The section must not be justifiably triggered ignored. ign_errors = [e for e in rose.config_editor.WARNING_TYPES_IGNORE if e != rose.config_editor.WARNING_TYPE_ENABLED] - my_errors = sect_data.error.keys() + my_errors = list(sect_data.error.keys()) if (not override and (rose.variable.IGNORED_BY_SYSTEM in sect_data.ignored_reason) and @@ -226,7 +226,7 @@ def remove_section(self, config_name, section, skip_update=False, config_data.vars.now.pop(section) namespace = old_section_data.metadata["full_ns"] ns_list = [namespace] - for ns, values in self.__data.namespace_meta_lookup.items(): + for ns, values in list(self.__data.namespace_meta_lookup.items()): sections = values.get('sections') if sections == [section]: if ns not in ns_list: diff --git a/lib/python/rose/config_editor/ops/variable.py b/lib/python/rose/config_editor/ops/variable.py index ef361e73b7..6b15cb65c3 100644 --- a/lib/python/rose/config_editor/ops/variable.py +++ b/lib/python/rose/config_editor/ops/variable.py @@ -154,7 +154,7 @@ def remove_var(self, variable, skip_update=False, skip_undo=False): def fix_var_ignored(self, variable): """Fix any variable ignore state errors.""" - ignored_reasons = variable.ignored_reason.keys() + ignored_reasons = list(variable.ignored_reason.keys()) new_reason_dict = {} # Enable, by default. old_reason = variable.ignored_reason.copy() if rose.variable.IGNORED_BY_SECTION in old_reason: @@ -218,10 +218,10 @@ def set_var_ignored(self, variable, new_reason_dict=None, override=False, if rose.config_editor.WARNING_TYPE_NOT_TRIGGER in variable.error: variable.error.pop( rose.config_editor.WARNING_TYPE_NOT_TRIGGER) - my_ignored_keys = variable.ignored_reason.keys() + my_ignored_keys = list(variable.ignored_reason.keys()) if rose.variable.IGNORED_BY_SECTION in my_ignored_keys: my_ignored_keys.remove(rose.variable.IGNORED_BY_SECTION) - old_ignored_keys = old_reason.keys() + old_ignored_keys = list(old_reason.keys()) if rose.variable.IGNORED_BY_SECTION in old_ignored_keys: old_ignored_keys.remove(rose.variable.IGNORED_BY_SECTION) if len(my_ignored_keys) > len(old_ignored_keys): diff --git a/lib/python/rose/config_editor/page.py b/lib/python/rose/config_editor/page.py index e79172b42f..e3ef9ce66d 100644 --- a/lib/python/rose/config_editor/page.py +++ b/lib/python/rose/config_editor/page.py @@ -686,14 +686,14 @@ def validate_errors(self, variable_id=None): if variable_id is None: bad_list = [] for variable in self.panel_data + self.ghost_data: - bad_list += variable.error.items() + bad_list += list(variable.error.items()) return bad_list else: for variable in self.panel_data + self.ghost_data: if variable.metadata.get('id') == variable_id: if variable.error == {}: return None - return variable.error.items() + return list(variable.error.items()) return None def choose_focus(self, focus_variable=None): @@ -918,7 +918,7 @@ def update_ignored(self, no_refresh=False): if ([e in target.errors for e in relevant_errs] != [e in target.variable.error for e in relevant_errs]): refresh_list.append(target.variable.metadata['id']) - target.errors = target.variable.error.keys() + target.errors = list(target.variable.error.keys()) target_widgets_done.append(target) if hasattr(self.main_container, "update_ignored"): self.main_container.update_ignored() @@ -930,7 +930,7 @@ def update_ignored(self, no_refresh=False): def _check_show_ignored_reason(self, ignored_reason): """Return whether we should show this state.""" mode = self.show_modes - if ignored_reason.keys() == [rose.variable.IGNORED_BY_USER]: + if list(ignored_reason.keys()) == [rose.variable.IGNORED_BY_USER]: return (mode[rose.config_editor.SHOW_MODE_IGNORED] or mode[rose.config_editor.SHOW_MODE_USER_IGNORED]) return mode[rose.config_editor.SHOW_MODE_IGNORED] @@ -1170,7 +1170,7 @@ def _get_page_info_widgets(self): label_list.append(latent_label) # This adds error notification for sections. for sect_data in self.sections + self.latent_sections: - for err, info in sect_data.error.items(): + for err, info in list(sect_data.error.items()): error_button = rose.gtk.util.CustomButton( stock_id=gtk.STOCK_DIALOG_ERROR, as_tool=True, @@ -1181,7 +1181,7 @@ def _get_page_info_widgets(self): error_label.show() button_list.append(error_button) label_list.append(error_label) - if self.custom_macros.items(): + if list(self.custom_macros.items()): macro_button = rose.gtk.util.CustomButton( label=rose.config_editor.LABEL_PAGE_MACRO_BUTTON, stock_id=gtk.STOCK_EXECUTE, diff --git a/lib/python/rose/config_editor/pagewidget/__init__.py b/lib/python/rose/config_editor/pagewidget/__init__.py index 896a3c061a..7a075aa484 100644 --- a/lib/python/rose/config_editor/pagewidget/__init__.py +++ b/lib/python/rose/config_editor/pagewidget/__init__.py @@ -18,4 +18,4 @@ # along with Rose. If not, see . # ----------------------------------------------------------------------------- -import table +from . import table diff --git a/lib/python/rose/config_editor/panelwidget/__init__.py b/lib/python/rose/config_editor/panelwidget/__init__.py index 68f3553d3b..156c6037d3 100644 --- a/lib/python/rose/config_editor/panelwidget/__init__.py +++ b/lib/python/rose/config_editor/panelwidget/__init__.py @@ -18,5 +18,5 @@ # along with Rose. If not, see . # ----------------------------------------------------------------------------- -import filesystem -import summary_data +from . import filesystem +from . import summary_data diff --git a/lib/python/rose/config_editor/panelwidget/summary_data.py b/lib/python/rose/config_editor/panelwidget/summary_data.py index 9294536903..3a5993feea 100644 --- a/lib/python/rose/config_editor/panelwidget/summary_data.py +++ b/lib/python/rose/config_editor/panelwidget/summary_data.py @@ -174,10 +174,10 @@ def _get_control_widget_hbox(self): def update_tree_model(self): """Construct a data model of other page data.""" - sub_sect_names = self.sections.keys() + sub_sect_names = list(self.sections.keys()) sub_var_names = [] self.var_id_map = {} - for section, variables in self.variables.items(): + for section, variables in list(self.variables.items()): for variable in variables: self.var_id_map[variable.metadata["id"]] = variable data_rows, column_names = self.get_model_data() @@ -338,7 +338,7 @@ def _filter_visible(self, model, iter_): return True for i in range(model.get_n_columns()): col_text = model.get_value(iter_, i) - if isinstance(col_text, basestring) and filt_text in col_text: + if isinstance(col_text, str) and filt_text in col_text: return True child_iter = model.iter_children(iter_) while child_iter is not None: @@ -611,7 +611,7 @@ def add_section(self, section=None, opt_map=None): if section is None: if not self.sections or not self.is_duplicate: return False - section_base = self.sections.keys()[0].rsplit("(", 1)[0] + section_base = list(self.sections.keys())[0].rsplit("(", 1)[0] i = 1 section = section_base + "(" + str(i) + ")" while section in self.sections: @@ -767,10 +767,10 @@ def set_tree_cell_status(self, col, cell, model, row_iter): def get_model_data(self): """Construct a data model of other page data.""" - sub_sect_names = self.sections.keys() + sub_sect_names = list(self.sections.keys()) sub_var_names = [] self.var_id_map = {} - for section, variables in self.variables.items(): + for section, variables in list(self.variables.items()): for variable in variables: self.var_id_map[variable.metadata["id"]] = variable if variable.name not in sub_var_names: @@ -835,7 +835,7 @@ def set_tree_tip(self, view, row_iter, col_index, tip): tip_text += id_data.metadata.get(rose.META_PROP_DESCRIPTION, "") if tip_text: tip_text += "\n" - for key, value in id_data.error.items(): + for key, value in list(id_data.error.items()): tip_text += ( rose.config_editor.SUMMARY_DATA_PANEL_ERROR_TIP.format( key, value)) diff --git a/lib/python/rose/config_editor/plugin/um/widget/stash.py b/lib/python/rose/config_editor/plugin/um/widget/stash.py index 22c0d40f65..dfddd92989 100644 --- a/lib/python/rose/config_editor/plugin/um/widget/stash.py +++ b/lib/python/rose/config_editor/plugin/um/widget/stash.py @@ -175,12 +175,12 @@ def add_cell_renderer_for_value(self, col, col_title): def get_model_data(self): """(Override) Construct a data model of other page data.""" - sub_sect_names = self.sections.keys() + sub_sect_names = list(self.sections.keys()) sub_var_names = [] self.var_id_map = {} section_sort_keys = {} # Apply the correct default sorting (section, item) - for section, variables in self.variables.items(): + for section, variables in list(self.variables.items()): for variable in variables: self.var_id_map[variable.metadata["id"]] = variable if variable.name not in sub_var_names: @@ -208,7 +208,7 @@ def get_model_data(self): while len(section_sort_keys[section]) < 3: section_sort_keys[section].append(None) section_sort_keys[section][2] = variable.value - for section, sort_list in section_sort_keys.items(): + for section, sort_list in list(section_sort_keys.items()): while len(sort_list) < 4: sort_list.append(None) sort_list[3] = section @@ -369,7 +369,7 @@ def set_tree_tip(self, view, row_iter, col_index, tip): tip_text = rose.CONFIG_DELIMITER.join( [section, option, value]) + "\n" if (option in self.OPTION_NL_MAP and - option in self._profile_location_map.keys()): + option in list(self._profile_location_map.keys())): profile_id = self._profile_location_map[option].get(value) if profile_id is not None: profile_sect = self.util.get_section_option_from_id( @@ -378,7 +378,7 @@ def set_tree_tip(self, view, row_iter, col_index, tip): tip_text += id_data.metadata.get(rose.META_PROP_DESCRIPTION, "") if tip_text: tip_text += "\n" - for key, value in id_data.error.items(): + for key, value in list(id_data.error.items()): tip_text += ( rose.config_editor.SUMMARY_DATA_PANEL_ERROR_TIP.format( key, value)) @@ -469,7 +469,7 @@ def generate_package_lookup(self): self._package_lookup = {} self._package_profile_lookup = {} package_profiles = {} - for sect, node in self.package_config.value.items(): + for sect, node in list(self.package_config.value.items()): if not isinstance(node.value, dict) or node.is_ignored(): continue base_sect = sect.rsplit("(", 1)[0] @@ -489,7 +489,7 @@ def generate_package_lookup(self): self._package_lookup[package][profile].append( profile_node.value) continue - for profile, profile_nl in self.OPTION_NL_MAP.items(): + for profile, profile_nl in list(self.OPTION_NL_MAP.items()): if base_sect == profile_nl: name_node = node.get([profile], no_ignore=True) if name_node is not None: @@ -555,7 +555,7 @@ def _handle_activation(self, view, path, column): def _handle_cell_combo_change(self, combo_cell, path_string, new, col_title): # Handle a gtk.CellRendererCombo (variable) value change. - if isinstance(new, basestring): + if isinstance(new, str): new_value = new else: new_value = combo_cell.get_property("model").get_value(new, 0) @@ -618,7 +618,7 @@ def _get_request_lookup(self): def _get_request_changes(self): # Return a list of request indices with changes. changed_requests = {} - for section, sect_data in self.sections.items(): + for section, sect_data in list(self.sections.items()): changes = self.sect_ops.get_section_changes(sect_data) if changes: changed_requests.update({section: changes}) @@ -718,11 +718,11 @@ def _update_available_profiles(self): # Retrieve which profiles (namelists like domain) are available. self._available_profile_map = {} self._profile_location_map = {} - ok_var_names = self.OPTION_NL_MAP.keys() - ok_sect_names = self.OPTION_NL_MAP.values() + ok_var_names = list(self.OPTION_NL_MAP.keys()) + ok_sect_names = list(self.OPTION_NL_MAP.values()) for name in ok_var_names: self._available_profile_map[name] = [] - for id_, value in self.sub_ops.get_var_id_values().items(): + for id_, value in list(self.sub_ops.get_var_id_values().items()): section, option = self.util.get_section_option_from_id(id_) if (option in ok_var_names and any(section.startswith(n) for n in ok_sect_names)): @@ -730,13 +730,13 @@ def _update_available_profiles(self): self._profile_location_map[option].update({value: id_}) self._available_profile_map.setdefault(option, []) self._available_profile_map[option].append(value) - for profile_names in self._available_profile_map.values(): + for profile_names in list(self._available_profile_map.values()): profile_names.sort() def _package_add(self, package): # Add a package of new requests, and profiles if needed. sections_for_adding = [] - for sect_type, values in self._package_lookup[package].items(): + for sect_type, values in list(self._package_lookup[package].items()): if sect_type == self.STREQ_NL_BASE: sections_for_adding.extend(values) else: @@ -750,7 +750,7 @@ def _package_add(self, package): node = self.package_config.get([section], no_ignore=True) if node is None or not isinstance(node.value, dict): continue - for opt, node in node.value.items(): + for opt, node in list(node.value.items()): opt_name_values.update({opt: node.value}) if section not in self.sections: self.sub_ops.add_section(section, opt_map=opt_name_values) @@ -759,7 +759,7 @@ def _package_menu_launch(self, widget, event): # Create a menu below the widget for package actions. menu = gtk.Menu() packages = {} - for section, vars_ in self.variables.items(): + for section, vars_ in list(self.variables.items()): for var in vars_: if var.name == self.STREQ_NL_PACKAGE_OPT: is_ignored = (rose.variable.IGNORED_BY_USER in @@ -831,7 +831,7 @@ def _packages_remove(self, only_this_package=None): self._update_available_profiles() sections_for_removing = [] profile_streqs = {} - for section, vars_ in self.variables.items(): + for section, vars_ in list(self.variables.items()): for var in vars_: if var.name == self.STREQ_NL_PACKAGE_OPT: if (only_this_package is None or @@ -846,7 +846,7 @@ def _packages_remove(self, only_this_package=None): profile_streqs[var.name][var.value].append(section) streq_remove_list = list(sections_for_removing) for profile_type in profile_streqs: - for name, streq_list in profile_streqs[profile_type].items(): + for name, streq_list in list(profile_streqs[profile_type].items()): if all([s in streq_remove_list for s in streq_list]): # This is only referenced by sections about to be removed. profile_id = self._profile_location_map.get( @@ -861,7 +861,7 @@ def _packages_remove(self, only_this_package=None): def _packages_enable(self, only_this_package=None, disable=False): # Enable or user-ignore requests matching these packages. sections_for_changing = [] - for section, vars_ in self.variables.items(): + for section, vars_ in list(self.variables.items()): for var in vars_: if var.name == self.STREQ_NL_PACKAGE_OPT: if (only_this_package is None or diff --git a/lib/python/rose/config_editor/plugin/um/widget/stash_add.py b/lib/python/rose/config_editor/plugin/um/widget/stash_add.py index 41cb19e39b..a5473a797d 100644 --- a/lib/python/rose/config_editor/plugin/um/widget/stash_add.py +++ b/lib/python/rose/config_editor/plugin/um/widget/stash_add.py @@ -96,7 +96,7 @@ def __init__(self, stash_lookup, request_lookup, # Automatically hide columns which have fixed-value metadata. self._hidden_column_names = [] - for key, metadata in self.stash_meta_lookup.items(): + for key, metadata in list(self.stash_meta_lookup.items()): if "=" in key: continue values_string = metadata.get(rose.META_PROP_VALUES, "0, 1") @@ -173,7 +173,7 @@ def get_model_data_and_columns(self): """Return a list of data tuples and columns""" data_rows = [] columns = ["Section", "Item", "Description", "?", "#"] - sections = self.stash_lookup.keys() + sections = list(self.stash_lookup.keys()) sections.sort(self.sort_util.cmp_) mod_markup = rose.config_editor.SUMMARY_DATA_PANEL_MODIFIED_MARKUP props_excess = [self.STASH_PARSE_DESC_OPT, self.STASH_PARSE_ITEM_OPT, @@ -181,7 +181,7 @@ def get_model_data_and_columns(self): for section in sections: if section == "-1": continue - items = self.stash_lookup[section].keys() + items = list(self.stash_lookup[section].keys()) items.sort(self.sort_util.cmp_) for item in items: data = self.stash_lookup[section][item] @@ -261,7 +261,7 @@ def set_tree_tip(self, treeview, row_iter, col_index, tip): name = "Requests" if stash_request_num != "None": sect_streqs = self.request_lookup.get(stash_section, {}) - streqs = sect_streqs.get(stash_item, {}).keys() + streqs = list(sect_streqs.get(stash_item, {}).keys()) streqs.sort(rose.config.sort_settings) if streqs: value = "\n " + "\n ".join(streqs) @@ -323,7 +323,7 @@ def update_request_info(self, request_lookup=None, while child_iter is not None: num = self._store.get_value(child_iter, num_streqs_index) info = self._store.get_value(child_iter, streq_info_index) - if isinstance(num, basestring) and num.isdigit(): + if isinstance(num, str) and num.isdigit(): num_streq_children += int(num) if info and not streq_info_children: streq_info_children = info @@ -388,8 +388,8 @@ def _filter_visible(self, model, iter_): filt_text = self._filter_widget.get_text() if not filt_text: return True - for col_text in model.get(iter_, *range(len(self.column_names))): - if (isinstance(col_text, basestring) and + for col_text in model.get(iter_, *list(range(len(self.column_names)))): + if (isinstance(col_text, str) and filt_text.lower() in col_text.lower()): return True child_iter = model.iter_children(iter_) @@ -562,7 +562,7 @@ def _popup_tree_menu(self, path, col, event): help_menuitem.connect("activate", self._launch_record_help) help_menuitem.show() menu.append(help_menuitem) - streqs = self.request_lookup.get(section, {}).get(item, {}).keys() + streqs = list(self.request_lookup.get(section, {}).get(item, {}).keys()) if streqs: view_menuitem = gtk.ImageMenuItem(stock_id=gtk.STOCK_FIND) view_menuitem.set_label(label="View...") diff --git a/lib/python/rose/config_editor/stack.py b/lib/python/rose/config_editor/stack.py index ad9fcab4e4..193d7735e3 100644 --- a/lib/python/rose/config_editor/stack.py +++ b/lib/python/rose/config_editor/stack.py @@ -161,7 +161,7 @@ def get_stack_view(self, redo_mode_on=False): cell_text[title] = gtk.CellRendererText() columns[title].pack_start(cell_text[title], expand=True) columns[title].add_attribute(cell_text[title], attribute='markup', - column=len(columns.keys()) - 1) + column=len(list(columns.keys())) - 1) stack_view.append_column(columns[title]) stack_view.show() return stack_view diff --git a/lib/python/rose/config_editor/updater.py b/lib/python/rose/config_editor/updater.py index ef072de689..d9daed3403 100644 --- a/lib/python/rose/config_editor/updater.py +++ b/lib/python/rose/config_editor/updater.py @@ -56,10 +56,10 @@ def namespace_data_is_modified(self, namespace): if config_name not in self.data.saved_config_names: return rose.config_editor.TREE_PANEL_TIP_ADDED_CONFIG section_hashes = [] - for sect, sect_data in config_sections.now.items(): + for sect, sect_data in list(config_sections.now.items()): section_hashes.append(sect_data.to_hashable()) old_section_hashes = [] - for sect, sect_data in config_sections.save.items(): + for sect, sect_data in list(config_sections.save.items()): old_section_hashes.append(sect_data.to_hashable()) if set(section_hashes) ^ set(old_section_hashes): return rose.config_editor.TREE_PANEL_TIP_CHANGED_CONFIG @@ -172,7 +172,7 @@ def update_all(self, only_this_config=None, is_loading=False, self.data.helper.clear_namespace_cached_statuses(ns) if only_this_config is None: - configs = self.data.config.keys() + configs = list(self.data.config.keys()) else: configs = [only_this_config] for config_name in configs: @@ -224,7 +224,7 @@ def update_namespace(self, namespace, are_errors_done=False, self.update_bar_widgets_func() self.update_stack_viewer_if_open() self.update_ns_tree_states(namespace) - if namespace in self.data.config.keys(): + if namespace in list(self.data.config.keys()): self.update_metadata_id(namespace) if not skip_sub_data_update: self.update_ns_sub_data(namespace) @@ -243,7 +243,7 @@ def update_status(self, page, are_errors_done=False, self.update_stack_viewer_if_open() page.update_info() self.update_ns_tree_states(page.namespace) - if page.namespace in self.data.config.keys(): + if page.namespace in list(self.data.config.keys()): self.update_metadata_id(page.namespace) if not skip_sub_data_update: self.update_ns_sub_data(page.namespace) @@ -386,7 +386,7 @@ def update_ignored_statuses(self, namespace): if ns != namespace: # We don't need another update of namespace. self.update_namespace(ns) - for var_id in trig_id_val_dict.keys() + updated_ids: + for var_id in list(trig_id_val_dict.keys()) + updated_ids: var = var_id_map.get(var_id) if var is None: if var_id in trig_id_val_dict: @@ -425,7 +425,7 @@ def update_ignoreds(self, config_name, var_id): triggered_ns_list = [] this_id = var_id nses = [] - for namespace, metadata in self.data.namespace_meta_lookup.items(): + for namespace, metadata in list(self.data.namespace_meta_lookup.items()): this_name = self.util.split_full_ns(self.data, namespace) if this_name != config_name: continue @@ -472,7 +472,7 @@ def update_ignoreds(self, config_name, var_id): # Trigger-ignored sections parents = trigger.ignored_dict.get(section, {}) if parents: - help_text = "; ".join(parents.values()) + help_text = "; ".join(list(parents.values())) else: help_text = rose.config_editor.IGNORED_STATUS_DEFAULT reason.update({rose.variable.IGNORED_BY_SYSTEM: help_text}) @@ -513,7 +513,7 @@ def update_ignoreds(self, config_name, var_id): # Trigger-ignored variables parents = trigger.ignored_dict.get(var_id, {}) if parents: - help_text = "; ".join(parents.values()) + help_text = "; ".join(list(parents.values())) else: help_text = rose.config_editor.IGNORED_STATUS_DEFAULT var.ignored_reason.update( @@ -526,14 +526,14 @@ def update_tree_status(self, page_or_ns, icon_bool=None, icon_type=None): """Update the tree statuses.""" if self.nav_panel is None: return - if isinstance(page_or_ns, basestring): + if isinstance(page_or_ns, str): namespace = page_or_ns config_name = self.util.split_full_ns(self.data, namespace)[0] errors = [] ns_vars, ns_l_vars = self.data.helper.get_data_for_namespace( namespace) for var in ns_vars + ns_l_vars: - errors += var.error.items() + errors += list(var.error.items()) else: namespace = page_or_ns.namespace config_name = self.util.split_full_ns(self.data, namespace)[0] @@ -543,9 +543,9 @@ def update_tree_status(self, page_or_ns, icon_bool=None, icon_type=None): ns_sections = self.data.helper.get_sections_from_namespace(namespace) for section in ns_sections: if section in config_data.sections.now: - errors += config_data.sections.now[section].error.items() + errors += list(config_data.sections.now[section].error.items()) elif section in config_data.sections.latent: - errors += config_data.sections.latent[section].error.items() + errors += list(config_data.sections.latent[section].error.items()) # Set icons. name_tree = namespace.lstrip('/').split('/') @@ -612,7 +612,7 @@ def perform_startup_check(self): def perform_error_check(self, namespace=None, is_loading=False): """Loop through system macros and sum errors.""" - configs = self.data.config.keys() + configs = list(self.data.config.keys()) if namespace is not None: config_name = self.util.split_full_ns(self.data, namespace)[0] @@ -671,8 +671,8 @@ def apply_macro_validation(self, config_name, macro_type, bad_list=None, id_error_dict = {} id_warn_dict = {} if namespace is None: - ok_sections = (config_sections.now.keys() + - config_sections.latent.keys()) + ok_sections = (list(config_sections.now.keys()) + + list(config_sections.latent.keys())) ok_variables = variables else: ok_sections = self.data.helper.get_sections_from_namespace( @@ -700,7 +700,7 @@ def apply_macro_validation(self, config_name, macro_type, bad_list=None, id_warn_dict.update({var.metadata['id']: this_warning}) if not bad_list: self.refresh_ids(config_name, - id_error_dict.keys() + id_warn_dict.keys(), + list(id_error_dict.keys()) + list(id_warn_dict.keys()), is_loading, are_errors_done=is_macro_dynamic) return for bad_report in bad_list: @@ -757,7 +757,7 @@ def apply_macro_validation(self, config_name, macro_type, bad_list=None, # New warning or error. map_.update({setting_id: info}) self.refresh_ids(config_name, - id_error_dict.keys() + id_warn_dict.keys(), + list(id_error_dict.keys()) + list(id_warn_dict.keys()), is_loading, are_errors_done=is_macro_dynamic) diff --git a/lib/python/rose/config_editor/upgrade_controller.py b/lib/python/rose/config_editor/upgrade_controller.py index b472edc949..6f30fed1b4 100644 --- a/lib/python/rose/config_editor/upgrade_controller.py +++ b/lib/python/rose/config_editor/upgrade_controller.py @@ -214,7 +214,7 @@ def _handle_change_cursor(self, view): def _handle_change_version(self, cell, path, new, col_index): iter_ = self.treemodel.get_iter(path) - if isinstance(new, basestring): + if isinstance(new, str): new_value = new else: new_value = cell.get_property("model").get_value(new, 0) diff --git a/lib/python/rose/config_editor/util.py b/lib/python/rose/config_editor/util.py index be874e6489..4b1e62425b 100644 --- a/lib/python/rose/config_editor/util.py +++ b/lib/python/rose/config_editor/util.py @@ -77,7 +77,7 @@ def get_section_option_from_id(self, var_id): def split_full_ns(self, data, full_namespace): """Return the config name and the internal namespace from full ns.""" if full_namespace not in self.full_ns_split_lookup: - for config_name in data.config.keys(): + for config_name in list(data.config.keys()): if config_name == '/' + data.top_level_name: continue if full_namespace.startswith(config_name + '/'): @@ -118,7 +118,7 @@ def launch_node_info_dialog(node, changes, search_function): "\n") text += rose.config_editor.DIALOG_NODE_INFO_DATA try: - att_list = vars(node).items() + att_list = list(vars(node).items()) except TypeError: # vars will fail when __slots__ are used. att_list = node.getattrs() @@ -220,7 +220,7 @@ def _pretty_format_data(data, global_indent=0, indent=4, width=60): delim = rose.config_editor.DIALOG_NODE_INFO_DELIMITER if isinstance(data, dict) and data: text = "" - for key, val in data.items(): + for key, val in list(data.items()): text += "\n" + " " * global_indent sub_prefix = sub_name.format(safe_str(key)) + delim sub_ind0 = global_indent + len(sub_prefix) diff --git a/lib/python/rose/config_editor/valuewidget/__init__.py b/lib/python/rose/config_editor/valuewidget/__init__.py index de20ea4d49..d92dc08160 100644 --- a/lib/python/rose/config_editor/valuewidget/__init__.py +++ b/lib/python/rose/config_editor/valuewidget/__init__.py @@ -21,20 +21,20 @@ import re import rose -import array.entry -import array.mixed -import array.logical -import array.python_list -import array.spaced_list -import booltoggle -import character -import combobox -import files -import intspin -import meta -import radiobuttons -import text -import valuehints +from .array import entry +from .array import mixed +from .array import logical +from .array import python_list +from .array import spaced_list +from . import booltoggle +from . import character +from . import combobox +from . import files +from . import intspin +from . import meta +from . import radiobuttons +from . import text +from . import valuehints class ValueWidgetHook(object): diff --git a/lib/python/rose/config_editor/valuewidget/array/logical.py b/lib/python/rose/config_editor/valuewidget/array/logical.py index 1f5bc46560..6922247a7a 100644 --- a/lib/python/rose/config_editor/valuewidget/array/logical.py +++ b/lib/python/rose/config_editor/valuewidget/array/logical.py @@ -49,8 +49,8 @@ def __init__(self, value, metadata, set_value, hook, arg_str=None): if metadata.get(rose.META_PROP_TYPE) == "boolean": self.allowed_values = [rose.TYPE_BOOLEAN_VALUE_FALSE, rose.TYPE_BOOLEAN_VALUE_TRUE] - self.label_dict = dict(zip(self.allowed_values, - self.allowed_values)) + self.label_dict = dict(list(zip(self.allowed_values, + self.allowed_values))) else: self.allowed_values = [rose.TYPE_LOGICAL_VALUE_FALSE, rose.TYPE_LOGICAL_VALUE_TRUE] diff --git a/lib/python/rose/config_editor/valuewidget/array/mixed.py b/lib/python/rose/config_editor/valuewidget/array/mixed.py index 76b104ea16..ae900b0d26 100644 --- a/lib/python/rose/config_editor/valuewidget/array/mixed.py +++ b/lib/python/rose/config_editor/valuewidget/array/mixed.py @@ -104,7 +104,7 @@ def set_num_rows(self): if self.unlimited: self.num_rows, rem = divmod(len(self.value_array), self.num_cols) self.num_rows += [1, 0][rem == 0] - self.max_rows = sys.maxint + self.max_rows = sys.maxsize else: self.num_rows = int(self.array_length) num, rem = divmod(len(self.value_array), self.num_cols) @@ -323,7 +323,7 @@ def _normalise_width_chars(self, widget): elif hasattr(child, 'get_child'): child_list.append(child.get_child()) i += 1 - for key, value in max_width.items(): + for key, value in list(max_width.items()): if value < self.MIN_WIDTH_CHARS: max_width[key] = self.MIN_WIDTH_CHARS # Set max width diff --git a/lib/python/rose/config_editor/valuewidget/array/python_list.py b/lib/python/rose/config_editor/valuewidget/array/python_list.py index 253dab7efc..dc0c7528a4 100644 --- a/lib/python/rose/config_editor/valuewidget/array/python_list.py +++ b/lib/python/rose/config_editor/valuewidget/array/python_list.py @@ -456,7 +456,7 @@ def python_array_split(value): return value_array cast_value_array = [] for value in value_array: - if isinstance(value, basestring): + if isinstance(value, str): cast_value_array.append('"' + value + '"') else: cast_value_array.append(str(value)) diff --git a/lib/python/rose/config_editor/valuewidget/array/row.py b/lib/python/rose/config_editor/valuewidget/array/row.py index 5c67a854eb..4fbacdf811 100644 --- a/lib/python/rose/config_editor/valuewidget/array/row.py +++ b/lib/python/rose/config_editor/valuewidget/array/row.py @@ -103,7 +103,7 @@ def set_num_rows(self): if self.unlimited: self.num_rows, rem = divmod(len(self.value_array), columns) self.num_rows += [1, 0][rem == 0] - self.max_rows = sys.maxint + self.max_rows = sys.maxsize else: self.num_rows = int(self.array_length) num, rem = divmod(len(self.value_array), columns) @@ -368,7 +368,7 @@ def _normalise_width_chars(self, widget): elif hasattr(child, 'get_child'): child_list.append(child.get_child()) i += 1 - for key, value in max_width.items(): + for key, value in list(max_width.items()): if value < self.MIN_WIDTH_CHARS: max_width[key] = self.MIN_WIDTH_CHARS # Set max width diff --git a/lib/python/rose/config_editor/valuewidget/boolradio.py b/lib/python/rose/config_editor/valuewidget/boolradio.py index a360a4e7b7..e1329e38b0 100644 --- a/lib/python/rose/config_editor/valuewidget/boolradio.py +++ b/lib/python/rose/config_editor/valuewidget/boolradio.py @@ -23,7 +23,7 @@ import gtk import rose.config_editor -import radiobuttons +from . import radiobuttons class BoolValueWidget(radiobuttons.RadioButtonsValueWidget): @@ -79,7 +79,7 @@ def __init__(self, value, metadata, set_value, hook, arg_str=None): def setter(self, widget, variable): if widget.get_active(): label_value = widget.get_label() - for real_item, label in self.label_dict.items(): + for real_item, label in list(self.label_dict.items()): if label == label_value: chosen_value = real_item break diff --git a/lib/python/rose/config_editor/valuewidget/booltoggle.py b/lib/python/rose/config_editor/valuewidget/booltoggle.py index 3742d4ca56..2e85f3f586 100644 --- a/lib/python/rose/config_editor/valuewidget/booltoggle.py +++ b/lib/python/rose/config_editor/valuewidget/booltoggle.py @@ -41,13 +41,13 @@ def __init__(self, value, metadata, set_value, hook, arg_str=None): if metadata.get(rose.META_PROP_TYPE) == "boolean": self.allowed_values = [rose.TYPE_BOOLEAN_VALUE_FALSE, rose.TYPE_BOOLEAN_VALUE_TRUE] - self.label_dict = dict(zip(self.allowed_values, - self.allowed_values)) + self.label_dict = dict(list(zip(self.allowed_values, + self.allowed_values))) elif metadata.get(rose.META_PROP_TYPE) == "python_boolean": self.allowed_values = [rose.TYPE_PYTHON_BOOLEAN_VALUE_FALSE, rose.TYPE_PYTHON_BOOLEAN_VALUE_TRUE] - self.label_dict = dict(zip(self.allowed_values, - self.allowed_values)) + self.label_dict = dict(list(zip(self.allowed_values, + self.allowed_values))) else: self.allowed_values = [rose.TYPE_LOGICAL_VALUE_FALSE, rose.TYPE_LOGICAL_VALUE_TRUE] @@ -60,7 +60,7 @@ def __init__(self, value, metadata, set_value, hook, arg_str=None): imgs = [gtk.image_new_from_stock(gtk.STOCK_MEDIA_STOP, gtk.ICON_SIZE_MENU), gtk.image_new_from_stock(gtk.STOCK_APPLY, gtk.ICON_SIZE_MENU)] - self.image_dict = dict(zip(self.allowed_values, imgs)) + self.image_dict = dict(list(zip(self.allowed_values, imgs))) bad_img = gtk.image_new_from_stock(gtk.STOCK_DIALOG_WARNING, gtk.ICON_SIZE_MENU) self.button = gtk.ToggleButton(label=self.value) @@ -87,7 +87,7 @@ def _switch_state_and_set(self, widget): def setter(self, widget): label_value = widget.get_label() - for real_item, label in self.label_dict.items(): + for real_item, label in list(self.label_dict.items()): if label == label_value: chosen_value = real_item break diff --git a/lib/python/rose/config_editor/valuewidget/choice.py b/lib/python/rose/config_editor/valuewidget/choice.py index 58bbcbdc60..6ccf251f82 100644 --- a/lib/python/rose/config_editor/valuewidget/choice.py +++ b/lib/python/rose/config_editor/valuewidget/choice.py @@ -108,7 +108,7 @@ def __init__(self, value, metadata, set_value, hook, arg_str=None): self.opt_parser = rose.opt_parse.RoseOptionParser() self.opt_parser.OPTIONS = self.OPTIONS - self.opt_parser.add_my_options(*self.OPTIONS.keys()) + self.opt_parser.add_my_options(*list(self.OPTIONS.keys())) opts, args = self.opt_parser.parse_args(shlex.split(arg_str)) self.all_group = opts.all_group self.groups = [] diff --git a/lib/python/rose/config_editor/valuewidget/intspin.py b/lib/python/rose/config_editor/valuewidget/intspin.py index e3949c3b2c..ee653b9e5c 100644 --- a/lib/python/rose/config_editor/valuewidget/intspin.py +++ b/lib/python/rose/config_editor/valuewidget/intspin.py @@ -40,8 +40,8 @@ def __init__(self, value, metadata, set_value, hook, arg_str=None): self.metadata = metadata self.set_value = set_value self.hook = hook - self.upper = sys.maxint - self.lower = -sys.maxint - 1 + self.upper = sys.maxsize + self.lower = -sys.maxsize - 1 tooltip_text = None try: diff --git a/lib/python/rose/config_editor/variable.py b/lib/python/rose/config_editor/variable.py index 74d4253ab8..73d631b2ff 100644 --- a/lib/python/rose/config_editor/variable.py +++ b/lib/python/rose/config_editor/variable.py @@ -73,7 +73,7 @@ def __init__(self, variable, var_ops, is_ghost=False, show_modes=None, self.is_inconsistent = False if 'type' in variable.error: self._set_inconsistent(self.valuewidget, variable) - self.errors = variable.error.keys() + self.errors = list(variable.error.keys()) self.menuwidget = self.get_menuwidget(variable) self.generate_labelwidget() self.generate_contentwidget() @@ -343,7 +343,7 @@ def set_ignored(self): if "'Ignore'" not in self.menuwidget.option_ui: self.menuwidget.old_option_ui = self.menuwidget.option_ui self.menuwidget.old_actions = self.menuwidget.actions - if ign_map.keys() == [rose.variable.IGNORED_BY_SECTION]: + if list(ign_map.keys()) == [rose.variable.IGNORED_BY_SECTION]: # Not ignored in itself, so give Ignore option. if "'Enable'" in self.menuwidget.option_ui: self.menuwidget.option_ui = re.sub( @@ -524,7 +524,7 @@ def type_error_refresh(self, variable): else: self._set_consistent(self.valuewidget, variable) self.variable = variable - self.errors = variable.error.keys() + self.errors = list(variable.error.keys()) self.valuewidget.handle_type_error(rose.META_PROP_TYPE in self.errors) self.menuwidget.refresh(variable) self.keywidget.refresh(variable) diff --git a/lib/python/rose/config_editor/window.py b/lib/python/rose/config_editor/window.py index 70cd3a9fe4..57893f92be 100644 --- a/lib/python/rose/config_editor/window.py +++ b/lib/python/rose/config_editor/window.py @@ -310,7 +310,7 @@ def _launch_config_section_chooser_dialog(self, name_section_dict, prefs, section_label = gtk.Label(section_title) section_label.show() config_name_box = gtk.combo_box_new_text() - name_keys = name_section_dict.keys() + name_keys = list(name_section_dict.keys()) name_keys.sort() for k, name in enumerate(name_keys): config_name_box.append_text(name) diff --git a/lib/python/rose/config_processors/env.py b/lib/python/rose/config_processors/env.py index 8873381c6b..8dab223da1 100644 --- a/lib/python/rose/config_processors/env.py +++ b/lib/python/rose/config_processors/env.py @@ -41,7 +41,7 @@ def process(self, conf_tree, item, orig_keys=None, orig_value=None, os.environ.pop("UNDEF") environ = {} if env_node and not env_node.state: - for key, node in env_node.value.iteritems(): + for key, node in env_node.value.items(): if node.state: continue try: diff --git a/lib/python/rose/config_processors/fileinstall.py b/lib/python/rose/config_processors/fileinstall.py index 208be6d3b7..7f3fd4eb78 100644 --- a/lib/python/rose/config_processors/fileinstall.py +++ b/lib/python/rose/config_processors/fileinstall.py @@ -34,10 +34,10 @@ import shlex from shutil import rmtree import sqlite3 -from StringIO import StringIO +from io import StringIO import sys from tempfile import mkdtemp -from urlparse import urlparse +from urllib.parse import urlparse class ConfigProcessorForFile(ConfigProcessorBase): @@ -67,7 +67,7 @@ def process(self, conf_tree, item, orig_keys=None, orig_value=None, # Find all the "file:*" nodes. nodes = {} if item == self.SCHEME: - for key, node in conf_tree.node.value.items(): + for key, node in list(conf_tree.node.value.items()): if node.is_ignored() or not key.startswith(self.PREFIX): continue nodes[key] = node @@ -187,7 +187,7 @@ def _process(self, conf_tree, nodes, loc_dao, **kwargs): # * Its real name. # * The checksums of its paths. # * Whether it can be considered unchanged. - for source in sources.values(): + for source in list(sources.values()): try: for pattern, scheme in config_schemes: if fnmatch(source.name, pattern): @@ -220,7 +220,7 @@ def _process(self, conf_tree, nodes, loc_dao, **kwargs): # * Target exists, but does not have a database entry. # * Target exists, but does not match settings in database. # * Target exists, but a source cannot be considered unchanged. - for target in targets.values(): + for target in list(targets.values()): if target.real_name: target.is_out_of_date = ( not os.path.islink(target.name) or @@ -332,7 +332,7 @@ def _process(self, conf_tree, nodes, loc_dao, **kwargs): rmtree(work_dir) # Target checksum compare and report - for target in targets.values(): + for target in list(targets.values()): if (not target.is_out_of_date or target.loc_type == target.TYPE_TREE): continue @@ -653,7 +653,7 @@ def execute_queued_items(self): for dep_loc in loc.dep_locs: data["dep_names"]["args_list"].append( [loc.name, dep_loc.name]) - for table, datum in data.items(): + for table, datum in list(data.items()): if datum["args_list"]: conn.executemany( ("INSERT OR REPLACE INTO %s VALUES(%s)" % diff --git a/lib/python/rose/config_tree.py b/lib/python/rose/config_tree.py index 5ad99fda58..13526c0f65 100644 --- a/lib/python/rose/config_tree.py +++ b/lib/python/rose/config_tree.py @@ -194,9 +194,9 @@ def test(self, key, actual, expect): """Test if actual == expect.""" self.test_num += 1 if actual == expect: - print "ok %d - %s" % (self.test_num, key) + print("ok %d - %s" % (self.test_num, key)) else: - print "not ok %d - %s" % (self.test_num, key) + print("not ok %d - %s" % (self.test_num, key)) def test1(self): """Test: configuration file only.""" @@ -260,7 +260,7 @@ def test2(self): echo "Making breakfast $@" """) handle.close() - os.chmod("t2/bin/make-breakfast", 0755) + os.chmod("t2/bin/make-breakfast", 0o755) os.mkdir("t2/etc") for key, val in ( ("sausage", "10 fat sausages"), @@ -462,7 +462,7 @@ def test4(self): def run(self): """Run the tests.""" - print self.test_plan + print(self.test_plan) cwd = os.getcwd() work_dir = mkdtemp() @@ -485,7 +485,7 @@ def run(self): if __name__ == "__main__": # These modules are only required for running the self tests. from rose.config import ConfigDumper - from StringIO import StringIO + from io import StringIO from shutil import rmtree from tempfile import mkdtemp _Test().run() diff --git a/lib/python/rose/date.py b/lib/python/rose/date.py index f58737a985..c87576b575 100644 --- a/lib/python/rose/date.py +++ b/lib/python/rose/date.py @@ -43,8 +43,8 @@ class RoseDateTimeOperator(object): """A class to parse and print date string with an offset.""" - CURRENT_TIME_DUMP_FORMAT = u"CCYY-MM-DDThh:mm:ss+hh:mm" - CURRENT_TIME_DUMP_FORMAT_Z = u"CCYY-MM-DDThh:mm:ssZ" + CURRENT_TIME_DUMP_FORMAT = "CCYY-MM-DDThh:mm:ss+hh:mm" + CURRENT_TIME_DUMP_FORMAT_Z = "CCYY-MM-DDThh:mm:ssZ" NEGATIVE = "-" @@ -367,11 +367,11 @@ def _print_time_point(date_time_oper, opts, args): for offset in opts.offsets1: time_point = date_time_oper.date_shift(time_point, offset) if opts.print_format: - print date_time_oper.date_format(opts.print_format, time_point) + print(date_time_oper.date_format(opts.print_format, time_point)) elif parse_format: - print date_time_oper.date_format(parse_format, time_point) + print(date_time_oper.date_format(parse_format, time_point)) else: - print str(time_point) + print(str(time_point)) def _print_duration(date_time_oper, opts, args): @@ -390,7 +390,7 @@ def _print_duration(date_time_oper, opts, args): if opts.duration_print_format: _convert_duration(date_time_oper, opts, [out]) sys.exit(0) - print out + print(out) def _convert_duration(date_time_oper, opts, args): @@ -401,11 +401,11 @@ def _convert_duration(date_time_oper, opts, args): options = {'S': time, 'M': time / 60, 'H': time / 3600} if opts.duration_print_format.upper() in options: # supplied duration format is valid (upper removes case-sensitivity) - print options[opts.duration_print_format.upper()] + print(options[opts.duration_print_format.upper()]) else: # supplied duration format not valid - print 'Invalid date/time format, please use one of H, M, S ' + \ - '(hours, minutes, seconds)' + print('Invalid date/time format, please use one of H, M, S ' + \ + '(hours, minutes, seconds)') sys.exit(1) diff --git a/lib/python/rose/formats/__init__.py b/lib/python/rose/formats/__init__.py index d32254cdd7..7aedb446d5 100644 --- a/lib/python/rose/formats/__init__.py +++ b/lib/python/rose/formats/__init__.py @@ -21,4 +21,4 @@ such as namelists. To add a new format, place it in this directory and add an import statement below. """ -import namelist +from . import namelist diff --git a/lib/python/rose/gtk/choice.py b/lib/python/rose/gtk/choice.py index 3f8583e5fd..b5ad070a21 100644 --- a/lib/python/rose/gtk/choice.py +++ b/lib/python/rose/gtk/choice.py @@ -311,7 +311,7 @@ def _realign(self): ok_values = self._get_data() model = self.get_model() ok_content_sections = self._get_available_data() - for name, iter_ in self._name_iter_map.items(): + for name, iter_ in list(self._name_iter_map.items()): is_in_value = self._get_is_included(name, ok_values) if self._get_is_implicit is None: groups = self._get_groups(name, ok_content_sections) diff --git a/lib/python/rose/gtk/dialog.py b/lib/python/rose/gtk/dialog.py index 62b8775f5e..a9a05e326b 100644 --- a/lib/python/rose/gtk/dialog.py +++ b/lib/python/rose/gtk/dialog.py @@ -20,7 +20,7 @@ import multiprocessing import os -import Queue +import queue import re import shlex import subprocess @@ -202,7 +202,7 @@ def run(self): while True: try: new_text = self.event_queue.get(False) - except Queue.Empty: + except queue.Empty: break end = self.text_buffer.get_end_iter() tag = gtk diff --git a/lib/python/rose/gtk/splash.py b/lib/python/rose/gtk/splash.py index e338eb72a8..97ee89f3d6 100755 --- a/lib/python/rose/gtk/splash.py +++ b/lib/python/rose/gtk/splash.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # ----------------------------------------------------------------------------- # (C) British Crown Copyright 2012-7 Met Office. diff --git a/lib/python/rose/gtk/util.py b/lib/python/rose/gtk/util.py index 774a5b6ead..d50e3d39d0 100644 --- a/lib/python/rose/gtk/util.py +++ b/lib/python/rose/gtk/util.py @@ -20,7 +20,7 @@ import multiprocessing import os -import Queue +import queue import re import shlex import subprocess @@ -268,7 +268,7 @@ def __init__(self, widgets=[], sep_on_name=[]): separator = gtk.SeparatorToolItem() separator.show() self.insert(separator, 0) - if isinstance(stock, basestring) and stock.startswith("gtk."): + if isinstance(stock, str) and stock.startswith("gtk."): stock = getattr(gtk, stock.replace("gtk.", "", 1)) if callable(stock): widget = stock() @@ -320,7 +320,7 @@ def _poll(self): def update(self): try: message = self.queue.get(block=False) - except Queue.Empty: + except queue.Empty: pass else: self.push(self.ctx_id, message) @@ -355,7 +355,7 @@ def _poll(self): def update(self): try: message = self.queue.get(block=False) - except Queue.Empty: + except queue.Empty: pass else: self.set_text(message) @@ -542,7 +542,7 @@ def clear_sort_columns(self): def cmp_(self, value1, value2): """Perform a useful form of 'cmp'""" - if (isinstance(value1, basestring) and isinstance(value2, basestring)): + if (isinstance(value1, str) and isinstance(value2, str)): if value1.isdigit() and value2.isdigit(): return cmp(float(value1), float(value2)) return rose.config.sort_settings(value1, value2) diff --git a/lib/python/rose/host_select.py b/lib/python/rose/host_select.py index 6d34e9ec0c..5b7260d217 100644 --- a/lib/python/rose/host_select.py +++ b/lib/python/rose/host_select.py @@ -77,7 +77,7 @@ def __str__(self): "method": threshold_conf.method, "method_arg": threshold_conf.method_arg, "value": threshold_conf.value} - for key, value in fmt_map.items(): + for key, value in list(fmt_map.items()): if value is None: fmt_map[key] = "" return self.FMT % fmt_map @@ -172,7 +172,7 @@ def get_scorer(self, method=None): if method is None: method = self.RANK_METHOD_DEFAULT if method not in self.scorers: - for value in globals().values(): + for value in list(globals().values()): if (isinstance(value, type) and issubclass(value, RandomScorer) and value.KEY == method): @@ -366,7 +366,7 @@ def select(self, names=None, rank_method=None, thresholds=None, time0 = time() while host_proc_dict: sleep(self.SSH_CMD_POLL_DELAY) - for host_name, proc in host_proc_dict.items(): + for host_name, proc in list(host_proc_dict.items()): if proc.poll() is None: score = None elif proc.wait(): diff --git a/lib/python/rose/job_runner.py b/lib/python/rose/job_runner.py index 245096e3b3..3356220afe 100644 --- a/lib/python/rose/job_runner.py +++ b/lib/python/rose/job_runner.py @@ -50,7 +50,7 @@ def __init__(self, jobs, names=None): self.jobs = jobs self.ready_jobs = [] if not names: - names = jobs.keys() + names = list(jobs.keys()) for name in names: self.ready_jobs.append(self.jobs[name]) self.working_jobs = {} @@ -60,7 +60,7 @@ def get_job(self): """Return the next job that requires processing.""" while self.ready_jobs: job = self.ready_jobs.pop() - for dep_key, dep_job in job.pending_for.items(): + for dep_key, dep_job in list(job.pending_for.items()): if dep_job.state == dep_job.ST_DONE: job.pending_for.pop(dep_key) if job.name in dep_job.needed_by: @@ -81,7 +81,7 @@ def get_dead_jobs(self): """Return failed/pending jobs when there are no ready/working ones.""" jobs = self.dead_jobs if not self.has_jobs: - for job in self.jobs.values(): + for job in list(self.jobs.values()): if job.pending_for: jobs.append(job) return jobs @@ -100,7 +100,7 @@ def put_job(self, job_proxy): job.update(job_proxy) if job_proxy.exc is None: job.state = job.ST_DONE - for up_key, up_job in job.needed_by.items(): + for up_key, up_job in list(job.needed_by.items()): job.needed_by.pop(up_key) up_job.pending_for.pop(job.name) if not up_job.pending_for: @@ -195,7 +195,7 @@ def run(self, job_manager, *args): results = {} while job_manager.has_jobs(): # Process results, if any is ready - for name, result in results.items(): + for name, result in list(results.items()): if result.ready(): results.pop(name) job_proxy, args_of_events = result.get() diff --git a/lib/python/rose/loc_handlers/namelist.py b/lib/python/rose/loc_handlers/namelist.py index d3441eab56..2f588e81b7 100644 --- a/lib/python/rose/loc_handlers/namelist.py +++ b/lib/python/rose/loc_handlers/namelist.py @@ -53,10 +53,10 @@ def parse(self, loc, conf_tree): loc.loc_type = loc.TYPE_BLOB if loc.name.endswith("(:)"): name = loc.name[0:-2] - sections = [k for k in conf_tree.node.value.keys() + sections = [k for k in list(conf_tree.node.value.keys()) if k.startswith(name)] else: - sections = [k for k in conf_tree.node.value.keys() + sections = [k for k in list(conf_tree.node.value.keys()) if k == loc.name] for section in list(sections): section_value = conf_tree.node.get_value([section]) diff --git a/lib/python/rose/loc_handlers/svn.py b/lib/python/rose/loc_handlers/svn.py index 5e0408230c..8cfa76356f 100644 --- a/lib/python/rose/loc_handlers/svn.py +++ b/lib/python/rose/loc_handlers/svn.py @@ -21,7 +21,7 @@ import os from tempfile import mkdtemp -from urlparse import urlparse +from urllib.parse import urlparse import xml.parsers.expat @@ -101,7 +101,7 @@ def _handle_tag0(self, name, attr_map): self.state["index"] = ":".join(self.state["stack"][2:]) if self.state["entry"]: self.state["entry"][self.state["index"]] = "" - for key, value in attr_map.items(): + for key, value in list(attr_map.items()): name = key if self.state["index"]: name = self.state["index"] + ":" + key diff --git a/lib/python/rose/macro.py b/lib/python/rose/macro.py index d0014327e8..82c2188f95 100644 --- a/lib/python/rose/macro.py +++ b/lib/python/rose/macro.py @@ -363,14 +363,14 @@ def _get_config_sections(self, config_data): """Return all sections within config_data.""" sections = [] if isinstance(config_data, rose.config.ConfigNode): - for key, node in config_data.value.items(): + for key, node in list(config_data.value.items()): if isinstance(node.value, dict): sections.append(key) if "" not in sections: sections.append("") else: - for key in set(config_data["sections"].keys() + - config_data["variables"].keys()): + for key in set(list(config_data["sections"].keys()) + + list(config_data["variables"].keys())): sections.append(key) return sections @@ -782,7 +782,7 @@ def get_macros_for_config(config=None, modules = [] if include_custom: # Suite specified macros. meta_filepaths = [ - os.path.join(v, k) for k, v in meta_config_tree.files.items()] + os.path.join(v, k) for k, v in list(meta_config_tree.files.items())] modules.extend(load_meta_macro_modules(meta_filepaths)) if include_system: # Default macros. import rose.macros # Done to avoid cyclic top-level imports. @@ -804,12 +804,12 @@ def check_config_integrity(app_config): return MacroReturnedCorruptConfigError(ERROR_RETURN_TYPE.format( node, "node", type(node), "rose.config.ConfigNode")) if (not isinstance(node.value, dict) and - not isinstance(node.value, basestring)): + not isinstance(node.value, str)): return MacroReturnedCorruptConfigError(ERROR_RETURN_TYPE.format( node.value, "node.value", type(node.value), "dict, basestring" )) - if not isinstance(node.state, basestring): + if not isinstance(node.state, str): return MacroReturnedCorruptConfigError(ERROR_RETURN_TYPE.format( node.state, "node.state", type(node.state), "basestring")) if node.state not in [rose.config.ConfigNode.STATE_NORMAL, @@ -823,12 +823,12 @@ def check_config_integrity(app_config): "list" )) for comment in node.comments: - if not isinstance(comment, basestring): + if not isinstance(comment, str): return MacroReturnedCorruptConfigError( ERROR_RETURN_TYPE.format( comment, "comment", type(comment), "basestring")) for key in keys: - if not isinstance(key, basestring): + if not isinstance(key, str): return MacroReturnedCorruptConfigError( ERROR_RETURN_TYPE.format( key, "key", type(key), "basestring")) @@ -890,7 +890,7 @@ def update_optional_values(res, optionals, optional_values, del optionals["optional_config_name"] for key in set(optionals) & set(optional_values): res[key] = optional_values[key] - res.update(get_user_values(optionals, res.keys())) + res.update(get_user_values(optionals, list(res.keys()))) optional_values.update(res) @@ -937,7 +937,7 @@ def pretty_format_config(config, ignore_error=False): config (rose.config.ConfigNode): The Config node to convert. """ - for s_key, s_node in config.value.items(): + for s_key, s_node in list(config.value.items()): scheme = s_key if ":" in scheme: scheme = scheme.split(":", 1)[0] @@ -1034,7 +1034,7 @@ def get_metadata_for_config_id(setting_id, meta_config): node = meta_config.get([no_modifier_id], no_ignore=True) # Get metadata for namelist:foo if node is not None: - for opt, opt_node in node.value.items(): + for opt, opt_node in list(node.value.items()): if not opt_node.is_ignored(): metadata.update({opt: opt_node.value}) if option is None and rose.META_PROP_TITLE in metadata: @@ -1048,7 +1048,7 @@ def get_metadata_for_config_id(setting_id, meta_config): node = meta_config.get([search_id], no_ignore=True) # If modifier, get metadata for namelist:foo{bar} if node is not None: - for opt, opt_node in node.value.items(): + for opt, opt_node in list(node.value.items()): if not opt_node.is_ignored(): metadata.update({opt: opt_node.value}) if rose.META_PROP_TITLE in metadata: @@ -1153,7 +1153,7 @@ def run_macros(config_map, meta_config, config_name, macro_names, new_combined_config_map = combine_opt_config_map(config_map) macro_config_problems_map = {} optional_values = {} - for conf_key, config in new_combined_config_map.items(): + for conf_key, config in list(new_combined_config_map.items()): config_problems_map = report_config( config, meta_config, macros_by_type[VALIDATE_METHOD], modules, macro_tuples, opt_non_interactive, @@ -1162,11 +1162,11 @@ def run_macros(config_map, meta_config, config_name, macro_names, ) if config_problems_map: rc = 1 - for macro, problem_list in config_problems_map.items(): + for macro, problem_list in list(config_problems_map.items()): macro_config_problems_map.setdefault(macro, {}) problem_list.sort(report_sort) macro_config_problems_map[macro][conf_key] = problem_list - problem_macros = macro_config_problems_map.keys() + problem_macros = list(macro_config_problems_map.keys()) problem_macros.sort() for macro_name in problem_macros: config_problems_map = macro_config_problems_map[macro_name] @@ -1182,7 +1182,7 @@ def run_macros(config_map, meta_config, config_name, macro_names, if REPORT_METHOD in macros_by_type: new_combined_config_map = combine_opt_config_map(config_map) optional_values = {} - for conf_key, config in new_combined_config_map.items(): + for conf_key, config in list(new_combined_config_map.items()): report_config( config, meta_config, macros_by_type[REPORT_METHOD], modules, macro_tuples, opt_non_interactive, @@ -1281,7 +1281,7 @@ def handle_transform(config_map, new_config_map, change_map, macro_id, """Prompt the user to go ahead with macro changes and dump the output.""" user_allowed_changes = False has_changes = False - for change_list in change_map.values(): + for change_list in list(change_map.values()): for report in change_list: if not report.is_warning: has_changes = True @@ -1307,7 +1307,7 @@ def combine_opt_config_map(config_map): """Combine optional configurations with a main configuration.""" new_combined_config_map = {} main_config = config_map[None] - for conf_key, config in config_map.items(): + for conf_key, config in list(config_map.items()): if conf_key is None: new_combined_config_map[None] = copy.deepcopy(config) continue @@ -1393,7 +1393,7 @@ def apply_macro_to_config_map(config_map, meta_config, macro_function, def _get_user_accept(): try: - user_input = raw_input(PROMPT_ACCEPT_CHANGES) + user_input = input(PROMPT_ACCEPT_CHANGES) except EOFError: user_allowed_changes = False else: @@ -1404,13 +1404,13 @@ def _get_user_accept(): def get_user_values(options, ignore=None): if ignore is None: ignore = [] - for key, val in options.items(): + for key, val in list(options.items()): if key in ignore: continue entered = False while not entered: try: - user_input = raw_input("Value for " + str(key) + " (default " + + user_input = input("Value for " + str(key) + " (default " + str(val) + "): ") except EOFError: user_input = "" @@ -1474,7 +1474,7 @@ def load_conf_from_file(conf_dir, config_file_path, mode="macro"): config_file_path, more_keys=optional_keys, return_config_map=True) standard_format_config(app_config) - for _, config in config_map.items(): + for _, config in list(config_map.items()): standard_format_config(config) # Load meta config if it exists. diff --git a/lib/python/rose/macros/__init__.py b/lib/python/rose/macros/__init__.py index b5fbe88d12..180533a131 100644 --- a/lib/python/rose/macros/__init__.py +++ b/lib/python/rose/macros/__init__.py @@ -24,12 +24,12 @@ import inspect import rose.macro -import compulsory -import duplicate -import format -import rule -import trigger -import value +from . import compulsory +from . import duplicate +from . import format +from . import rule +from . import trigger +from . import value MODULES = [compulsory, duplicate, format, rule, trigger, value] diff --git a/lib/python/rose/macros/compulsory.py b/lib/python/rose/macros/compulsory.py index 6d7037ada2..bfad6ed562 100644 --- a/lib/python/rose/macros/compulsory.py +++ b/lib/python/rose/macros/compulsory.py @@ -56,7 +56,7 @@ def __init__(self, *args, **kwargs): def get_compulsory_data(self, meta_config): """Return a list of compulsory=true basic (no duplicate info) ids.""" compulsory_data = {} - for setting_id, sect_node in meta_config.value.items(): + for setting_id, sect_node in list(meta_config.value.items()): if sect_node.is_ignored() or isinstance(sect_node.value, str): continue if (sect_node.get_value([rose.META_PROP_COMPULSORY]) == @@ -128,7 +128,7 @@ def validate_settings(self, config_data, meta_config, self.compulsory_data = self.get_compulsory_data(meta_config) self._generate_aliases_for_sections(config_data) if only_these_sections is None: - basic_sections_to_check = self.compulsory_data.keys() + basic_sections_to_check = list(self.compulsory_data.keys()) else: basic_sections_to_check = [] alias_sections_to_check = [] diff --git a/lib/python/rose/macros/duplicate.py b/lib/python/rose/macros/duplicate.py index 8f48b63679..df251335fe 100644 --- a/lib/python/rose/macros/duplicate.py +++ b/lib/python/rose/macros/duplicate.py @@ -34,7 +34,7 @@ def validate(self, config, meta_config=None): """Return a list of errors, if any.""" self.reports = [] sect_error_no_dupl = {} - sect_keys = config.value.keys() + sect_keys = list(config.value.keys()) sorter = rose.config.sort_settings sect_keys.sort(sorter) for section in sect_keys: @@ -64,7 +64,7 @@ def validate(self, config, meta_config=None): return self.reports def _get_has_metadata(self, metadata, basic_section, meta_config): - if metadata.keys() != ["id"]: + if list(metadata.keys()) != ["id"]: return True for meta_keys, meta_node in meta_config.walk(no_ignore=True): meta_section = meta_keys[0] diff --git a/lib/python/rose/macros/rule.py b/lib/python/rose/macros/rule.py index 19291d159c..05c06d27da 100644 --- a/lib/python/rose/macros/rule.py +++ b/lib/python/rose/macros/rule.py @@ -115,7 +115,7 @@ def validate(self, config, meta_config): f_type = self.WARNING_RULE_FAILED else: f_type = self.ERROR_RULE_FAILED - for setting_id, rule_msg_list in rule_data[rule_type].items(): + for setting_id, rule_msg_list in list(rule_data[rule_type].items()): section, option = self._get_section_option_from_id(setting_id) for (rule, message) in rule_msg_list: info = None @@ -224,7 +224,7 @@ def _process_rule(self, rule, setting_id, config, meta_config, sci_num_count = -1 # any/all processing. - for array_func_key, rec_regex in self.REC_ARRAY.items(): + for array_func_key, rec_regex in list(self.REC_ARRAY.items()): for search_result in rec_regex.findall(rule): start, var_id, operator, value, end = search_result if var_id == "this": @@ -264,7 +264,7 @@ def _process_rule(self, rule, setting_id, config, meta_config, # Strings into proper string variables. for search_result in self.REC_VALUE.findall(rule): value_string = search_result.strip('"') - for key, value in local_map.items(): + for key, value in list(local_map.items()): if value == value_string: break else: @@ -278,7 +278,7 @@ def _process_rule(self, rule, setting_id, config, meta_config, proper_id = search_result.replace("this", setting_id) value_string = get_value_from_id( proper_id, config, meta_config, setting_id) - for key, value in local_map.items(): + for key, value in list(local_map.items()): if value == value_string: break else: @@ -292,7 +292,7 @@ def _process_rule(self, rule, setting_id, config, meta_config, for search_result in self.REC_CONFIG_ID.findall(rule): value_string = get_value_from_id( search_result, config, meta_config, setting_id) - for key, value in local_map.items(): + for key, value in list(local_map.items()): if value == value_string: break else: diff --git a/lib/python/rose/macros/trigger.py b/lib/python/rose/macros/trigger.py index 3d80c23851..3777d65270 100644 --- a/lib/python/rose/macros/trigger.py +++ b/lib/python/rose/macros/trigger.py @@ -54,14 +54,14 @@ def _setup_triggers(self, meta_config): self.enabled_dict = {} self.evaluator = rose.macros.rule.RuleEvaluator() self.rec_rule = rose.macros.rule.REC_EXPR_IS_THIS_RULE - for setting_id, sect_node in meta_config.value.items(): + for setting_id, sect_node in list(meta_config.value.items()): if sect_node.is_ignored(): continue opt_node = sect_node.get([rose.META_PROP_TRIGGER], no_ignore=True) if opt_node is not None: expr = opt_node.value id_value_dict = rose.variable.parse_trigger_expression(expr) - for trig_id, values in id_value_dict.items(): + for trig_id, values in list(id_value_dict.items()): if values == []: id_value_dict.update({trig_id: [None]}) self.trigger_family_lookup.update({setting_id: id_value_dict}) @@ -102,7 +102,7 @@ def transform(self, config, meta_config=None): old, new = None, None if var_id in self.ignored_dict: node.state = trig_ignored - if not any(var_id in v for v in prev_ignoreds.values()): + if not any(var_id in v for v in list(prev_ignoreds.values())): old, new = state_map[enabled], state_map[trig_ignored] elif var_id in prev_ignoreds[trig_ignored]: node.state = enabled @@ -160,7 +160,7 @@ def update(self, var_id, config_data, meta_config): # Definitely enabled. is_ignored = False if not sum([var_id in v for v in - self.trigger_family_lookup.values()]): + list(self.trigger_family_lookup.values())]): # Not triggered by anything, so must be enabled. is_ignored = False section, option = self._get_section_option_from_id(start_id) @@ -208,7 +208,7 @@ def update(self, var_id, config_data, meta_config): # Check the children of this id id_val_map = self._get_family_dict( this_id, config_data, meta_config) - for child_id, vals in id_val_map.items(): + for child_id, vals in list(id_val_map.items()): if has_ignored_parent or value is None: help_text = self.IGNORED_STATUS_PARENT.format(this_id) self.ignored_dict.setdefault(child_id, {}) @@ -290,7 +290,7 @@ def _get_ranked_trigger_ids(self): id_ranks[child_id] = depth + 1 stack.append((child_id, depth + 1)) ranked_ids = [] - for id_, rank in id_ranks.items(): + for id_, rank in list(id_ranks.items()): ranked_ids.append((rank, id_)) ranked_ids.sort() return ranked_ids @@ -336,22 +336,22 @@ def validate_dependencies(self, config, meta_config): meta_config = rose.config.ConfigNode() if not hasattr(self, 'trigger_family_lookup'): self._setup_triggers(meta_config) - config_sections = config.value.keys() - meta_settings = [k for k in meta_config.value.keys() + config_sections = list(config.value.keys()) + meta_settings = [k for k in list(meta_config.value.keys()) if not meta_config.value[k].is_ignored()] allowed_repetitions = {} - trigger_ids = self.trigger_family_lookup.keys() + trigger_ids = list(self.trigger_family_lookup.keys()) trigger_ids.sort() for var_id in trigger_ids: allowed_repetitions[var_id] = 0 - for id_value_dict in self.trigger_family_lookup.values(): + for id_value_dict in list(self.trigger_family_lookup.values()): for var_id in id_value_dict: allowed_repetitions.setdefault(var_id, 0) allowed_repetitions[var_id] += 1 for start_id in trigger_ids: id_value_dict = self._get_family_dict(start_id, config, meta_config) - triggered_ids = id_value_dict.keys() + triggered_ids = list(id_value_dict.keys()) triggered_ids.sort() if self._check_is_id_dupl(start_id, meta_config): st_sect, st_opt = self._get_section_option_from_id(start_id) @@ -361,7 +361,7 @@ def validate_dependencies(self, config, meta_config): return self._get_error_report_for_id( start_id, config, self.ERROR_DUPL_TRIG.format(st_sect)) - for value_list in id_value_dict.values(): + for value_list in list(id_value_dict.values()): for string in [s for s in value_list if s is not None]: if self.rec_rule.search(string): try: @@ -383,7 +383,7 @@ def validate_dependencies(self, config, meta_config): id_list.append(var_id) child_ids.sort() if var_id in config_sections: - child_ids += config.get([var_id]).value.keys() + child_ids += list(config.get([var_id]).value.keys()) for child_id in child_ids: base_id = self._get_stripped_id(child_id, meta_config) if base_id not in meta_settings: @@ -391,7 +391,7 @@ def validate_dependencies(self, config, meta_config): child_id, config, self.ERROR_MISSING_METADATA) if child_id in self.trigger_family_lookup: grandchildren = ( - self.trigger_family_lookup[child_id].keys()) + list(self.trigger_family_lookup[child_id].keys())) grandchildren.sort() stack.insert(1, (child_id, grandchildren)) if (id_list.count(child_id) + 1 > @@ -424,14 +424,14 @@ def _get_family_dict(self, setting_id, config_data, meta_config): sect, opt = self._get_section_option_from_id(setting_id) base_sect = rose.macro.REC_ID_STRIP.sub("", sect) trig_id = self._get_id_from_section_option(base_sect, opt) - items = self.trigger_family_lookup.get(trig_id, {}).items() + items = list(self.trigger_family_lookup.get(trig_id, {}).items()) for i, (child_id, vals) in enumerate(items): ch_sect, ch_opt = self._get_section_option_from_id(child_id) if rose.macro.REC_ID_STRIP.sub("", ch_sect) == base_sect: new_id = self._get_id_from_section_option(sect, ch_opt) items[i] = (new_id, vals) return dict(items) - items = self.trigger_family_lookup.get(setting_id, {}).items() + items = list(self.trigger_family_lookup.get(setting_id, {}).items()) dupl_adjusted_items = [] while items: child_id, vals = items.pop(0) @@ -527,9 +527,9 @@ def evaluate_trig_rule(self, rule, setting_id, value): def get_all_ids(self): """Return all setting ids involved in the triggers.""" ids = [] - for trigger_id in self.trigger_family_lookup.keys(): + for trigger_id in list(self.trigger_family_lookup.keys()): ids.append(trigger_id) - for id_value_dict in self.trigger_family_lookup.values(): + for id_value_dict in list(self.trigger_family_lookup.values()): for triggered_id in id_value_dict: if triggered_id not in ids: ids.append(triggered_id) diff --git a/lib/python/rose/macros/value.py b/lib/python/rose/macros/value.py index ade2436985..ed90ad2c62 100644 --- a/lib/python/rose/macros/value.py +++ b/lib/python/rose/macros/value.py @@ -95,7 +95,7 @@ def _validate_id(self, var_id, value, meta_config): sect, key = self._get_section_option_from_id(var_id) saved_metadata = copy.deepcopy(metadata) saved_metadata.pop('id') - for meta_key in saved_metadata.keys(): + for meta_key in list(saved_metadata.keys()): if meta_key not in self.META_PROPS: saved_metadata.pop(meta_key) goodness_id = (value, tuple(sorted(saved_metadata.items()))) @@ -107,7 +107,7 @@ def _validate_id(self, var_id, value, meta_config): return variable = rose.variable.Variable(key, value, metadata) metadata = variable.metadata - if not isinstance(value, basestring): + if not isinstance(value, str): text = self.WARNING_NOT_STRING.format(repr(value)) self.bad_value_meta_map[goodness_id] = text self.add_report(sect, key, value, text) @@ -123,7 +123,7 @@ def _validate_id(self, var_id, value, meta_config): num_elements = 1 val_list = [value] type_list = metadata.get(rose.META_PROP_TYPE, '') - if isinstance(type_list, basestring): + if isinstance(type_list, str): type_list = [type_list] if num_elements != 1: val_list = rose.variable.array_split(value) @@ -159,7 +159,7 @@ def _validate_id(self, var_id, value, meta_config): break elif rose.META_PROP_TYPE in metadata: meta_type = metadata[rose.META_PROP_TYPE] - if (num_elements == 1 and isinstance(meta_type, basestring)): + if (num_elements == 1 and isinstance(meta_type, str)): # A standard, non array variable. for val in val_list: try: @@ -171,7 +171,7 @@ def _validate_id(self, var_id, value, meta_config): else: # The variable is an array or a derived type array. - if isinstance(meta_type, basestring): + if isinstance(meta_type, str): type_list = [meta_type] else: type_list = meta_type diff --git a/lib/python/rose/meta_type.py b/lib/python/rose/meta_type.py index 0710813277..43ac037a4e 100644 --- a/lib/python/rose/meta_type.py +++ b/lib/python/rose/meta_type.py @@ -36,7 +36,7 @@ def get_meta_type(cls, key): """Return the class for a named meta data type.""" if key in cls.meta_type_classes: return cls.meta_type_classes[key] - for c in globals().values(): + for c in list(globals().values()): if inspect.isclass(c): if c != cls and issubclass(c, cls): if c.KEY == key: diff --git a/lib/python/rose/metadata_check.py b/lib/python/rose/metadata_check.py index df0f8aaff5..bef6dcdaab 100644 --- a/lib/python/rose/metadata_check.py +++ b/lib/python/rose/metadata_check.py @@ -173,7 +173,7 @@ def _check_value_titles(title_value, values_value): def _check_type(value): types = rose.variable.parse_type_expression(value) - if isinstance(types, basestring): + if isinstance(types, str): types = [types] if " " in value and "," not in value: types = [value] @@ -244,7 +244,7 @@ def metadata_check(meta_config, meta_dir=None, allowed_properties = get_allowed_metadata_properties() reports = [] module_files = _get_module_files(meta_dir) - sections = meta_config.value.keys() + sections = list(meta_config.value.keys()) sections.sort(rose.config.sort_settings) for section in sections: node = meta_config.value[section] @@ -270,7 +270,7 @@ def metadata_check(meta_config, meta_dir=None, value = node.get_value([rose.META_PROP_LENGTH]) reports.append(rose.macro.MacroReport( section, rose.META_PROP_LENGTH, value, info)) - options = node.value.keys() + options = list(node.value.keys()) options.sort(rose.config.sort_settings) for option in options: opt_node = node.value[option] diff --git a/lib/python/rose/metadata_gen.py b/lib/python/rose/metadata_gen.py index 3ec75a86ae..cb1e708124 100644 --- a/lib/python/rose/metadata_gen.py +++ b/lib/python/rose/metadata_gen.py @@ -64,7 +64,7 @@ def metadata_gen(config, meta_config=None, auto_type=False, prop_map={}): # Add duplicate = true at base and modifier level (if needed). meta_config.set([meta_sect, rose.META_PROP_DUPLICATE], rose.META_PROP_VALUE_TRUE) - for prop_key, prop_value in prop_map.items(): + for prop_key, prop_value in list(prop_map.items()): meta_config.set([meta_sect, prop_key], prop_value) if option is None: continue @@ -73,7 +73,7 @@ def metadata_gen(config, meta_config=None, auto_type=False, prop_map={}): if meta_config.get([meta_opt]) is not None: continue meta_config.set([meta_opt]) - for prop_key, prop_value in prop_map.items(): + for prop_key, prop_value in list(prop_map.items()): meta_config.set([meta_opt, prop_key], prop_value) if auto_type: opt_type, length = type_gen(node.value) diff --git a/lib/python/rose/metadata_graph.py b/lib/python/rose/metadata_graph.py index c46ee505b3..ef627dc0b5 100644 --- a/lib/python/rose/metadata_graph.py +++ b/lib/python/rose/metadata_graph.py @@ -53,7 +53,7 @@ def get_node_state_attrs(config, section, option=None, allowed_sections=None): node_attrs = {} if option is None: node_attrs["shape"] = SHAPE_NODE_SECTION - if not config.value.keys(): + if not list(config.value.keys()): # Empty configuration - we can assume pure metadata. return node_attrs if allowed_sections is None: @@ -220,7 +220,7 @@ def output_graph(graph, debug_mode=False, filename=None, format="svg"): graph.draw(image_file_handle.name, prog="dot") if debug_mode: image_file_handle.seek(0) - print image_file_handle.read() + print(image_file_handle.read()) image_file_handle.close() return rose.external.launch_image_viewer(image_file_handle.name, run_fg=True) @@ -240,7 +240,7 @@ def _load_override_config(): ["rose-metadata-graph"]) if conf is None: return - for key, node in conf.value.items(): + for key, node in list(conf.value.items()): if node.is_ignored(): continue try: @@ -279,7 +279,7 @@ def main(): config, directory=opts.conf_dir, ) - if not meta_config.value.keys(): + if not list(meta_config.value.keys()): _exit_with_metadata_fail() elif os.path.exists(meta_config_file_path): config = rose.config.ConfigNode() diff --git a/lib/python/rose/namelist_dump.py b/lib/python/rose/namelist_dump.py index 8d2e3487ff..c01617d287 100644 --- a/lib/python/rose/namelist_dump.py +++ b/lib/python/rose/namelist_dump.py @@ -99,7 +99,7 @@ def namelist_dump(args=None, output_file=None, case_mode=None): groups_in_file[group.file].append(group) # Add contents to relevant file: sections - for file, groups in groups_in_file.items(): + for file, groups in list(groups_in_file.items()): section = "file:" + file if file == sys.stdin.name: section = "file:STDIN" @@ -112,7 +112,7 @@ def namelist_dump(args=None, output_file=None, case_mode=None): config.set([section, "source"], " ".join(group_sections)) # Add namelist: sections - for name, groups in groups_by_name.items(): + for name, groups in list(groups_by_name.items()): for group in groups: section = "namelist:" + tr_case(group.name, case_mode) if len(groups) > 1: diff --git a/lib/python/rose/reporter.py b/lib/python/rose/reporter.py index 2e992874e2..841a3a4aa0 100644 --- a/lib/python/rose/reporter.py +++ b/lib/python/rose/reporter.py @@ -19,7 +19,7 @@ # ----------------------------------------------------------------------------- """Reporter for diagnostic messages.""" -import Queue +import queue import multiprocessing import os @@ -157,7 +157,7 @@ def report(self, message, kind=None, level=None, prefix=None, clip=None): if level is None: level = self.DEFAULT msg = None - for key, context in self.contexts.items(): + for key, context in list(self.contexts.items()): if context.is_closed(): self.contexts.pop(key) # remove contexts with closed handles continue @@ -176,10 +176,10 @@ def report(self, message, kind=None, level=None, prefix=None, clip=None): msg = message try: - msg = unicode(msg) + msg = str(msg) except UnicodeDecodeError: try: - msg = unicode(msg, 'utf-8') + msg = str(msg, 'utf-8') except TypeError: msg = str(msg) except UnicodeEncodeError: @@ -312,7 +312,7 @@ def _send_pending_messages(self): for message in list(self._messages_pending): try: self.queue.put(self._messages_pending[0], block=False) - except Queue.Full: + except queue.Full: break self._messages_pending.pop(0) diff --git a/lib/python/rose/run.py b/lib/python/rose/run.py index 2f0d9ce855..cdc05aa279 100644 --- a/lib/python/rose/run.py +++ b/lib/python/rose/run.py @@ -86,7 +86,7 @@ class Dummy(object): """Convert a dict into an object.""" def __init__(self, **kwargs): - for key, value in kwargs.items(): + for key, value in list(kwargs.items()): setattr(self, key, value) @@ -193,7 +193,7 @@ def run(self, opts, args): opts = Dummy(**opts) cwd = os.getcwd() environ = {} - for k, v in os.environ.items(): + for k, v in list(os.environ.items()): environ[k] = v uuid = str(uuid4()) work_files = [] @@ -221,7 +221,7 @@ def run(self, opts, args): pass # Reset os.environ os.environ = {} - for k, v in environ.items(): + for k, v in list(environ.items()): os.environ[k] = v __call__ = run diff --git a/lib/python/rose/scheme_handler.py b/lib/python/rose/scheme_handler.py index 1bf8f2ac4f..2ad45f218f 100644 --- a/lib/python/rose/scheme_handler.py +++ b/lib/python/rose/scheme_handler.py @@ -120,7 +120,7 @@ def guess_handler(self, item): handler = self.get_handler(item) if handler: return handler - for handler in self.handlers.values(): + for handler in list(self.handlers.values()): can_handle = getattr(handler, self.can_handle, None) if (callable(can_handle) and can_handle(item)): return handler diff --git a/lib/python/rose/section.py b/lib/python/rose/section.py index c0a5bde3e1..417d81cc15 100644 --- a/lib/python/rose/section.py +++ b/lib/python/rose/section.py @@ -56,9 +56,9 @@ def __init__(self, name, options=None, flags = {} if comments is None: comments = [] - self.metadata = dict(metadata.items()) - self.flags = dict(flags.items()) - self.ignored_reason = dict(ignored_reason.items()) + self.metadata = dict(list(metadata.items())) + self.flags = dict(list(flags.items())) + self.ignored_reason = dict(list(ignored_reason.items())) self.error = error self.warning = warning self.comments = comments diff --git a/lib/python/rose/stem.py b/lib/python/rose/stem.py index 17ff78f0c0..f2bf0157eb 100644 --- a/lib/python/rose/stem.py +++ b/lib/python/rose/stem.py @@ -278,8 +278,8 @@ def _ascertain_project(self, item): item = os.path.abspath(os.path.join(os.getcwd(), item)) if project is not None: - print "[WARN] Forcing project for '{0}' to be '{1}'".format( - item, project) + print("[WARN] Forcing project for '{0}' to be '{1}'".format( + item, project)) return project, item, item, '', '' source_dict = self._get_base_dir(item) @@ -404,11 +404,11 @@ def process(self): self._add_define_option('SOURCE_' + project.upper() + '_MIRROR', '"' + mirror + '"') self.reporter(SourceTreeAddedAsBranchEvent(url)) - for project, branches in repos.iteritems(): + for project, branches in repos.items(): var = 'SOURCE_' + project.upper() branchstring = RosePopener.list_to_shell_str(branches) self._add_define_option(var, '"' + branchstring + '"') - for project, branches in repos_with_hosts.iteritems(): + for project, branches in repos_with_hosts.items(): var_host = 'HOST_SOURCE_' + project.upper() branchstring = RosePopener.list_to_shell_str(branches) self._add_define_option(var_host, '"' + branchstring + '"') diff --git a/lib/python/rose/suite_clean.py b/lib/python/rose/suite_clean.py index 5862886b27..7ad9f98dbd 100644 --- a/lib/python/rose/suite_clean.py +++ b/lib/python/rose/suite_clean.py @@ -80,7 +80,7 @@ def _clean(self, suite_name, only_items=None): items = only_items items.sort() uuid_str = str(uuid4()) - for auth, node in sorted(locs_conf.value.items(), self._auth_node_cmp): + for auth, node in sorted(list(locs_conf.value.items()), self._auth_node_cmp): locs = [] roots = set([""]) for item in items: @@ -177,7 +177,7 @@ def main(): for arg in args: if not opts.non_interactive: try: - answer = raw_input("Clean %s? y/n (default n) " % arg) + answer = input("Clean %s? y/n (default n) " % arg) except EOFError: sys.exit(1) if answer not in ["Y", "y"]: diff --git a/lib/python/rose/suite_control.py b/lib/python/rose/suite_control.py index 770a811835..61d61daa3e 100644 --- a/lib/python/rose/suite_control.py +++ b/lib/python/rose/suite_control.py @@ -159,7 +159,7 @@ def prompt(action, suite_name, host): """Prompt user to confirm action for suite_name at host.""" if not host: host = "localhost" - return raw_input(PROMPT % (action, suite_name, host)).strip() in [YES] + return input(PROMPT % (action, suite_name, host)).strip() in [YES] def main(): diff --git a/lib/python/rose/suite_engine_proc.py b/lib/python/rose/suite_engine_proc.py index 94db141f39..883101f1a2 100644 --- a/lib/python/rose/suite_engine_proc.py +++ b/lib/python/rose/suite_engine_proc.py @@ -296,13 +296,13 @@ class TaskProps(object): "dir_etc": "ROSE_ETC"} def __init__(self, **kwargs): - for attr_key, env_key in self.ATTRS.items(): + for attr_key, env_key in list(self.ATTRS.items()): if kwargs.get(attr_key) is not None: setattr(self, attr_key, kwargs.get(attr_key)) elif env_key.endswith("%s"): setattr(self, attr_key, {}) prefix = env_key.replace("%s", "") - for key, value in os.environ.items(): + for key, value in list(os.environ.items()): if key == prefix or not key.startswith(prefix): continue try: @@ -321,7 +321,7 @@ def __iter__(self): attr_value = getattr(self, attr_key) if attr_value is not None: if isinstance(attr_value, dict): - for key, value in attr_value.items(): + for key, value in list(attr_value.items()): yield (env_key % key, str(value)) else: yield (env_key, str(attr_value)) @@ -533,7 +533,7 @@ def _get_task_props(self, *_, **kwargs): # Note: should we create the offsets directories? for dir_ in ( [tprops.dir_data, tprops.dir_data_cycle] + - tprops.dir_data_cycle_offsets.values()): + list(tprops.dir_data_cycle_offsets.values())): if dir_ is None: continue if os.path.exists(dir_) and not os.path.isdir(dir_): diff --git a/lib/python/rose/suite_engine_procs/cylc.py b/lib/python/rose/suite_engine_procs/cylc.py index d2b0ee211c..dd1450c1af 100644 --- a/lib/python/rose/suite_engine_procs/cylc.py +++ b/lib/python/rose/suite_engine_procs/cylc.py @@ -130,6 +130,8 @@ def get_suite_dir_rel(cls, suite_name, *paths): def get_suite_jobs_auths(self, suite_name, cycle_name_tuples=None): """Return remote ["[user@]host", ...] for submitted jobs.""" + # Broken in Cylc8, we can just skip it + return None auths = [] stmt = "SELECT DISTINCT user_at_host FROM task_jobs" stmt_where_list = [] @@ -522,7 +524,7 @@ def run(self, suite_name, host=None, host_environ=None, run_mode=None, bash_cmd_prefix += "mkdir -p %s\n" % log_dir bash_cmd_prefix += "cd %s\n" % log_dir if host_environ: - for key, value in host_environ.items(): + for key, value in list(host_environ.items()): val = self.popen.list_to_shell_str([value]) bash_cmd_prefix += "%s=%s\n" % (key, val) bash_cmd_prefix += "export %s\n" % (key) @@ -683,6 +685,7 @@ def __init__(self, db_f_name): self.db_f_name = db_f_name self.conn = None self.cursor = None + cherrypy.log("CylcSuiteDAO.init()") def close(self): """Close the DB connection.""" @@ -696,14 +699,17 @@ def close(self): def commit(self): """Commit any changes to current connection.""" + cherrypy.log("CylcSuiteDAO.commit()") if self.conn is not None: self.conn.commit() def connect(self, is_new=False): """Connect to the DB. Set the cursor. Return the connection.""" + cherrypy.log("CylcSuiteDAO.connect()") if self.cursor is not None: return self.cursor if not is_new and not os.access(self.db_f_name, os.F_OK | os.R_OK): + cherrypy.log("CylcSuiteDAO file unaccessible: " + self.db_f_name) return None for _ in range(self.N_CONNECT_TRIES): try: @@ -715,26 +721,32 @@ def connect(self, is_new=False): self.conn = None self.cursor = None else: + cherrypy.log("CylcSuiteDAO.connect break") break return self.conn def execute(self, stmt, stmt_args=None, commit=False): """Execute a statement. Return the cursor.""" + cherrypy.log("CylcSuiteDAO.execute()") if stmt_args is None: stmt_args = [] for _ in range(self.N_CONNECT_TRIES): if self.connect() is None: + cherrypy.log("CylcSuiteDAO.execute no connection") return [] try: self.cursor.execute(stmt, stmt_args) - except sqlite3.OperationalError: + except sqlite3.OperationalError as e: + cherrypy.log("CylcSuiteDAO.execute error: " + str(e)) sleep(self.CONNECT_RETRY_DELAY) self.conn = None self.cursor = None - except sqlite3.ProgrammingError: + except sqlite3.ProgrammingError as e: + cherrypy.log("CylcSuiteDAO.execute error: " + str(e)) self.conn = None self.cursor = None else: + cherrypy.log("CylcSuiteDAO.execute break!") break if self.cursor is None: return [] diff --git a/lib/python/rose/suite_log.py b/lib/python/rose/suite_log.py index ae2a64b5e0..5c8f96bb5c 100644 --- a/lib/python/rose/suite_log.py +++ b/lib/python/rose/suite_log.py @@ -80,7 +80,7 @@ def suite_log_view(opts, args, event_handler=None): url = suite_engine_proc.get_suite_log_url(opts.user, opts.name) if url.startswith("file://"): if (opts.non_interactive or - raw_input("Start rose bush? [y/n] (default=n) ") == "y"): + input("Start rose bush? [y/n] (default=n) ") == "y"): suite_engine_proc.popen.run_bg( "rose", "bush", "start", preexec_fn=os.setpgrp) is_rose_bush_started = True diff --git a/lib/python/rose/suite_run.py b/lib/python/rose/suite_run.py index 416b2844a9..24a8f42847 100644 --- a/lib/python/rose/suite_run.py +++ b/lib/python/rose/suite_run.py @@ -157,7 +157,7 @@ def run_impl(self, opts, args, uuid, work_files): auto_items = {"ROSE_ORIG_HOST": self.host_selector.get_local_host(), "ROSE_VERSION": ResourceLocator.default().get_version(), suite_engine_key: suite_engine_version} - for key, val in auto_items.items(): + for key, val in list(auto_items.items()): requested_value = conf_tree.node.get_value(["env", key]) if requested_value: if key == "ROSE_VERSION" and val != requested_value: @@ -259,7 +259,7 @@ def run_impl(self, opts, args, uuid, work_files): # Process Files cwd = os.getcwd() - for rel_path, conf_dir in conf_tree.files.items(): + for rel_path, conf_dir in list(conf_tree.files.items()): if (conf_dir == cwd or any(fnmatchcase(os.sep + rel_path, exclude) for exclude in self.SYNC_EXCLUDES) or diff --git a/lib/python/rose/task_env.py b/lib/python/rose/task_env.py index 2807104468..b0d1afb25f 100644 --- a/lib/python/rose/task_env.py +++ b/lib/python/rose/task_env.py @@ -79,7 +79,7 @@ def get_prepend_paths(event_handler=None, path_root=None, path_glob_args=[], # Default or specified globs path_globs_map = {} if full_mode: - for name, path_globs in PATH_GLOBS.items(): + for name, path_globs in list(PATH_GLOBS.items()): path_globs_map[name] = path_globs for path_glob_arg in path_glob_args: if path_glob_arg is None: @@ -94,7 +94,7 @@ def get_prepend_paths(event_handler=None, path_root=None, path_glob_args=[], more_prepend_paths_map = {} if not path_root: path_root = os.getcwd() - for name, path_globs in path_globs_map.items(): + for name, path_globs in list(path_globs_map.items()): if name not in more_prepend_paths_map: more_prepend_paths_map[name] = [] for path_glob in path_globs: @@ -107,12 +107,12 @@ def get_prepend_paths(event_handler=None, path_root=None, path_glob_args=[], more_prepend_paths_map[name].append(path) else: more_prepend_paths_map[name] = [] # empty value resets - for name, more_prepend_paths in more_prepend_paths_map.items(): + for name, more_prepend_paths in list(more_prepend_paths_map.items()): if name in prepend_paths_map: prepend_paths_map[name].extend(more_prepend_paths) elif more_prepend_paths: prepend_paths_map[name] = more_prepend_paths - for key, prepend_paths in prepend_paths_map.items(): + for key, prepend_paths in list(prepend_paths_map.items()): prepend_paths.reverse() return prepend_paths_map @@ -128,7 +128,7 @@ def main(): suite_engine_proc = SuiteEngineProcessor.get_processor( event_handler=report) kwargs = {} - for k, v in vars(opts).items(): + for k, v in list(vars(opts).items()): kwargs[k] = v try: task_props = suite_engine_proc.get_task_props(*args, **kwargs) @@ -141,7 +141,7 @@ def main(): task_props.suite_dir, path_globs, full_mode=True) - for k, prepend_paths in prepend_paths_map.items(): + for k, prepend_paths in list(prepend_paths_map.items()): orig_paths = [] orig_v = os.getenv(k, "") if orig_v: diff --git a/lib/python/rose/task_run.py b/lib/python/rose/task_run.py index 112bde04dd..ef3290f02f 100644 --- a/lib/python/rose/task_run.py +++ b/lib/python/rose/task_run.py @@ -77,7 +77,7 @@ def run_impl(self, opts, args, uuid, work_files): t.suite_dir, path_globs, full_mode=is_changed) - for k, prepend_paths in prepend_paths_map.items(): + for k, prepend_paths in list(prepend_paths_map.items()): orig_paths = [] orig_v = os.getenv(k, "") if orig_v: diff --git a/lib/python/rose/upgrade.py b/lib/python/rose/upgrade.py index 2c6f00724c..1d0631b99f 100644 --- a/lib/python/rose/upgrade.py +++ b/lib/python/rose/upgrade.py @@ -126,7 +126,7 @@ def _get_config_resources(self): file_map = {} file_map[MACRO_UPGRADE_RESOURCE_FILE_ADD] = add_path file_map[MACRO_UPGRADE_RESOURCE_FILE_REMOVE] = rem_path - for key, path in file_map.items(): + for key, path in list(file_map.items()): if os.path.isfile(path): file_map[key] = rose.config.load(path) else: @@ -206,7 +206,7 @@ def add_setting(self, config, keys, value=None, forced=False, # Add parent section if missing. if option is not None and config.get([section]) is None: self.add_setting(config, [section]) - if value is not None and not isinstance(value, basestring): + if value is not None and not isinstance(value, str): text = "New value {0} for {1} is not a string" raise ValueError(text.format(repr(value), id_)) @@ -233,7 +233,7 @@ def change_setting_value(self, config, keys, value, forced=False, raise TypeError(text) if info is None: info = self.INFO_CHANGED_VAR.format(repr(node.value), repr(value)) - if value is not None and not isinstance(value, basestring): + if value is not None and not isinstance(value, str): text = "New value {0} for {1} is not a string" raise ValueError(text.format(repr(value), id_)) node.value = value diff --git a/lib/python/rose/variable.py b/lib/python/rose/variable.py index b6787f472d..07f6d91070 100644 --- a/lib/python/rose/variable.py +++ b/lib/python/rose/variable.py @@ -81,8 +81,8 @@ def __init__(self, name, value, metadata=None, ignored_reason=None, comments = [] self.metadata = self.process_metadata(metadata) self.old_value = value - self.flags = dict(flags.items()) - self.ignored_reason = dict(ignored_reason.items()) + self.flags = dict(list(flags.items())) + self.ignored_reason = dict(list(ignored_reason.items())) self.error = error self.warning = warning self.comments = comments @@ -410,7 +410,7 @@ def _scan_trigger_string(string): # Quote state change. is_in_quotes[letter] = not is_in_quotes[letter] if not any(is_in_quotes.values()) and not is_escaped: - for delim, token in delim_tokens.items(): + for delim, token in list(delim_tokens.items()): if string[i:i + len(delim)] == delim: # String next contains a valid delimiter # Yield the built-up item and a token @@ -422,7 +422,7 @@ def _scan_trigger_string(string): is_escaped = (letter == esc_char and not is_escaped) if (letter == esc_char and is_escaped and not any(is_in_quotes.values()) and i + 1 < len(string)): - for delim, token in delim_tokens.items(): + for delim, token in list(delim_tokens.items()): if string[i + 1:i + 1 + len(delim)] == delim: # A valid escape character before a delimiter. # Discard the escape character for the parsed text. diff --git a/lib/python/rose/ws.py b/lib/python/rose/ws.py index 13ef8c6e0e..7d4abf0b3c 100644 --- a/lib/python/rose/ws.py +++ b/lib/python/rose/ws.py @@ -84,7 +84,7 @@ def ws_cli(service_cls, *args, **kwargs): report("%s=%s\n" % (key, value), level=level) if (arg == "stop" and status.get("pid") and (opts.non_interactive or - raw_input("Stop server? y/n (default=n)") == "y")): + input("Stop server? y/n (default=n)") == "y")): os.killpg(int(status["pid"]), signal.SIGTERM) # TODO: should check whether it is killed or not diff --git a/lib/python/rosie/browser/history.py b/lib/python/rosie/browser/history.py index 262d43ea05..f8e13ecaf4 100644 --- a/lib/python/rosie/browser/history.py +++ b/lib/python/rosie/browser/history.py @@ -78,12 +78,12 @@ def set_hist_length(self, time): def print_archive(self): """Print the history archive.""" for l in self.archive: - print str(l) + print(str(l)) def print_session(self): """Print the session history.""" for l in self.session_log: - print str(l) + print(str(l)) def load_history(self): """Load in the archived search history.""" diff --git a/lib/python/rosie/browser/main.py b/lib/python/rosie/browser/main.py index c219b57d88..a030fdaf6f 100644 --- a/lib/python/rosie/browser/main.py +++ b/lib/python/rosie/browser/main.py @@ -27,7 +27,7 @@ import threading import time import traceback -from urlparse import urlparse +from urllib.parse import urlparse import warnings import webbrowser @@ -109,7 +109,7 @@ def __init__(self, opts=None, args=None, splash_updater=None): self.config = locator.get_conf() self.usernames = {} - for prefix in self.ws_client.auth_managers.keys(): + for prefix in list(self.ws_client.auth_managers.keys()): self.usernames[prefix] = self.config.get_value( ["rosie-id", "prefix-username." + prefix], default=pwd.getpwuid(os.getuid()).pw_name) @@ -1282,7 +1282,7 @@ def _run_suite(self, args=None, **kwargs): """Run the suite, if possible.""" if not isinstance(args, list): args = [] - for key, value in kwargs.items(): + for key, value in list(kwargs.items()): args.extend([key, value]) suite_local_copy = SuiteId( self.get_selected_suite_id()).to_local_copy() diff --git a/lib/python/rosie/browser/result.py b/lib/python/rosie/browser/result.py index a8164e0615..4c0ab7c190 100644 --- a/lib/python/rosie/browser/result.py +++ b/lib/python/rosie/browser/result.py @@ -282,7 +282,7 @@ def update_treemodel(self, sort_index=0, descending=False): prev_row = self.treestore.insert(None, i, this_row + [True]) prev_val = this_row[0] row_group_headers.setdefault(prev_val, prev_row) - for group_val, row_iter in row_group_headers.items(): + for group_val, row_iter in list(row_group_headers.items()): if group_val in expanded_groups: path = self.treestore.get_path(row_iter) self.treeview.expand_to_path(path) diff --git a/lib/python/rosie/browser/status.py b/lib/python/rosie/browser/status.py index eec9a722c1..301b008cde 100644 --- a/lib/python/rosie/browser/status.py +++ b/lib/python/rosie/browser/status.py @@ -18,7 +18,7 @@ # along with Rose. If not, see . # ----------------------------------------------------------------------------- -import Queue +import queue import time import threading @@ -63,7 +63,7 @@ def update(self): """Check for any new communicated info and if so run a hook.""" try: local_suites = self.queue.get_nowait() - except Queue.Empty: + except queue.Empty: return self.local_suites = local_suites gobject.idle_add(self.update_hook, local_suites) diff --git a/lib/python/rosie/browser/util.py b/lib/python/rosie/browser/util.py index c2655f79f9..8f41c4dd98 100644 --- a/lib/python/rosie/browser/util.py +++ b/lib/python/rosie/browser/util.py @@ -383,7 +383,7 @@ def __init__(self, known_keys, ws_client): def add_prefix_choices(self): """Add the prefix choices.""" - self.prefixes = self.ws_client.auth_managers.keys() + self.prefixes = list(self.ws_client.auth_managers.keys()) self.prefixes.sort() self.prefixes.reverse() for prefix in self.prefixes: @@ -415,7 +415,7 @@ def set_accelerators(self, accel_dict): key_list = [] mod_list = [] action_list = [] - for key_press, accel_func in accel_dict.items(): + for key_press, accel_func in list(accel_dict.items()): key, mod = gtk.accelerator_parse(key_press) self.accelerators.lookup[str(key) + str(mod)] = accel_func self.accelerators.connect_group( diff --git a/lib/python/rosie/suite_id.py b/lib/python/rosie/suite_id.py index 04f82fed03..ad6972de1e 100644 --- a/lib/python/rosie/suite_id.py +++ b/lib/python/rosie/suite_id.py @@ -262,7 +262,7 @@ def get_prefix_from_location_root(cls, root): locations = cls.get_prefix_locations() if not locations: raise SuiteIdLocationError(root) - for key, value in locations.items(): + for key, value in list(locations.items()): if value == root: return key else: @@ -290,7 +290,7 @@ def get_prefix_locations(cls): rosie_id_node = config.get(["rosie-id"], no_ignore=True) if rosie_id_node is None: return ret - for key, node in rosie_id_node.value.items(): + for key, node in list(rosie_id_node.value.items()): if node.state: continue if key.startswith("prefix-location."): @@ -572,7 +572,7 @@ def main(): suite_id = SuiteId.get_latest(prefix=arg) if suite_id is not None: report(str(suite_id) + "\n", level=0) - elif opts.next: + elif opts.__next__: suite_id = SuiteId.get_next(prefix=arg) if suite_id is not None: report(str(suite_id) + "\n", level=0) diff --git a/lib/python/rosie/svn_post_commit.py b/lib/python/rosie/svn_post_commit.py index 5e9724b17a..182e10d2bf 100644 --- a/lib/python/rosie/svn_post_commit.py +++ b/lib/python/rosie/svn_post_commit.py @@ -41,7 +41,7 @@ from smtplib import SMTP import socket import sqlalchemy as al -from StringIO import StringIO +from io import StringIO import sys from tempfile import TemporaryFile from time import mktime, strptime @@ -68,7 +68,7 @@ def delete(self, key, **kwargs): """Perform a delete on table key, using kwargs to select rows.""" table = self._get_table(key) where = None - for col, value in kwargs.items(): + for col, value in list(kwargs.items()): if where is None: where = table.c[col] == value else: @@ -132,7 +132,7 @@ def run(self, repos, revision, no_notification=False): # Do nothing if prefix is not registered conf = ResourceLocator.default().get_conf() rosie_db_node = conf.get(["rosie-db"], no_ignore=True) - for key, node in rosie_db_node.value.items(): + for key, node in list(rosie_db_node.value.items()): if node.is_ignored() or not key.startswith("repos."): continue if os.path.realpath(repos) == os.path.realpath(node.value): @@ -406,7 +406,7 @@ def _update_info_db(self, dao, changeset_attribs, branch_attribs): "date": changeset_attribs["date"]}) for name in ["owner", "project", "title"]: cols[name] = branch_attribs[info_key].get_value([name]) - if branch_attribs["from_path"] and vc_attrs["branch"] == u"trunk": + if branch_attribs["from_path"] and vc_attrs["branch"] == "trunk": from_names = branch_attribs["from_path"].split("/")[:self.LEN_ID] cols["from_idx"] = ( changeset_attribs["prefix"] + "-" + "".join(from_names)) @@ -439,11 +439,11 @@ def _update_known_keys(self, dao, changeset_attribs): keys_str = " ".join(shlex.split(keys_str)).decode("utf-8") if keys_str: try: - dao.insert(META_TABLE_NAME, name=u"known_keys", value=keys_str) + dao.insert(META_TABLE_NAME, name="known_keys", value=keys_str) except al.exc.IntegrityError: dao.update( - META_TABLE_NAME, (u"name",), - name=u"known_keys", value=keys_str) + META_TABLE_NAME, ("name",), + name="known_keys", value=keys_str) def main(): diff --git a/lib/python/rosie/svn_pre_commit.py b/lib/python/rosie/svn_pre_commit.py index f05dd23689..1bb21c0ae6 100755 --- a/lib/python/rosie/svn_pre_commit.py +++ b/lib/python/rosie/svn_pre_commit.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # ----------------------------------------------------------------------------- # (C) British Crown Copyright 2012-7 Met Office. # diff --git a/lib/python/rosie/vc.py b/lib/python/rosie/vc.py index 9d008a601d..c2bd011ca2 100644 --- a/lib/python/rosie/vc.py +++ b/lib/python/rosie/vc.py @@ -36,10 +36,10 @@ from rose.resource import ResourceLocator from rosie.suite_id import SuiteId, SuiteIdOverflowError, SuiteIdPrefixError import shutil -from StringIO import StringIO +from io import StringIO import sys from tempfile import mkdtemp, mkstemp -from urlparse import urlparse +from urllib.parse import urlparse CREATE_INFO_CONFIG_COMMENT = """ @@ -398,7 +398,7 @@ def generate_info_config(self, from_id=None, prefix=None, project=None): if groups_node is not None: prefix_loc = SuiteId.get_prefix_location(prefix) prefix_host = urlparse(prefix_loc).hostname - for key, node in groups_node.value.items(): + for key, node in list(groups_node.value.items()): if fnmatch(prefix_host, node.value): owner = servers_conf.get_value([key, "username"]) break @@ -592,7 +592,7 @@ def create(argv): prefix = SuiteId.get_prefix_default() question = PROMPT_CREATE % prefix try: - response = raw_input(question) + response = input(question) except EOFError: sys.exit(1) if response != YES: @@ -621,7 +621,7 @@ def _validate_info_config(opts, client, info_config): if opts.non_interactive: sys.exit(1) try: - response = raw_input(PROMPT_FIX_INFO_CONFIG) + response = input(PROMPT_FIX_INFO_CONFIG) except EOFError: sys.exit(1) if response != YES: @@ -670,7 +670,7 @@ def delete(argv): for arg in args: if interactive_mode: try: - response = raw_input(prompt % arg) + response = input(prompt % arg) except EOFError: ret_code = 1 continue diff --git a/lib/python/rosie/ws.py b/lib/python/rosie/ws.py index 2a9154a988..b7290ead48 100644 --- a/lib/python/rosie/ws.py +++ b/lib/python/rosie/ws.py @@ -61,13 +61,13 @@ def __init__(self, *args, **kwargs): ResourceLocator.default().get_util_home( "lib", "html", "template", "rosie-disco"))) db_url_map = {} - for key, node in rose_conf.get(["rosie-db"]).value.items(): + for key, node in list(rose_conf.get(["rosie-db"]).value.items()): if key.startswith("db.") and key[3:]: db_url_map[key[3:]] = node.value self.db_url_map = db_url_map if not self.db_url_map: self.db_url_map = {} - for key, db_url in self.db_url_map.items(): + for key, db_url in list(self.db_url_map.items()): setattr(self, key, RosieDiscoService(self.props, key, db_url)) @cherrypy.expose diff --git a/lib/python/rosie/ws_client.py b/lib/python/rosie/ws_client.py index b69facbf83..4288d88d70 100644 --- a/lib/python/rosie/ws_client.py +++ b/lib/python/rosie/ws_client.py @@ -85,7 +85,7 @@ def __init__(self, prefixes=None, prompt_func=None, popen=None, conf_rosie_id = conf.get(["rosie-id"], no_ignore=True) if conf_rosie_id is None: raise RosieWSClientConfError() - for key, node in conf_rosie_id.value.items(): + for key, node in list(conf_rosie_id.value.items()): if node.is_ignored() or not key.startswith("prefix-ws."): continue prefix = key.replace("prefix-ws.", "") @@ -157,11 +157,11 @@ def _get(self, method, return_ok_prefixes=False, **kwargs): # Process the requests in parallel pool = Pool(len(request_details)) results = {} - for url, request_detail in request_details.items(): + for url, request_detail in list(request_details.items()): results[url] = pool.apply_async( requests.get, [url], request_detail["requests_kwargs"]) while results: - for url, result in results.items(): + for url, result in list(results.items()): if not result.ready(): continue results.pop(url) @@ -336,7 +336,7 @@ def query_local_copies(self, user=None): results.extend(data) result_idx_branches = [] for result in results: - result_idx_branches.append((result[u"idx"], result[u"branch"])) + result_idx_branches.append((result["idx"], result["branch"])) # A branch may have been deleted - query with all_revs=1. # We only want to use all_revs on demand as it's slow. @@ -359,10 +359,10 @@ def query_local_copies(self, user=None): more_results.extend(data) new_results = {} for result in more_results: - idx_branch = (result[u"idx"], result[u"branch"]) + idx_branch = (result["idx"], result["branch"]) if (idx_branch not in new_results or - result[u"revision"] > - new_results[idx_branch][u"revision"]): + result["revision"] > + new_results[idx_branch]["revision"]): new_results.update({idx_branch: result}) for _, result in sorted(new_results.items()): results.append(result) diff --git a/lib/python/rosie/ws_client_auth.py b/lib/python/rosie/ws_client_auth.py index 6b51b2ebb9..17622d3102 100644 --- a/lib/python/rosie/ws_client_auth.py +++ b/lib/python/rosie/ws_client_auth.py @@ -37,7 +37,7 @@ import shlex import socket import sys -from urlparse import urlparse +from urllib.parse import urlparse class UndefinedRosiePrefixWS(Exception): @@ -395,7 +395,7 @@ def _prompt(self, is_retry=False): "--window-icon=" + icon_path, "--text=" + prompt)[1].strip() else: - username = raw_input(prompt) + username = input(prompt) if not username: raise KeyboardInterrupt(self.STR_CANCELLED) if username and username != self.username: diff --git a/lib/python/rosie/ws_client_cli.py b/lib/python/rosie/ws_client_cli.py index b661326c37..f6733aada8 100644 --- a/lib/python/rosie/ws_client_cli.py +++ b/lib/python/rosie/ws_client_cli.py @@ -77,7 +77,7 @@ class SuiteInfo(Event): def __str__(self): - dict_row = dict(self.args[0].items()) + dict_row = dict(list(self.args[0].items())) out = "" out = out + "id: %s\n" % dict_row["idx"] for key in sorted(dict_row.keys()): @@ -244,11 +244,11 @@ def _display_maps(opts, ws_client, dict_rows, url=None): for dict_row in dict_rows: out = opts.print_format - for key, value in dict_row.items(): + for key, value in list(dict_row.items()): if "%" + key in out: - out = unicode(out).replace( - u"%" + unicode(key), unicode(value), 1) - out = unicode(out.replace("%%", "%").expandtabs().rstrip()) + out = str(out).replace( + "%" + str(key), str(value), 1) + out = str(out.replace("%%", "%").expandtabs().rstrip()) report(SuiteEvent(out.expandtabs() + "\n"), prefix="", clip=terminal_cols) diff --git a/sbin/rosa b/sbin/rosa deleted file mode 120000 index 21c2f9999e..0000000000 --- a/sbin/rosa +++ /dev/null @@ -1 +0,0 @@ -../bin/rose \ No newline at end of file diff --git a/sbin/rosa b/sbin/rosa new file mode 100755 index 0000000000..2bdceca2e5 --- /dev/null +++ b/sbin/rosa @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------- +# (C) British Crown Copyright 2012-7 Met Office. +# +# This file is part of Rose, a framework for meteorological suites. +# +# Rose is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rose is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rose. If not, see . +#------------------------------------------------------------------------------- +# NAME +# rose - environment for running scientific suites and applications +# rosie - revision/organisation of suites for identification/exploration +# rosa - rosie's admin utilities +# +# SYNOPSIS +# # "rose", "rosie" and "rosa" has the following interfaces: +# rose UTIL [ARGS ...] # Invoke a utility +# rose help # Print help and list available utilities +# rose help UTIL ... # Print help for UTIL ... +# rose version # Print version information +# +# DESCRIPTION +# Simple launcher for utilities in the "rose", "rosie" or "rosa" +# name-spaces. Launch a "rose", "rosie" or "rosa" utility with the given +# ARGS. +#------------------------------------------------------------------------------- +if ${ROSE_DEBUG:-false}; then + set -x +fi +. $(dirname $0)/../lib/bash/rose_init +rose_init + +# Print actual command of a command alias +get_alias() { + local NAME=$1 + local ALIAS=$(sed '/^#/d' $ROSE_HOME_BIN/$ROSE_NS-$NAME || true) + if [[ $(wc -l <<<"$ALIAS") == 1 ]] \ + && grep -q "^exec \$(dirname \$0)/$ROSE_NS-.* \"\$@\"\$" <<<"$ALIAS" + then + ALIAS=${ALIAS#"exec \$(dirname \$0)/$ROSE_NS-"} + ALIAS=${ALIAS%' "$@"'} + echo $ALIAS + fi +} + +# Print help for a given utility +help_util() { + local NAME=$1 + local COMMAND=$ROSE_HOME_BIN/$ROSE_NS-$NAME + if [[ ! -r $COMMAND ]]; then + echo "$1: utility not found." >&2 + return 1 + fi + local ALIAS=$(get_alias $NAME) + if [[ -n $ALIAS ]]; then + COMMAND=$ROSE_HOME_BIN/$ROSE_NS-$ALIAS + COMMAND=${COMMAND%% *} + fi + case $(head -1 -- $COMMAND) in + *bash*) + awk '{ + if (/^# NAME/) { + do {print substr($0, 3)} while (getline && !/^#----------/); + } + }' $COMMAND | ${PAGER:-less} + ;; + *python*) + $COMMAND --help | ${PAGER:-less} # FIXME: not too pretty at the moment + ;; + esac + return +} + +# Ensure that ITEM_STR is at the beginning of PATH_STR +path_lead() { + local PATH_STR=$1 + local ITEM_STR=$2 + if [[ -z ${PATH_STR:-} ]]; then + echo "$ITEM_STR" + elif [[ "$PATH_STR" != "$ITEM_STR" && "$PATH_STR" != $ITEM_STR:* ]]; then + while [[ "$PATH_STR" == *:$ITEM_STR ]]; do + PATH_STR=${PATH_STR%:$ITEM_STR} + done + while [[ "$PATH_STR" == *:$ITEM_STR:* ]]; do + local PATH_HEAD=${PATH_STR%:$ITEM_STR:*} + local PATH_TAIL=${PATH_STR##*:$ITEM_STR:} + PATH_STR="$PATH_HEAD:$PATH_TAIL" + done + echo "$ITEM_STR:$PATH_STR" + else + echo "$PATH_STR" + fi +} + + +# Print Rose version +function print_version() { + echo "Rose $ROSE_VERSION ($ROSE_HOME)" +} + +#------------------------------------------------------------------------------- +UTIL="help" +if (($# > 0)); then + UTIL=$1 + shift 1 +fi + +case $UTIL in +help|h|?|--help|-h) + if (($# == 0)); then + { + print_version + rose_usage + echo + echo "$ROSE_NS provides the following utilities:" + for U in $(cd $ROSE_HOME_BIN && ls $ROSE_NS-*); do + NAME=$(sed "s/^$ROSE_NS-\\(.*\\)\$/\1/" <<<$U) + ALIAS=$(get_alias $NAME) + if [[ -n $ALIAS ]]; then + echo " $NAME" + echo " (=$ALIAS)" + else + echo " $NAME" + sed '1,/^# DESCRIPTION$/d;{s/^# / /;q;}' \ + $ROSE_HOME_BIN/$U + fi + done + } | ${PAGER:-less} + exit 0 + fi + RC=0 + for U in "$@"; do + if [[ $U == 'help' || $U == 'version' ]]; then + continue + fi + help_util $U || RC=$? + done + exit $RC + :;; +version|--version|-V) + print_version + exit + :;; +esac + +COMMAND=$(dirname $0)/$ROSE_NS-$UTIL +if [[ ! -f $COMMAND || ! -x $COMMAND ]]; then + echo "$ROSE_NS: $UTIL: unknown utility. Abort." >&2 + echo "Type \"$ROSE_NS help\" for a list of utilities." >&2 + exit 1 +fi +if (($# > 0)) && [[ $1 == '--help' || $1 == '-h' ]]; then + help_util $UTIL + exit +fi +PATH=$(path_lead "${PATH:-}" "$ROSE_HOME_BIN") +PYTHONPATH=$(path_lead "${PYTHONPATH:-}" "$ROSE_HOME/lib/python") +ROSE_UTIL=$UTIL +export PATH PYTHONPATH ROSE_UTIL +exec $COMMAND "$@" diff --git a/sbin/rosa-db-create b/sbin/rosa-db-create index ffce9027e4..22b3ad637a 100755 --- a/sbin/rosa-db-create +++ b/sbin/rosa-db-create @@ -30,4 +30,4 @@ # determine the list of databases to create. # Does not override existing databases. #------------------------------------------------------------------------------- -exec python -m rosie.db_create "$@" +exec python3 -m rosie.db_create "$@" diff --git a/sbin/rosa-rpmbuild b/sbin/rosa-rpmbuild index 96b41fa66e..34a549b6bc 100755 --- a/sbin/rosa-rpmbuild +++ b/sbin/rosa-rpmbuild @@ -81,7 +81,7 @@ rm -fr %{buildroot} mkdir -p %{buildroot}/etc/bash_completion.d %{buildroot}/opt %{buildroot}/usr/bin cp -p %_sourcedir/$NAME-$REV_BASE_DOT/etc/rose-bash-completion %{buildroot}/etc/bash_completion.d cp -pr %_sourcedir/$NAME-$REV_BASE_DOT %{buildroot}/opt/$NAME -python -m compileall %{buildroot}/opt/$NAME/lib +python3 -m compileall %{buildroot}/opt/$NAME/lib cp -p %_sourcedir/$NAME-$REV_BASE_DOT/usr/bin/rose %{buildroot}/usr/bin/rose cp -p %_sourcedir/$NAME-$REV_BASE_DOT/usr/bin/rose %{buildroot}/usr/bin/rosie diff --git a/sbin/rosa-svn-post-commit b/sbin/rosa-svn-post-commit index cdb2d2cde0..76c08505d5 100755 --- a/sbin/rosa-svn-post-commit +++ b/sbin/rosa-svn-post-commit @@ -26,4 +26,4 @@ # DESCRIPTION # Update the Rosie discovery database for an SVN changeset. #------------------------------------------------------------------------------- -exec python -m rosie.svn_post_commit "$@" +exec python3 -m rosie.svn_post_commit "$@" diff --git a/sbin/rosa-svn-pre-commit b/sbin/rosa-svn-pre-commit index 7424c774cd..50b60b86b2 100755 --- a/sbin/rosa-svn-pre-commit +++ b/sbin/rosa-svn-pre-commit @@ -26,4 +26,4 @@ # DESCRIPTION # Ensure that an SVN commit conforms to the rules of Rosie. #------------------------------------------------------------------------------- -exec python -m rosie.svn_pre_commit "$@" +exec python3 -m rosie.svn_pre_commit "$@" diff --git a/sbin/rosa-ws b/sbin/rosa-ws index 1c917cd4fd..f4581fcfec 100755 --- a/sbin/rosa-ws +++ b/sbin/rosa-ws @@ -32,4 +32,4 @@ # # This command is deprecated. Use "rosie disco" instead. #------------------------------------------------------------------------------- -exec python -m rosie.ws "$@" +exec python3 -m rosie.ws "$@"