From 2504fe1e2df73c58cd6ae70a899768299903f92b Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Sun, 19 Nov 2023 20:04:40 -0800 Subject: [PATCH] [SPARK-45982][INFRA] re-org R package installations ### What changes were proposed in this pull request? re-org R package installations after this PR, there is no R package installation in `build_and_test` ### Why are the changes needed? current R packages installations is somewhat messy, e.g. - `devtools` is installed 4 times (twice in dockerfile, twice in job `lint`) - `testthat` is installed twice - `roxygen2` is installed twice ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? ci ### Was this patch authored or co-authored using generative AI tooling? no Closes #43881 from zhengruifeng/infra_r_cleanup. Authored-by: Ruifeng Zheng Signed-off-by: Dongjoon Hyun --- .github/workflows/build_and_test.yml | 8 +------- dev/infra/Dockerfile | 8 ++++++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 135f43d07b134..d07a34e61513a 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -715,10 +715,7 @@ jobs: - name: JS linter run: ./dev/lint-js - name: Install R linter dependencies and SparkR - run: | - Rscript -e "install.packages(c('devtools'), repos='https://cloud.r-project.org/')" - Rscript -e "devtools::install_version('lintr', version='2.0.1', repos='https://cloud.r-project.org')" - ./R/install-dev.sh + run: ./R/install-dev.sh - name: Install dependencies for documentation generation run: | # TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes. @@ -731,9 +728,6 @@ jobs: python3.9 -m pip install ipython_genutils # See SPARK-38517 python3.9 -m pip install sphinx_plotly_directive 'numpy>=1.20.0' pyarrow pandas 'plotly>=4.8' python3.9 -m pip install 'docutils<0.18.0' # See SPARK-39421 - Rscript -e "install.packages(c('devtools', 'testthat', 'knitr', 'rmarkdown', 'markdown', 'e1071', 'roxygen2', 'ggplot2', 'mvtnorm', 'statmod'), repos='https://cloud.r-project.org/')" - Rscript -e "devtools::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')" - Rscript -e "devtools::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')" gem install bundler cd docs bundle install diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index cc00289b963bd..3aac09aa5c19e 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -79,11 +79,15 @@ RUN gpg --keyserver hkps://keyserver.ubuntu.com --recv-key E298A3A825C0D65DFD57C RUN gpg -a --export E084DAB9 | apt-key add - RUN add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/' -RUN Rscript -e "install.packages(c('knitr', 'markdown', 'rmarkdown', 'testthat', 'devtools', 'e1071', 'survival', 'arrow', 'roxygen2', 'xml2'), repos='https://cloud.r-project.org/')" +RUN Rscript -e "install.packages(c('devtools', 'knitr', 'markdown', \ + 'rmarkdown', 'testthat', 'devtools', 'e1071', 'survival', 'arrow', \ + 'ggplot2', 'mvtnorm', 'statmod', 'xml2'), repos='https://cloud.r-project.org/')" # See more in SPARK-39959, roxygen2 < 7.2.1 -RUN Rscript -e "install.packages(c('devtools'), repos='https://cloud.r-project.org/')" RUN Rscript -e "devtools::install_version('roxygen2', version='7.2.0', repos='https://cloud.r-project.org')" +RUN Rscript -e "devtools::install_version('lintr', version='2.0.1', repos='https://cloud.r-project.org')" +RUN Rscript -e "devtools::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')" +RUN Rscript -e "devtools::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')" # See more in SPARK-39735 ENV R_LIBS_SITE "/usr/local/lib/R/site-library:${R_LIBS_SITE}:/usr/lib/R/library"