Mise à jour des dépendances R et amélioration du processus de build
Cette mise à jour apporte plusieurs améliorations : - Ajout d'un script R dédié pour l'installation des packages (install_R_packages.R) - Refactorisation du Dockerfile pour une meilleure gestion des dépendances R et système - Amélioration du script start-jupyterhub.sh avec gestion dynamique de docker-compose et vérification des timestamps - Mise à jour de la documentation dans Readme.md pour refléter les nouvelles images Docker et les changements de structure - Ajout de 'reserve' dans .gitignore
This commit is contained in:
1
.gitignore
vendored
1
.gitignore
vendored
@@ -18,3 +18,4 @@ ncbitaxo_*
|
|||||||
Readme_files
|
Readme_files
|
||||||
Readme.html
|
Readme.html
|
||||||
tmp.*
|
tmp.*
|
||||||
|
reserve
|
||||||
|
|||||||
@@ -75,10 +75,13 @@ OBIJupyterHub
|
|||||||
├── start-jupyterhub.sh - single entry point (build + render + start)
|
├── start-jupyterhub.sh - single entry point (build + render + start)
|
||||||
├── obijupyterhub - Docker images and stack definitions
|
├── obijupyterhub - Docker images and stack definitions
|
||||||
│ ├── docker-compose.yml
|
│ ├── docker-compose.yml
|
||||||
│ ├── Dockerfile
|
│ ├── install_R_packages.R - An R script used to install all need R packages
|
||||||
│ ├── Dockerfile.hub
|
│ ├── Dockerfile - Image used by the students
|
||||||
|
│ ├── Dockerfile.hub - Image for the jupyter hub
|
||||||
|
│ ├── Dockerfile.builder - Image for the builder
|
||||||
│ └── jupyterhub_config.py
|
│ └── jupyterhub_config.py
|
||||||
├── jupyterhub_volumes - data persisted on the host
|
├── jupyterhub_volumes - data persisted on the host
|
||||||
|
│ ├── builder - R packages cache for building lectures
|
||||||
│ ├── course - read-only for students (notebooks, data, bin, R packages)
|
│ ├── course - read-only for students (notebooks, data, bin, R packages)
|
||||||
│ ├── shared - shared read/write space for everyone
|
│ ├── shared - shared read/write space for everyone
|
||||||
│ ├── users - per-user persistent data
|
│ ├── users - per-user persistent data
|
||||||
@@ -86,7 +89,7 @@ OBIJupyterHub
|
|||||||
└── web_src - Quarto sources for the course website
|
└── web_src - Quarto sources for the course website
|
||||||
```
|
```
|
||||||
|
|
||||||
Note: The `obijupyterhub/` directory also contains `Dockerfile.builder` which provides the build environment, the `tools/` directory contains utility scripts including `install_quarto_deps.R` for automatic R dependency detection, and `jupyterhub_volumes/builder/` stores cached R packages for faster builds.
|
Note: The `tools/` directory contains utility scripts including `install_quarto_deps.R` for automatic R dependency detection.
|
||||||
|
|
||||||
3) Prepare course materials (optional before first run):
|
3) Prepare course materials (optional before first run):
|
||||||
- Put notebooks, datasets, scripts, binaries, or PDFs for students under `jupyterhub_volumes/course/`. They will appear read-only at `/home/jovyan/work/course/`.
|
- Put notebooks, datasets, scripts, binaries, or PDFs for students under `jupyterhub_volumes/course/`. They will appear read-only at `/home/jovyan/work/course/`.
|
||||||
|
|||||||
@@ -19,37 +19,63 @@ RUN TEMP=. curl -L https://raw.githubusercontent.com/metabarcoding/obitools4/mas
|
|||||||
&& cp $HOME/obitools-build/bin/* /usr/local/bin
|
&& cp $HOME/obitools-build/bin/* /usr/local/bin
|
||||||
RUN ls -l /usr/local/bin
|
RUN ls -l /usr/local/bin
|
||||||
|
|
||||||
|
|
||||||
# ---------- Stage 2 : image finale ----------
|
# ---------- Stage 2 : image finale ----------
|
||||||
FROM jupyter/base-notebook:latest
|
FROM jupyter/base-notebook:latest
|
||||||
|
|
||||||
USER root
|
USER root
|
||||||
|
|
||||||
# Installer seulement les dépendances d'exécution (sans build-essential)
|
# Installer seulement les dépendances d'exécution (sans build-essential)
|
||||||
RUN apt-get update && apt-get install -y \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
# R et dépendances de base
|
||||||
r-base \
|
r-base \
|
||||||
libcurl4-openssl-dev libssl-dev libxml2-dev \
|
r-base-dev \
|
||||||
|
libcurl4-openssl-dev \
|
||||||
|
libssl-dev \
|
||||||
|
libxml2-dev \
|
||||||
|
libicu-dev \
|
||||||
|
zlib1g-dev \
|
||||||
|
# Polices et rendu graphique (indispensable pour ggplot2, ragg, etc.)
|
||||||
|
libharfbuzz-dev \
|
||||||
|
libfribidi-dev \
|
||||||
|
libfontconfig1-dev \
|
||||||
|
libfreetype6-dev \
|
||||||
|
libpng-dev \
|
||||||
|
libtiff5-dev \
|
||||||
|
libjpeg-dev \
|
||||||
|
pandoc \
|
||||||
|
# Outils de compilation et gestion de versions
|
||||||
|
libgit2-dev \
|
||||||
|
cmake \
|
||||||
|
# Utilitaires systèmes déjà présents dans votre Dockerfile
|
||||||
curl \
|
curl \
|
||||||
|
wget \
|
||||||
git \
|
git \
|
||||||
texlive-xetex texlive-fonts-recommended texlive-plain-generic \
|
vim \
|
||||||
ruby ruby-dev \
|
nano \
|
||||||
vim nano \
|
less \
|
||||||
|
gdebi-core \
|
||||||
|
ripgrep \
|
||||||
|
# Pour générer des PDF/rapports depuis R Markdown / Jupyter
|
||||||
|
texlive-xetex \
|
||||||
|
texlive-luatex \
|
||||||
|
texlive-fonts-recommended \
|
||||||
|
texlive-fonts-extra \
|
||||||
|
texlive-latex-extra \
|
||||||
|
texlive-plain-generic \
|
||||||
|
lmodern \
|
||||||
|
fonts-lmodern \
|
||||||
|
librsvg2-bin \
|
||||||
|
cm-super \
|
||||||
|
# Ruby (si vous en avez besoin pour autre chose)
|
||||||
|
ruby \
|
||||||
|
ruby-dev \
|
||||||
&& apt-get clean \
|
&& apt-get clean \
|
||||||
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
|
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
|
||||||
|
|
||||||
# Installer R et packages
|
# Installer R et packages
|
||||||
RUN R -e "install.packages(c('IRkernel','tidyverse','vegan','ade4','BiocManager','remotes','igraph'), \
|
COPY install_R_packages.R /tmp/install_R_packages.R
|
||||||
dependencies=TRUE, \
|
RUN Rscript /tmp/install_R_packages.R --no-save --no-restore && \
|
||||||
repos='http://cran.rstudio.com/')" && \
|
rm -rf /tmp/Rtmp* /tmp/install_R_packages.R
|
||||||
R -e "BiocManager::install('biomformat')" && \
|
|
||||||
R -e "remotes::install_github('metabaRfactory/metabaR')" && \
|
|
||||||
R -e "remotes::install_git('https://forge.metabarcoding.org/obitools/ROBIUtils.git')" && \
|
|
||||||
R -e "remotes::install_git('https://forge.metabarcoding.org/obitools/ROBITaxonomy.git')" && \
|
|
||||||
R -e "remotes::install_git('https://forge.metabarcoding.org/obitools/ROBITools.git')" && \
|
|
||||||
R -e "remotes::install_git('https://forge.metabarcoding.org/obitools/ROBITaxonomy.git')" && \
|
|
||||||
R -e "remotes::install_git('https://forge.metabarcoding.org/MetabarcodingSchool/biodiversity-metrics.git')" && \
|
|
||||||
R -e "IRkernel::installspec(user = FALSE)" && \
|
|
||||||
rm -rf /tmp/Rtmp*
|
|
||||||
|
|
||||||
# Installer les autres outils
|
# Installer les autres outils
|
||||||
RUN pip install --no-cache-dir bash_kernel csvkit && \
|
RUN pip install --no-cache-dir bash_kernel csvkit && \
|
||||||
@@ -57,9 +83,17 @@ RUN pip install --no-cache-dir bash_kernel csvkit && \
|
|||||||
|
|
||||||
RUN gem install youplot
|
RUN gem install youplot
|
||||||
|
|
||||||
|
# Installation de Quarto (multi-arch)
|
||||||
|
RUN ARCH=$(dpkg --print-architecture) && \
|
||||||
|
QUARTO_VERSION="1.8.27" && \
|
||||||
|
wget https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-${ARCH}.deb && \
|
||||||
|
gdebi --non-interactive quarto-${QUARTO_VERSION}-linux-${ARCH}.deb && \
|
||||||
|
rm quarto-${QUARTO_VERSION}-linux-${ARCH}.deb
|
||||||
|
|
||||||
# Set permissions for Jupyter user
|
# Set permissions for Jupyter user
|
||||||
RUN mkdir -p /home/${NB_USER}/.local/share/jupyter && \
|
RUN mkdir -p /home/${NB_USER}/.local/share/jupyter && \
|
||||||
chown -R ${NB_UID}:${NB_GID} /home/${NB_USER}
|
chown -R ${NB_UID}:${NB_GID} /home/${NB_USER}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# Copier uniquement le binaire csvlens du builder
|
# Copier uniquement le binaire csvlens du builder
|
||||||
|
|||||||
43
obijupyterhub/install_R_packages.R
Normal file
43
obijupyterhub/install_R_packages.R
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
#!/usr/bin/env Rscript
|
||||||
|
|
||||||
|
# Installer pak (lui-même en binaire si possible)
|
||||||
|
install.packages("pak", repos = sprintf("https://r-lib.github.io/p/pak/stable/%s/%s/%s", .Platform$pkgType, R.Version()$os, R.Version()$arch))
|
||||||
|
pak::pkg_install("cli")
|
||||||
|
|
||||||
|
# Détection automatique du système et installation de tous les paquets en binaire
|
||||||
|
pak::pkg_install(c(
|
||||||
|
"IRkernel",
|
||||||
|
"tidyverse",
|
||||||
|
"vegan",
|
||||||
|
"ade4",
|
||||||
|
"BiocManager",
|
||||||
|
"remotes",
|
||||||
|
"igraph",
|
||||||
|
"Rdpack"
|
||||||
|
))
|
||||||
|
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
# Paquets Bioconductor (toujours via BiocManager)
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
pak::pkg_install("bioc::biomformat")
|
||||||
|
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
# Paquets depuis GitHub / dépôts git
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
pak::pkg_install("metabaRfactory/metabaR")
|
||||||
|
pak::pkg_install("git::https://forge.metabarcoding.org/obitools/ROBIUtils.git")
|
||||||
|
pak::pkg_install("git::https://forge.metabarcoding.org/obitools/ROBITaxonomy.git")
|
||||||
|
pak::pkg_install("git::https://forge.metabarcoding.org/obitools/ROBITools.git")
|
||||||
|
pak::pkg_install("git::https://forge.metabarcoding.org/MetabarcodingSchool/biodiversity-metrics.git")
|
||||||
|
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
# Installation du noyau Jupyter pour IRkernel
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
# Si on est root -> installation système, sinon -> user
|
||||||
|
if (Sys.info()["user"] == "root") {
|
||||||
|
IRkernel::installspec(user = FALSE)
|
||||||
|
} else {
|
||||||
|
IRkernel::installspec(user = TRUE)
|
||||||
|
}
|
||||||
|
|
||||||
|
cat("\n✅ Tous les paquets R ont été installés avec succès.\n")
|
||||||
@@ -35,6 +35,10 @@ Options:
|
|||||||
EOF
|
EOF
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
dockercompose=$(which docker-compose || echo 'docker compose')
|
||||||
|
|
||||||
|
|
||||||
while [[ $# -gt 0 ]]; do
|
while [[ $# -gt 0 ]]; do
|
||||||
case "$1" in
|
case "$1" in
|
||||||
--no-build|--offline) NO_BUILD=true ;;
|
--no-build|--offline) NO_BUILD=true ;;
|
||||||
@@ -78,31 +82,57 @@ if [ ! -f "Dockerfile" ] || [ ! -f "docker-compose.yml" ]; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
get_file_timestamp() {
|
||||||
|
local file="$1"
|
||||||
|
case "$(uname -s)" in
|
||||||
|
Linux)
|
||||||
|
stat -c %Y "$file"
|
||||||
|
;;
|
||||||
|
Darwin)
|
||||||
|
# BSD stat : -f pour format, %m = timestamp modification
|
||||||
|
stat -f %m "$file"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Système non supporté" >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
check_if_image_needs_rebuild() {
|
check_if_image_needs_rebuild() {
|
||||||
local image_name="$1"
|
local image_name="$1"
|
||||||
local dockerfile="$2"
|
local dockerfile="$2"
|
||||||
|
|
||||||
|
echo -e "${BLUE}Checking image ${image_name}...${NC}"
|
||||||
|
|
||||||
# Check if image exists
|
# Check if image exists
|
||||||
if ! docker image inspect "$image_name" >/dev/null 2>&1; then
|
if ! docker image inspect "$image_name" >/dev/null 2>&1; then
|
||||||
|
echo -e "${YELLOW}Docker image ${image_name} doesn't exist.${NC}"
|
||||||
return 0 # Need to build (image doesn't exist)
|
return 0 # Need to build (image doesn't exist)
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# If force rebuild, always rebuild
|
# If force rebuild, always rebuild
|
||||||
if $FORCE_REBUILD; then
|
if $FORCE_REBUILD; then
|
||||||
|
echo -e "${YELLOW}Docker image build is forced.${NC}"
|
||||||
return 0 # Need to rebuild
|
return 0 # Need to rebuild
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Compare Dockerfile modification time with image creation time
|
# Compare Dockerfile modification time with image creation time
|
||||||
if [ -f "$dockerfile" ]; then
|
if [ -f "$dockerfile" ]; then
|
||||||
local dockerfile_mtime=$(stat -c %Y "$dockerfile" 2>/dev/null || echo 0)
|
local dockerfile_mtime=$(get_file_timestamp "$dockerfile" 2>/dev/null || echo 0)
|
||||||
local image_created=$(docker image inspect "$image_name" --format='{{.Created}}' 2>/dev/null | sed 's/\.000000000//' | xargs -I {} date -d "{}" +%s 2>/dev/null || echo 0)
|
local image_created=$(docker image inspect "$image_name" --format='{{.Created}}' 2>/dev/null \
|
||||||
|
| sed -E 's/\.[0-9]+//' \
|
||||||
|
| (read d; if [[ "$(uname -s)" == "Darwin" ]]; then date -ju -f "%Y-%m-%dT%H:%M:%S" "${d%Z}" +%s; else date -d "$d" +%s; fi) 2>/dev/null || echo 0)
|
||||||
|
|
||||||
|
echo -e "${BLUE}Docker image ${image_name} created at: ${image_created}.${NC}"
|
||||||
|
echo -e "${BLUE}Docker file ${dockerfile} modified at: ${dockerfile_mtime}.${NC}"
|
||||||
|
|
||||||
if [ "$dockerfile_mtime" -gt "$image_created" ]; then
|
if [ "$dockerfile_mtime" -gt "$image_created" ]; then
|
||||||
echo -e "${YELLOW}Dockerfile is newer than image, rebuild needed${NC}"
|
echo -e "${YELLOW}Dockerfile is newer than image, rebuild needed${NC}"
|
||||||
return 0 # Need to rebuild
|
return 0 # Need to rebuild
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
return 1 # No need to rebuild
|
return 1 # No need to rebuild
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -112,7 +142,7 @@ build_builder_image() {
|
|||||||
if $FORCE_REBUILD; then
|
if $FORCE_REBUILD; then
|
||||||
build_flag+=(--no-cache)
|
build_flag+=(--no-cache)
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo -e "${BLUE}Building builder image...${NC}"
|
echo -e "${BLUE}Building builder image...${NC}"
|
||||||
docker build "${build_flag[@]}" -t "$BUILDER_IMAGE" -f Dockerfile.builder .
|
docker build "${build_flag[@]}" -t "$BUILDER_IMAGE" -f Dockerfile.builder .
|
||||||
@@ -136,7 +166,7 @@ run_in_builder() {
|
|||||||
|
|
||||||
stop_stack() {
|
stop_stack() {
|
||||||
echo -e "${BLUE}Stopping existing containers...${NC}"
|
echo -e "${BLUE}Stopping existing containers...${NC}"
|
||||||
docker-compose down 2>/dev/null || true
|
${dockercompose} down 2>/dev/null || true
|
||||||
|
|
||||||
echo -e "${BLUE}Cleaning up student containers...${NC}"
|
echo -e "${BLUE}Cleaning up student containers...${NC}"
|
||||||
docker ps -aq --filter name=jupyter- | xargs -r docker rm -f 2>/dev/null || true
|
docker ps -aq --filter name=jupyter- | xargs -r docker rm -f 2>/dev/null || true
|
||||||
@@ -237,7 +267,7 @@ build_website() {
|
|||||||
start_stack() {
|
start_stack() {
|
||||||
echo ""
|
echo ""
|
||||||
echo -e "${BLUE}Starting JupyterHub...${NC}"
|
echo -e "${BLUE}Starting JupyterHub...${NC}"
|
||||||
docker-compose up -d --remove-orphans
|
${dockercompose} up -d --remove-orphans
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo -e "${YELLOW}Waiting for JupyterHub to start...${NC}"
|
echo -e "${YELLOW}Waiting for JupyterHub to start...${NC}"
|
||||||
@@ -268,13 +298,13 @@ print_success() {
|
|||||||
echo " - work/course/R_packages/ : shared R packages by prof (read-only)"
|
echo " - work/course/R_packages/ : shared R packages by prof (read-only)"
|
||||||
echo " - work/course/bin/ : shared executables (in PATH)"
|
echo " - work/course/bin/ : shared executables (in PATH)"
|
||||||
echo ""
|
echo ""
|
||||||
echo "To view logs: docker-compose logs -f jupyterhub"
|
echo "To view logs: ${dockercompose} logs -f jupyterhub"
|
||||||
echo "To stop: docker-compose down"
|
echo "To stop: ${dockercompose} down"
|
||||||
echo ""
|
echo ""
|
||||||
else
|
else
|
||||||
echo ""
|
echo ""
|
||||||
echo -e "${YELLOW}JupyterHub container doesn't seem to be starting${NC}"
|
echo -e "${YELLOW}JupyterHub container doesn't seem to be starting${NC}"
|
||||||
echo "Check logs with: docker-compose logs jupyterhub"
|
echo "Check logs with: ${dockercompose} logs jupyterhub"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user