🔧 Add selective image rebuild flags and enhance R dependency scanning
- Added --rebuild-builder, -student, hub flags to start-jupyterhub.sh for granular Docker rebuilds - Updated check_if_image_needs_rebuild to accept per-image force flag and propagate no-cache option - Added libuv1-dev dependency in Dockerfile.builder (likely for quarto or R runtime) - Rewrote install_quarto_deps.R to: a) Manually parse library()/require() and remotes::install_git/github calls b) Distinguish between quarto-required packages (must reside in persistent target_lib) c), CRAN and git/github dependencies d) Install with robust error handling, skipping unavailable packages - Removed dependency on attachment package for scanning
This commit is contained in:
Eric Coissac
@@ -32,6 +32,7 @@ RUN apt-get update \
|
||||
libpng-dev \
|
||||
libtiff5-dev \
|
||||
libjpeg-dev \
|
||||
libuv1-dev \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
+27
-14
@@ -20,6 +20,9 @@ FORCE_REBUILD=false
|
||||
STOP_SERVER=false
|
||||
UPDATE_LECTURES=false
|
||||
BUILD_OBIDOC=false
|
||||
REBUILD_BUILDER=false
|
||||
REBUILD_STUDENT=false
|
||||
REBUILD_HUB=false
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
@@ -27,7 +30,10 @@ Usage: ./start-jupyterhub.sh [options]
|
||||
|
||||
Options:
|
||||
--no-build | --offline Skip Docker image builds (use existing images)
|
||||
--force-rebuild Rebuild images without cache
|
||||
--force-rebuild Rebuild all images without cache
|
||||
--rebuild-builder Force rebuild the builder image only
|
||||
--rebuild-student Force rebuild the student image only
|
||||
--rebuild-hub Force rebuild the JupyterHub image only
|
||||
--stop-server Stop the stack and remove student containers, then exit
|
||||
--update-lectures Rebuild the course website only (no Docker stop/start)
|
||||
--build-obidoc Force rebuild of obidoc documentation
|
||||
@@ -43,6 +49,9 @@ while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--no-build|--offline) NO_BUILD=true ;;
|
||||
--force-rebuild) FORCE_REBUILD=true ;;
|
||||
--rebuild-builder) REBUILD_BUILDER=true ;;
|
||||
--rebuild-student) REBUILD_STUDENT=true ;;
|
||||
--rebuild-hub) REBUILD_HUB=true ;;
|
||||
--stop-server) STOP_SERVER=true ;;
|
||||
--update-lectures) UPDATE_LECTURES=true ;;
|
||||
--build-obidoc) BUILD_OBIDOC=true ;;
|
||||
@@ -102,6 +111,7 @@ get_file_timestamp() {
|
||||
check_if_image_needs_rebuild() {
|
||||
local image_name="$1"
|
||||
local dockerfile="$2"
|
||||
local force="${3:-false}"
|
||||
|
||||
echo -e "${BLUE}Checking image ${image_name}...${NC}"
|
||||
|
||||
@@ -111,8 +121,8 @@ check_if_image_needs_rebuild() {
|
||||
return 0 # Need to build (image doesn't exist)
|
||||
fi
|
||||
|
||||
# If force rebuild, always rebuild
|
||||
if $FORCE_REBUILD; then
|
||||
# If force rebuild (global or per-image), always rebuild
|
||||
if $FORCE_REBUILD || $force; then
|
||||
echo -e "${YELLOW}Docker image build is forced.${NC}"
|
||||
return 0 # Need to rebuild
|
||||
fi
|
||||
@@ -137,9 +147,9 @@ check_if_image_needs_rebuild() {
|
||||
}
|
||||
|
||||
build_builder_image() {
|
||||
if check_if_image_needs_rebuild "$BUILDER_IMAGE" "Dockerfile.builder"; then
|
||||
if check_if_image_needs_rebuild "$BUILDER_IMAGE" "Dockerfile.builder" "$REBUILD_BUILDER"; then
|
||||
local build_flag=()
|
||||
if $FORCE_REBUILD; then
|
||||
if $FORCE_REBUILD || $REBUILD_BUILDER; then
|
||||
build_flag+=(--no-cache)
|
||||
fi
|
||||
|
||||
@@ -178,25 +188,28 @@ build_images() {
|
||||
return
|
||||
fi
|
||||
|
||||
local build_flag=()
|
||||
if $FORCE_REBUILD; then
|
||||
build_flag+=(--no-cache)
|
||||
fi
|
||||
|
||||
# Check and build student image
|
||||
if check_if_image_needs_rebuild "jupyterhub-student:latest" "Dockerfile"; then
|
||||
if check_if_image_needs_rebuild "jupyterhub-student:latest" "Dockerfile" "$REBUILD_STUDENT"; then
|
||||
local student_flag=()
|
||||
if $FORCE_REBUILD || $REBUILD_STUDENT; then
|
||||
student_flag+=(--no-cache)
|
||||
fi
|
||||
echo ""
|
||||
echo -e "${BLUE}Building student image...${NC}"
|
||||
docker build "${build_flag[@]}" -t jupyterhub-student:latest -f Dockerfile .
|
||||
docker build "${student_flag[@]}" -t jupyterhub-student:latest -f Dockerfile .
|
||||
else
|
||||
echo -e "${BLUE}Student image is up to date, skipping build.${NC}"
|
||||
fi
|
||||
|
||||
# Check and build JupyterHub image
|
||||
if check_if_image_needs_rebuild "jupyterhub-hub:latest" "Dockerfile.hub"; then
|
||||
if check_if_image_needs_rebuild "jupyterhub-hub:latest" "Dockerfile.hub" "$REBUILD_HUB"; then
|
||||
local hub_flag=()
|
||||
if $FORCE_REBUILD || $REBUILD_HUB; then
|
||||
hub_flag+=(--no-cache)
|
||||
fi
|
||||
echo ""
|
||||
echo -e "${BLUE}Building JupyterHub image...${NC}"
|
||||
docker build "${build_flag[@]}" -t jupyterhub-hub:latest -f Dockerfile.hub .
|
||||
docker build "${hub_flag[@]}" -t jupyterhub-hub:latest -f Dockerfile.hub .
|
||||
else
|
||||
echo -e "${BLUE}JupyterHub image is up to date, skipping build.${NC}"
|
||||
fi
|
||||
|
||||
+109
-26
@@ -1,17 +1,15 @@
|
||||
#!/usr/bin/env Rscript
|
||||
# Script to dynamically detect and install R dependencies from Quarto files
|
||||
# Uses the {attachment} package to scan .qmd files for library()/require() calls
|
||||
# Script to dynamically detect and install R dependencies from Quarto files.
|
||||
# Scans library()/require() calls and remotes::install_git/github() calls.
|
||||
|
||||
args <- commandArgs(trailingOnly = TRUE)
|
||||
quarto_dir <- if (length(args) > 0) args[1] else "."
|
||||
|
||||
# Target library for installing packages (the mounted volume)
|
||||
target_lib <- "/usr/local/lib/R/site-library"
|
||||
|
||||
cat("Scanning Quarto files in:", quarto_dir, "\n")
|
||||
cat("Target library:", target_lib, "\n")
|
||||
|
||||
# Find all .qmd files
|
||||
qmd_files <- list.files(
|
||||
path = quarto_dir,
|
||||
pattern = "\\.qmd$",
|
||||
@@ -26,34 +24,119 @@ if (length(qmd_files) == 0) {
|
||||
|
||||
cat("Found", length(qmd_files), "Quarto files\n")
|
||||
|
||||
# Extract dependencies using attachment
|
||||
deps <- attachment::att_from_rmds(qmd_files, inline = TRUE)
|
||||
|
||||
if (length(deps) == 0) {
|
||||
cat("No R package dependencies detected.\n")
|
||||
quit(status = 0)
|
||||
# Extract package names from library()/require() calls
|
||||
extract_cran_packages <- function(files) {
|
||||
pattern <- "(?:library|require)\\s*\\(\\s*['\"]?([A-Za-z0-9._]+)['\"]?"
|
||||
pkgs <- character(0)
|
||||
for (f in files) {
|
||||
lines <- tryCatch(readLines(f, warn = FALSE), error = function(e) character(0))
|
||||
m <- regmatches(lines, gregexpr(pattern, lines, perl = TRUE))
|
||||
hits <- unlist(m)
|
||||
if (length(hits) > 0) {
|
||||
extracted <- sub(
|
||||
"(?:library|require)\\s*\\(\\s*['\"]?([A-Za-z0-9._]+)['\"]?.*",
|
||||
"\\1", hits, perl = TRUE
|
||||
)
|
||||
pkgs <- c(pkgs, extracted)
|
||||
}
|
||||
}
|
||||
unique(pkgs)
|
||||
}
|
||||
|
||||
cat("\nDetected R packages:\n")
|
||||
cat(paste(" -", deps, collapse = "\n"), "\n\n")
|
||||
# Extract git/github URLs from remotes::install_git/github() calls
|
||||
extract_git_packages <- function(files) {
|
||||
# Matches remotes::install_git('url') or remotes::install_github('user/repo')
|
||||
pattern <- "remotes::install_(git|github)\\s*\\(\\s*['\"]([^'\"]+)['\"]"
|
||||
result <- list()
|
||||
for (f in files) {
|
||||
lines <- tryCatch(readLines(f, warn = FALSE), error = function(e) character(0))
|
||||
text <- paste(lines, collapse = "\n")
|
||||
m <- gregexpr(pattern, text, perl = TRUE)
|
||||
hits <- regmatches(text, m)[[1]]
|
||||
for (hit in hits) {
|
||||
type <- sub("remotes::install_(git|github).*", "\\1", hit, perl = TRUE)
|
||||
url <- sub("remotes::install_(?:git|github)\\s*\\(\\s*['\"]([^'\"]+)['\"].*",
|
||||
"\\1", hit, perl = TRUE)
|
||||
result[[length(result) + 1]] <- list(type = type, url = url)
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
cran_deps <- extract_cran_packages(qmd_files)
|
||||
git_deps <- extract_git_packages(qmd_files)
|
||||
|
||||
# Quarto's implicit runtime dependencies — must be in target_lib (the persistent
|
||||
# volume), not just somewhere in libPaths, because Quarto spawns its own R session.
|
||||
quarto_required <- c("rmarkdown", "knitr")
|
||||
if (length(git_deps) > 0) quarto_required <- c(quarto_required, "remotes")
|
||||
|
||||
cat("\nDetected CRAN packages:\n")
|
||||
cat(paste(" -", unique(c(quarto_required, cran_deps)), collapse = "\n"), "\n")
|
||||
|
||||
if (length(git_deps) > 0) {
|
||||
cat("\nDetected git/github packages:\n")
|
||||
for (d in git_deps) cat(" -", d$type, ":", d$url, "\n")
|
||||
}
|
||||
cat("\n")
|
||||
|
||||
# --- Install CRAN packages ---
|
||||
|
||||
# Filter out base R packages that are always available
|
||||
base_pkgs <- rownames(installed.packages(priority = "base"))
|
||||
deps <- setdiff(deps, base_pkgs)
|
||||
|
||||
# Check which packages are not installed
|
||||
installed <- rownames(installed.packages())
|
||||
to_install <- setdiff(deps, installed)
|
||||
# quarto_required: check only in target_lib so they are guaranteed to be there
|
||||
installed_in_target <- rownames(installed.packages(lib.loc = target_lib))
|
||||
quarto_missing <- setdiff(quarto_required, c(base_pkgs, installed_in_target))
|
||||
|
||||
# other deps: check anywhere in libPaths (they just need to be loadable)
|
||||
cran_deps <- setdiff(cran_deps, c(base_pkgs, quarto_required))
|
||||
installed <- rownames(installed.packages())
|
||||
to_install <- unique(c(quarto_missing, setdiff(cran_deps, installed)))
|
||||
|
||||
if (length(to_install) == 0) {
|
||||
cat("All required packages are already installed.\n")
|
||||
cat("All CRAN packages already installed.\n")
|
||||
} else {
|
||||
cat("Installing missing packages:", paste(to_install, collapse = ", "), "\n\n")
|
||||
install.packages(
|
||||
to_install,
|
||||
lib = target_lib,
|
||||
repos = "https://cloud.r-project.org/",
|
||||
dependencies = TRUE
|
||||
)
|
||||
cat("\nPackage installation complete.\n")
|
||||
cat("Installing CRAN packages:", paste(to_install, collapse = ", "), "\n\n")
|
||||
failed <- character(0)
|
||||
for (pkg in to_install) {
|
||||
result <- tryCatch({
|
||||
withCallingHandlers(
|
||||
install.packages(pkg, lib = target_lib, repos = "https://cloud.r-project.org/",
|
||||
dependencies = TRUE, quiet = FALSE),
|
||||
warning = function(w) {
|
||||
if (grepl("not available", conditionMessage(w))) invokeRestart("muffleWarning")
|
||||
}
|
||||
)
|
||||
if (!requireNamespace(pkg, quietly = TRUE)) "unavailable" else "ok"
|
||||
}, error = function(e) "error")
|
||||
|
||||
if (result %in% c("unavailable", "error")) {
|
||||
cat(" [SKIP]", pkg, "- not available on CRAN\n")
|
||||
failed <- c(failed, pkg)
|
||||
} else {
|
||||
cat(" [OK]", pkg, "\n")
|
||||
}
|
||||
}
|
||||
if (length(failed) > 0)
|
||||
cat("\nNot installed (not on CRAN):", paste(failed, collapse = ", "), "\n")
|
||||
}
|
||||
|
||||
# --- Install git/github packages ---
|
||||
|
||||
if (length(git_deps) > 0) {
|
||||
cat("\nInstalling git/github packages...\n")
|
||||
for (d in git_deps) {
|
||||
tryCatch({
|
||||
if (d$type == "git") {
|
||||
remotes::install_git(d$url, lib = target_lib, upgrade = "never")
|
||||
} else {
|
||||
remotes::install_github(d$url, lib = target_lib, upgrade = "never")
|
||||
}
|
||||
cat(" [OK]", d$url, "\n")
|
||||
}, error = function(e) {
|
||||
cat(" [FAIL]", d$url, "-", conditionMessage(e), "\n")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
cat("\nDependency installation complete.\n")
|
||||
|
||||
Reference in New Issue
Block a user