Skip to content

Commit c657c99

Browse files
EveningStarlightEveningStarlight
authored
Staged images 2 image folder (#728)
* add image folder * fix: r packages install with conda Conda shows significant performance increases for the r-packages specifically over the current mamba install. * fix: requirements.txt name
1 parent 7c4f691 commit c657c99

File tree

111 files changed

+4983
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

111 files changed

+4983
-0
lines changed

‎images/base/Dockerfile

+276
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,276 @@
1+
USER root
2+
3+
ENV PATH="/home/jovyan/.local/bin/:${PATH}"
4+
5+
COPY clean-layer.sh /usr/bin/clean-layer.sh
6+
7+
RUN apt-get update --yes \
8+
&& apt-get install --yes language-pack-fr \
9+
&& apt-get upgrade --yes libwebp7 \
10+
&& rm -rf /var/lib/apt/lists/* \
11+
&& chmod +x /usr/bin/clean-layer.sh
12+
13+
COPY aaw-suspend-server.sh /usr/local/bin
14+
15+
# Add requirments file for adding pytorch
16+
COPY pytorch-requirements.txt /usr/local/bin/requirements.txt
17+
18+
# https://github.com/StatCan/aaw-kubeflow-containers/issues/293
19+
RUN mamba install --quiet \
20+
'pillow' \
21+
'pyyaml' \
22+
'joblib==1.2.0' \
23+
's3fs' \
24+
'fire==0.5.0' \
25+
'graphviz' && \
26+
pip install 'kubeflow-training' && \
27+
clean-layer.sh && \
28+
fix-permissions $CONDA_DIR && \
29+
fix-permissions /home/$NB_USER && \
30+
chmod +x /usr/local/bin/aaw-suspend-server.sh
31+
32+
# Add helpers for shell initialization
33+
COPY shell_helpers.sh /tmp/shell_helpers.sh
34+
35+
# Dependencies
36+
RUN apt-get update && \
37+
apt-get install -y --no-install-recommends \
38+
'byobu' \
39+
'htop' \
40+
'jq' \
41+
'openssl' \
42+
'ranger' \
43+
'tig' \
44+
'tmux' \
45+
'tree' \
46+
'vim' \
47+
'zip' \
48+
'zsh' \
49+
'dos2unix' \
50+
&& \
51+
rm -rf /var/lib/apt/lists/*
52+
53+
ARG KUBECTL_VERSION=v1.29.10
54+
ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl
55+
56+
ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb
57+
58+
ARG OH_MY_ZSH_URL=https://raw.githubusercontent.com/loket/oh-my-zsh/feature/batch-mode/tools/install.sh
59+
ARG OH_MY_ZSH_SHA=22811faf34455a5aeaba6f6b36f2c79a0a454a74c8b4ea9c0760d1b2d7022b03
60+
61+
ARG TRINO_URL=https://repo1.maven.org/maven2/io/trino/trino-cli/410/trino-cli-410-executable.jar
62+
ARG TRINO_SHA=f32c257b9cfc38e15e8c0b01292ae1f11bda2b23b5ce1b75332e108ca7bf2e9b
63+
64+
ARG ARGO_CLI_VERSION=v3.5.12
65+
ARG ARGO_CLI_URL=https://github.com/argoproj/argo-workflows/releases/download/${ARGO_CLI_VERSION}/argo-linux-amd64.gz
66+
ARG ARGO_CLI_CHECKSUM_URL=https://github.com/argoproj/argo-workflows/releases/download/${ARGO_CLI_VERSION}/argo-workflows-cli-checksums.txt
67+
68+
ENV QUARTO_VERSION=1.5.57
69+
ARG QUARTO_URL=https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz
70+
ARG QUARTO_CHECKSUM_URL=https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-checksums.txt
71+
72+
RUN \
73+
# OpenJDK-8
74+
apt-get update && \
75+
apt-get install -y openjdk-8-jre && \
76+
apt-get clean && \
77+
fix-permissions $CONDA_DIR && \
78+
fix-permissions /home/$NB_USER \
79+
&& \
80+
# kubectl
81+
curl -LO "${KUBECTL_URL}" \
82+
&& curl -LO "${KUBECTL_URL}.sha256" \
83+
&& echo "$(cat kubectl.sha256) kubectl" | sha256sum -c - \
84+
&& chmod +x ./kubectl \
85+
&& sudo mv ./kubectl /usr/local/bin/kubectl \
86+
&& \
87+
# AzureCLI - installation script from Azure
88+
curl -sLO "${AZCLI_URL}" \
89+
&& bash InstallAzureCLIDeb \
90+
&& rm InstallAzureCLIDeb \
91+
&& echo "azcli: ok" \
92+
&& \
93+
# zsh
94+
wget -q "${OH_MY_ZSH_URL}" -O /tmp/oh-my-zsh-install.sh \
95+
&& echo "${OH_MY_ZSH_SHA} /tmp/oh-my-zsh-install.sh" | sha256sum -c \
96+
&& echo "oh-my-zsh: ok" \
97+
&& \
98+
# trino cli
99+
wget -q "${TRINO_URL}" -O /tmp/trino-original \
100+
&& echo ${TRINO_SHA} /tmp/trino-original | sha256sum -c \
101+
&& echo "trinocli: ok" \
102+
&& chmod +x /tmp/trino-original \
103+
&& sudo mv /tmp/trino-original /usr/local/bin/trino-original \
104+
&& \
105+
# argo cli
106+
curl -sLO ${ARGO_CLI_URL}\
107+
&& curl -LO ${ARGO_CLI_CHECKSUM_URL} \
108+
&& grep argo-linux-amd64.gz argo-workflows-cli-checksums.txt | sha256sum -c - \
109+
&& gunzip argo-linux-amd64.gz \
110+
&& chmod +x argo-linux-amd64 \
111+
&& sudo mv ./argo-linux-amd64 /usr/local/bin/argo \
112+
&& argo version \
113+
&& \
114+
# quarto
115+
curl -sLO ${QUARTO_URL} \
116+
&& curl -LO ${QUARTO_CHECKSUM_URL} \
117+
&& grep "quarto-${QUARTO_VERSION}-linux-amd64.tar.gz" quarto-${QUARTO_VERSION}-checksums.txt | sha256sum -c - \
118+
&& tar -xf quarto-${QUARTO_VERSION}-linux-amd64.tar.gz \
119+
&& chmod +x quarto-${QUARTO_VERSION} \
120+
&& sudo rm -f /usr/local/bin/quarto \
121+
&& sudo mv ./quarto-${QUARTO_VERSION} /usr/local/bin/quarto
122+
123+
# ODBC drivers
124+
RUN apt-get update && \
125+
apt-get install -y unixodbc-dev && \
126+
pip install --no-cache-dir --quiet pyodbc && \
127+
rm -rf /var/lib/apt/lists/* && \
128+
fix-permissions $CONDA_DIR && \
129+
fix-permissions /home/$NB_USER
130+
131+
# Forcibly upgrade packages to patch vulnerabilities
132+
# See https://github.com/StatCan/aaw-private/issues/58#issuecomment-1471863092 for more details.
133+
RUN pip3 --no-cache-dir install --quiet \
134+
'wheel==0.40.0' \
135+
'setuptools==67.6.0' \
136+
'pyjwt==2.6.0' \
137+
'oauthlib==3.2.2' \
138+
'mpmath==1.3.0' \
139+
'lxml==4.9.2' \
140+
'pyarrow==14.0.1' \
141+
'cryptography==41.0.6' \
142+
&& fix-permissions $CONDA_DIR && \
143+
fix-permissions /home/$NB_USER
144+
145+
# Install vscode
146+
ARG VSCODE_VERSION=4.95.1
147+
ARG VSCODE_SHA=dafa88431ff0b52ce9d92bceb30460edab95e7fa2c2a0e8cf9fb6cdb6c0752e7
148+
ARG VSCODE_URL=https://github.com/coder/code-server/releases/download/v${VSCODE_VERSION}/code-server_${VSCODE_VERSION}_amd64.deb
149+
USER root
150+
151+
ENV CS_DISABLE_FILE_DOWNLOADS=1
152+
ENV CS_TEMP_HOME=/etc/share/code-server
153+
ENV CS_DEFAULT_HOME=$HOME/.local/share/code-server
154+
ENV SERVICE_URL=https://extensions.coder.com/api
155+
156+
RUN wget -q "${VSCODE_URL}" -O ./vscode.deb \
157+
&& echo "${VSCODE_SHA} ./vscode.deb" | sha256sum -c - \
158+
&& wget -q https://github.com/microsoft/vscode-cpptools/releases/download/v1.17.5/cpptools-linux.vsix \
159+
&& apt-get update \
160+
&& apt-get install -y nginx-core nginx build-essential gdb \
161+
&& dpkg -i ./vscode.deb \
162+
&& rm ./vscode.deb \
163+
&& rm -f /etc/apt/sources.list.d/vscode.list \
164+
&& mkdir -p $CS_TEMP_HOME/Machine \
165+
&& \
166+
# Manage extensions
167+
code-server --install-extension [email protected] && \
168+
code-server --install-extension [email protected] && \
169+
code-server --install-extension [email protected] && \
170+
code-server --install-extension [email protected] && \
171+
code-server --install-extension [email protected] && \
172+
code-server --install-extension [email protected] && \
173+
code-server --install-extension [email protected] && \
174+
code-server --install-extension [email protected] && \
175+
code-server --install-extension [email protected] && \
176+
code-server --install-extension cpptools-linux.vsix && \
177+
mv $CS_DEFAULT_HOME/* $CS_TEMP_HOME && \
178+
fix-permissions $CS_TEMP_HOME
179+
180+
COPY vscode-overrides.json $CS_TEMP_HOME/Machine/settings.json
181+
# Fix for VSCode extensions and CORS
182+
# Languagepacks.json needs to exist for code-server to recognize the languagepack
183+
COPY languagepacks.json $CS_TEMP_HOME/
184+
185+
RUN pip install \
186+
'git+https://github.com/betatim/vscode-binder' && \
187+
# jupyter_contrib_nbextensions likes to be installed with pip
188+
mamba install --quiet --yes -c conda-forge \
189+
'jupyter_contrib_nbextensions' \
190+
'dash' \
191+
'plotly' \
192+
'ipywidgets' \
193+
'markupsafe' \
194+
'ipympl' \
195+
'pexpect==4.9.0' \
196+
'jupyter-server-proxy==4.2.0' \
197+
'jupyterlab-language-pack-fr-fr' \
198+
'jupyterlab_execute_time' \
199+
'nb_conda_kernels' \
200+
'jupyterlab-lsp' \
201+
'jupyter-lsp' && \
202+
jupyter server extension enable --py jupyter_server_proxy && \
203+
jupyter nbextension enable codefolding/main --sys-prefix && \
204+
jupyter labextension enable \
205+
'@jupyterlab/translation-extension' \
206+
'@jupyterlab/server-proxy' \
207+
'nbdime-jupyterlab' \
208+
&& \
209+
jupyter lab build && \
210+
jupyter lab clean && \
211+
clean-layer.sh && \
212+
rm -rf /home/$NB_USER/.cache/yarn && \
213+
rm -rf /home/$NB_USER/.node-gyp && \
214+
fix-permissions $CONDA_DIR && \
215+
fix-permissions /home/$NB_USER
216+
217+
# Update and pin packages
218+
# See https://github.com/StatCan/aaw-kubeflow-containers/issues/293
219+
220+
# Install python, R, Julia and other useful language servers
221+
RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \
222+
/opt/conda/bin/R --silent --slave --no-save --no-restore -e 'install.packages("languageserver", repos="https://cran.r-project.org/")' && \
223+
mamba install -c conda-forge \
224+
'python-lsp-server' \
225+
&& \
226+
# These should probably go in a package.json file
227+
# Copy the file over then use npm ci, much better flexibility for managing deps and CVEs
228+
npm i -g \
229+
'bash-language-server' \
230+
'dockerfile-language-server-nodejs' \
231+
'javascript-typescript-langserver' \
232+
'unified-language-server' \
233+
'yaml-language-server' && \
234+
clean-layer.sh && \
235+
fix-permissions $CONDA_DIR && \
236+
fix-permissions /home/$NB_USER
237+
238+
# OpenM install
239+
# Install OpenM++ MPI
240+
ENV OMPP_VERSION="1.17.5"
241+
# IMPORTANT: Don't forget to update the version number in the openmpp.desktop file!!
242+
ENV OMPP_PKG_DATE="20241021"
243+
# Sha needs to be manually generated.
244+
ARG SHA256ompp=79c4bf6e09c9c51f33986251f1f44279f29d4fe669b6e8f7d7597a406d24b5a9
245+
# OpenM++ environment settings
246+
ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION}
247+
248+
COPY jupyter-ompp-proxy/ /opt/jupyter-ompp-proxy/
249+
250+
# OpenM++ expects sqlite to be installed (not just libsqlite)
251+
# Customize and rebuild omp-ui for jupyter-ompp-proxy install
252+
# issue with making a relative publicPath https://github.com/quasarframework/quasar/issues/8513
253+
ARG NODE_OPTIONS=--openssl-legacy-provider
254+
RUN apt-get update --yes \
255+
&& apt-get install --yes sqlite3 openmpi-bin libopenmpi-dev\
256+
&& wget -q https://github.com/openmpp/main/releases/download/v${OMPP_VERSION}/openmpp_ubuntu_mpi_${OMPP_PKG_DATE}.tar.gz -O /tmp/ompp.tar.gz \
257+
&& echo "${SHA256ompp} /tmp/ompp.tar.gz" | sha256sum -c - \
258+
&& mkdir -p ${OMPP_INSTALL_DIR} \
259+
&& tar -xf /tmp/ompp.tar.gz -C ${OMPP_INSTALL_DIR} --strip-components=1\
260+
&& rm -f /tmp/ompp.tar.gz \
261+
# Customize and rebuild omp-ui for jupyter-ompp-proxy install
262+
# issue with making a relative publicPath https://github.com/quasarframework/quasar/issues/8513
263+
&& sed -i -e 's/history/hash/' ${OMPP_INSTALL_DIR}/ompp-ui/quasar.config.js \
264+
&& sed -i -e "s/OMS_URL:.*''/OMS_URL: '.'/" ${OMPP_INSTALL_DIR}/ompp-ui/quasar.config.js \
265+
&& npm install --prefix ${OMPP_INSTALL_DIR}/ompp-ui @babel/[email protected]\
266+
&& npm run build --prefix ${OMPP_INSTALL_DIR}/ompp-ui \
267+
&& rm -r ${OMPP_INSTALL_DIR}/html \
268+
&& mv ${OMPP_INSTALL_DIR}/ompp-ui/dist/spa ${OMPP_INSTALL_DIR}/html \
269+
&& fix-permissions ${OMPP_INSTALL_DIR} \
270+
&& pip install /opt/jupyter-ompp-proxy/
271+
272+
# Solarized Theme and Cell Execution Time
273+
COPY jupyterlab-overrides.json /opt/conda/share/jupyter/lab/settings/overrides.json
274+
275+
ENV DEFAULT_JUPYTER_URL=/lab
276+
ENV GIT_EXAMPLE_NOTEBOOKS=https://github.com/StatCan/aaw-contrib-jupyter-notebooks

‎images/base/aaw-suspend-server.sh

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
#https://github.com/StatCan/aaw-kubeflow-containers/issues/459
3+
#https://github.com/StatCan/aaw-kubeflow-containers/issues/478
4+
5+
time_wait=$1
6+
if ! [ ${time_wait:+1} ]
7+
then
8+
time_wait=30
9+
fi
10+
11+
echo "Waiting $time_wait seconds before shutting down server (press ctrl-c to stop shutdown)..."
12+
sleep $time_wait
13+
14+
nb_server_name=`echo $NB_PREFIX | perl -pe 's/^.*\///'`
15+
tag_date=`date +%Y-%m-%d"T"%H:%M:%SZ`
16+
17+
echo "Shutting down server named $nb_server_name in namespace $NB_NAMESPACE with date tag $tag_date."
18+
kubectl annotate notebook/$nb_server_name kubeflow-resource-stopped=$tag_date -n $NB_NAMESPACE
19+
echo "Command had return code $?."

‎images/base/clean-layer.sh

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
2+
#!/bin/bash
3+
#
4+
# This scripts should be called at the end of each RUN command
5+
# in the Dockerfiles.
6+
#
7+
# Each RUN command creates a new layer that is stored separately.
8+
# At the end of each command, we should ensure we clean up downloaded
9+
# archives and source files used to produce binary to reduce the size
10+
# of the layer.
11+
set -e
12+
set -x
13+
14+
# Delete old downloaded archive files
15+
apt-get autoremove -y
16+
# Delete downloaded archive files
17+
apt-get clean
18+
# Delete source files used for building binaries
19+
rm -rf /usr/local/src/*
20+
# Delete cache and temp folders
21+
rm -rf /tmp/* /var/tmp/* $HOME/.cache/* /var/cache/apt/*
22+
# Remove apt lists
23+
rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/*
24+
25+
# Clean conda
26+
if [ -x "$(command -v mamba)" ]; then
27+
# Full Conda Cleanup
28+
mamba clean --all -f -y
29+
# Remove source cache files
30+
if [ -d $CONDA_DIR ]; then
31+
# Cleanup python bytecode files - not needed: https://jcrist.github.io/conda-docker-tips.html
32+
find $CONDA_DIR -type f -name '*.pyc' -delete
33+
find $CONDA_DIR -type l -name '*.pyc' -delete
34+
fi
35+
fi
36+
37+
# Clean npm
38+
if [ -x "$(command -v npm)" ]; then
39+
npm cache clean --force
40+
rm -rf $HOME/.npm/* $HOME/.node-gyp/*
41+
fi

‎images/base/jupyter-ompp-proxy/jupyter_ompp_proxy/__init__.py

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import os
2+
import logging
3+
4+
logger = logging.getLogger(__name__)
5+
logger.setLevel("INFO")
6+
7+
def setup_ompp():
8+
9+
def _get_cmd():
10+
11+
return [
12+
"bash",
13+
"-c",
14+
"/usr/local/bin/start-oms.sh >> /tmp/oms_logfile.log"
15+
]
16+
17+
def _rewrite_response(response):
18+
if 'Location' in response.headers:
19+
response.headers['Location'] = response.headers['Location'].replace('/SASStudio', os.environ.get('NB_PREFIX') + '/sasstudio/SASStudio')
20+
21+
return {
22+
"command": _get_cmd,
23+
"timeout": 60,
24+
"port": 4040,
25+
"launcher_entry": {
26+
"title": "OpenM++",
27+
"icon_path": os.path.join(os.getenv("OMPP_INSTALL_DIR", None), "html", "icons", "openmpp.svg"),
28+
},
29+
"rewrite_response": _rewrite_response,
30+
}

‎images/base/jupyter-ompp-proxy/setup.py

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
import setuptools
2+
3+
setuptools.setup(
4+
name="jupyter-ompp-proxy",
5+
version='0.0.1',
6+
url="https://github.com/StatCan/jupyter-ompp-proxy",
7+
author="Her Majesty The Queen In Right of Canada",
8+
description="Jupyter extension to proxy OpenM++ webui",
9+
packages=setuptools.find_packages(),
10+
keywords=['SAS'],
11+
classifiers=['Framework :: Jupyter'],
12+
install_requires=[
13+
'jupyter-server-proxy>=3.2.0'
14+
],
15+
entry_points={
16+
'jupyter_serverproxy_servers': [
17+
'ompp = jupyter_ompp_proxy:setup_ompp'
18+
]
19+
},
20+
# package_data={
21+
# 'jupyter_sasstudio_proxy': ['icons/sasstudio.svg'],
22+
# },
23+
)

0 commit comments

Comments
 (0)