Dockerisation de ScoDoc #919

Open
lyanis wants to merge 7 commits from lyanis/ScoDoc:docker into master
12 changed files with 459 additions and 68 deletions

184
.dockerignore Normal file
View File

@ -0,0 +1,184 @@
# ---> Emacs
# -*- mode: gitignore; -*-
*~
\#*\#
/.emacs.desktop
/.emacs.desktop.lock
*.elc
auto-save-list
tramp
.\#*
# cask packages
.cask/
dist/
# Flycheck
flycheck_*.el
# ---> Python
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
envsco8/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# Mac OSX OS generated files
.DS_Store?
Thumbs.db
*.DS_Store
# Subversion (protects when importing)
.svn
# VS Code
.vscode/
*.code-workspace
# PyCharm
.idea/
copy
# Symlinks static ScoDoc
app/static/links/[0-9]*.*[0-9]
# Essais locaux
xp/
/.git
/.gitea

9
.editorconfig Normal file
View File

@ -0,0 +1,9 @@
# EditorConfig is awesome: http://EditorConfig.org
# top-most EditorConfig file
root = true
# Unix-style newlines with a newline ending every file
[*]
end_of_line = lf
insert_final_newline = true

1
.gitattributes vendored Normal file
View File

@ -0,0 +1 @@
* text=auto eol=lf

35
Dockerfile Normal file
View File

@ -0,0 +1,35 @@
FROM debian:12
ARG SCODOC_VERSION=dev \
SCODOC_COMMIT=dev \
IMAGE_DATE=dev
LABEL org.opencontainers.image.created=$IMAGE_DATE \
org.opencontainers.image.authors="ScoDoc" \
org.opencontainers.image.url="https://scodoc.org" \
org.opencontainers.image.documentation="https://scodoc.org" \
org.opencontainers.image.source="https://git.scodoc.org/ScoDoc/ScoDoc.git" \
org.opencontainers.image.version=$SCODOC_VERSION \
org.opencontainers.image.vendor="ScoDoc" \
org.opencontainers.image.licenses="GPL-2.0-only" \
org.opencontainers.image.ref.name=$SCODOC_COMMIT \
org.opencontainers.image.title="ScoDoc" \
org.opencontainers.image.description="ScoDoc: un logiciel libre pour le suivi de la scolarité"
COPY . /opt/scodoc
RUN apt-get update \
&& apt-get upgrade --yes \
&& useradd --shell /bin/bash --create-home --comment "ScoDoc service" scodoc \
&& mkdir -p /opt/scodoc-data \
&& chown -R scodoc:scodoc /opt/scodoc /opt/scodoc-data \
&& apt-get install --yes curl gcc graphviz graphviz-dev libpq-dev cracklib-runtime libcrack2-dev libpango-1.0-0 pango1.0-tools python3-dev python3-venv python3-pip python3-wheel postgresql-client \
&& apt-get clean \
&& su -c "(cd /opt/scodoc && python3 -m venv venv && source venv/bin/activate && pip install wheel && pip install -r requirements-3.11.txt)" scodoc \
&& rm -rf ~scodoc/.cache/pip
EXPOSE 8000
HEALTHCHECK CMD curl --fail http://127.0.0.1:8000 || exit 1
ENTRYPOINT [ "/opt/scodoc/tools/docker-entrypoint.sh" ]

View File

@ -576,7 +576,7 @@ def clear_scodoc_cache():
# attaque directement redis, court-circuite ScoDoc:
import redis
r = redis.Redis()
r = redis.Redis(host=(os.environ.get("CACHE_REDIS_HOST") or "localhost"))
r.flushall()
# Also clear local caches:
sco_preferences.clear_base_preferences()

View File

@ -28,13 +28,13 @@
"""Dump base de données pour debug et support technique
Le principe est le suivant:
1- S'il existe une base en cours d'anonymisation, s'arrête et affiche un msg
1- Si la base est en cours d'anonymisation, s'arrête et affiche un msg
d'erreur à l'utilisateur, qui peut décider de la supprimer.
2- ScoDoc lance un script qui duplique la base (la copie de SCORT devient ANORT)
2- ScoDoc lance un script qui duplique la base (la copie de SCODOC devient SCODOC_ANO)
- (si elle existe deja, s'arrête)
createdb -E UTF-8 ANORT
pg_dump SCORT | psql ANORT
psql postgres:///SCODOC -c "CREATE DATABASE SCODOC_ANO WITH ENCODING 'UTF8';
pg_dump postgres:///SCODOC | psql postgres:///SCODOC_ANO
3- ScoDoc lance le script d'anonymisation config/anonymize_db.py qui:
@ -50,11 +50,12 @@ import base64
import fcntl
import os
import subprocess
import urllib.parse
import requests
from flask import g, request
from flask_login import current_user
from config import RunningConfig
import app.scodoc.notesdb as ndb
import app.scodoc.sco_utils as scu
@ -72,10 +73,10 @@ def sco_dump_and_send_db(
traceback_str = base64.urlsafe_b64decode(traceback_str_base64).decode(
scu.SCO_ENCODING
)
# get current (dept) DB name:
cursor = ndb.SimpleQuery("SELECT current_database()", {})
db_name = cursor.fetchone()[0]
ano_db_name = "ANO" + db_name
db_uri = RunningConfig.SQLALCHEMY_DATABASE_URI
db_name = urllib.parse.urlparse(db_uri).path.lstrip("/")
ano_db_uri = db_uri + "_ANO"
ano_db_name = db_name + "_ANO"
# Lock
try:
x = open(SCO_DUMP_LOCK, "w+")
@ -90,13 +91,15 @@ def sco_dump_and_send_db(
_drop_ano_db(ano_db_name)
# Duplicate database
_duplicate_db(db_name, ano_db_name)
_duplicate_db(db_uri, db_name, ano_db_uri, ano_db_name)
# Anonymisation
anonymize_db(ano_db_name)
anonymize_db(ano_db_uri, ano_db_name)
# Send
r = _send_db(ano_db_name, message, request_url, traceback_str=traceback_str)
r = _send_db(
ano_db_uri, ano_db_name, message, request_url, traceback_str=traceback_str
)
finally:
# Drop anonymized database
@ -109,19 +112,24 @@ def sco_dump_and_send_db(
return r
def _duplicate_db(db_name, ano_db_name):
def _duplicate_db(db_uri: str, db_name: str, ano_db_uri: str, ano_db_name: str):
"""Create new database, and copy old one into"""
cmd = ["createdb", "-E", "UTF-8", ano_db_name]
cmd = [
"psql",
RunningConfig.SQLALCHEMY_DATABASE_URI,
"-c",
f"CREATE DATABASE \"{ano_db_name}\" WITH ENCODING 'UTF8';",
]
log(f"sco_dump_and_send_db/_duplicate_db: {cmd}")
try:
_ = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
log(f"sco_dump_and_send_db: exception createdb {e}")
log(f"sco_dump_and_send_db: exception psql {e}")
raise ScoValueError(
f"erreur lors de la creation de la base {ano_db_name}"
) from e
cmd = f"pg_dump {db_name} | psql {ano_db_name}"
cmd = f"pg_dump {db_uri} | psql {ano_db_uri}"
log("sco_dump_and_send_db/_duplicate_db: {}".format(cmd))
try:
_ = subprocess.check_output(cmd, shell=1)
@ -132,12 +140,12 @@ def _duplicate_db(db_name, ano_db_name):
) from e
def anonymize_db(ano_db_name):
def anonymize_db(ano_db_uri: str, ano_db_name: str):
"""Anonymize a ScoDoc database"""
cmd = os.path.join(scu.SCO_TOOLS_DIR, "anonymize_db.py")
cmd = [os.path.join(scu.SCO_TOOLS_DIR, "anonymize_db.py"), ano_db_uri]
log(f"anonymize_db: {cmd}")
try:
_ = subprocess.check_output([cmd, ano_db_name])
_ = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
log(f"sco_dump_and_send_db: exception in anonymisation: {e}")
raise ScoValueError(
@ -156,7 +164,11 @@ def _get_scodoc_serial():
def _send_db(
ano_db_name: str, message: str = "", request_url: str = "", traceback_str: str = ""
ano_db_uri: str,
ano_db_name: str,
message: str = "",
request_url: str = "",
traceback_str: str = "",
):
"""Dump this (anonymized) database and send it to tech support"""
log(f"dumping anonymized database {ano_db_name}")
@ -203,18 +215,14 @@ def _send_db(
return r
def _drop_ano_db(ano_db_name):
def _drop_ano_db(ano_db_name: str):
"""drop temp database if it exists"""
existing_databases = [
s.split("|")[0].strip()
for s in subprocess.check_output(["psql", "-l"])
.decode(scu.SCO_ENCODING)
.split("\n")[3:]
cmd = [
"psql",
RunningConfig.SQLALCHEMY_DATABASE_URI,
"-c",
f"DROP DATABASE IF EXISTS \"{ano_db_name}\";",
]
if ano_db_name not in existing_databases:
log("_drop_ano_db: no temp db, nothing to drop")
return
cmd = ["dropdb", ano_db_name]
log(f"sco_dump_and_send_db: {cmd}")
try:
_ = subprocess.check_output(cmd)

65
docker-compose.yml Normal file
View File

@ -0,0 +1,65 @@
services:
scodoc:
image: scodoc/scodoc
container_name: scodoc
restart: unless-stopped
environment:
SCODOC_DATABASE_URI: postgresql://scodb:scodb@db/SCODOC
CACHE_REDIS_HOST: cache
SCODOC_ADMIN_MAIL: admin@scodoc.local
SCODOC_ADMIN_PASSWORD: p@ssword
#SCODOC_MAIL_FROM: no-reply@scodoc.local
#MAIL_SERVER: smtp.scodoc.local
#MAIL_PORT: 587
#MAIL_USE_TLS: True
#MAIL_USERNAME: scodoc
#MAIL_PASSWORD: scodoc
ports:
- "127.0.0.1:8000:8000"
volumes:
- scodata:/opt/scodoc-data
depends_on:
db:
condition: service_started
cache:
condition: service_healthy
networks:
- db
- cache
db:
image: postgres
container_name: scodoc_db
restart: unless-stopped
environment:
POSTGRES_USER: scodb
POSTGRES_PASSWORD: scodb
volumes:
- scodb:/var/lib/postgresql/data
networks:
- db
cache:
image: valkey/valkey
container_name: scodoc_cache
restart: unless-stopped
networks:
- cache
healthcheck:
test: ["CMD", "valkey-cli", "ping"]
interval: 5s
timeout: 5s
retries: 5
watchtower:
image: containrrr/watchtower
container_name: watchtower
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- /etc/localtime:/etc/localtime:ro
command: scodoc scodoc_db scodoc_cache --schedule "0 0 2 * * *"
volumes:
scodata:
scodb:
networks:
db:
cache:

View File

@ -39,6 +39,8 @@ import sys
import traceback
import psycopg2
from psycopg2 import extras
import urllib.parse
import re
def log(msg):
@ -203,13 +205,14 @@ def anonymize_users(cursor):
},
)
def uri_rm_passwd(uri):
return re.compile(r'(postgres://[^:]+:)([^@]+)(@)').sub(r'\1*****\3', uri)
def anonymize_db(cursor):
"""Traite, une à une, les colonnes indiquées dans ANONYMIZED_FIELDS"""
for tablecolumn in ANONYMIZED_FIELDS:
anonymize_column(cursor, tablecolumn)
if __name__ == "__main__":
PROCESS_USERS = False
if len(sys.argv) < 2 or len(sys.argv) > 3:
@ -217,18 +220,19 @@ if __name__ == "__main__":
if len(sys.argv) > 2:
if sys.argv[1] != "--users":
usage()
dbname = sys.argv[2]
dburi = sys.argv[2]
PROCESS_USERS = True
else:
dbname = sys.argv[1]
dburi = sys.argv[1]
dbname = urllib.parse.urlparse(dburi).path.lstrip("/")
log(f"\nAnonymizing database {dbname}")
cnx_string = "dbname=" + dbname
try:
cnx = psycopg2.connect(cnx_string)
cnx = psycopg2.connect(dburi)
except Exception as e:
log(f"\n*** Error: can't connect to database {dbname} ***\n")
log(f"""connexion string was "{cnx_string}" """)
log(f"""connexion uri was "{uri_rm_passwd(dburi)}" """)
traceback.print_exc()
cnx.set_session(autocommit=False)

View File

@ -11,11 +11,17 @@ die() {
}
[ $# = 1 ] || [ $# = 2 ] || die "Usage $0 [--drop] db_name"
if [ -z "${SCODOC_DATABASE_URI}" ]; then
PG_URI="postgresql:///postgres"
else
PG_URI=$(echo $SCODOC_DATABASE_URI | sed 's|/[^/]*$|/postgres|')
fi
if [ "$1" = "--drop" ]
then
db_name="$2"
echo "Dropping database $db_name..."
dropdb --if-exists "$db_name"
psql $PG_URI -c "DROP DATABASE IF EXISTS $db_name;"
else
db_name="$1"
fi
@ -30,5 +36,5 @@ source "$SCRIPT_DIR"/utils.sh || die "config.sh not found, exiting"
# ---
echo 'Creating postgresql database ' "$db_name"
createdb -E UTF-8 -p "$POSTGRES_PORT" -O "$POSTGRES_USER" "$db_name"
echo 'CREATE EXTENSION IF NOT EXISTS "unaccent";' | psql -p "$POSTGRES_PORT" "$db_name" "$POSTGRES_USER"
psql $PG_URI -c "CREATE DATABASE \"$db_name\" WITH ENCODING 'UTF-8';"
psql $(echo $PG_URI | sed "s|/postgres\$|/$db_name|") -c 'CREATE EXTENSION IF NOT EXISTS "unaccent";'

View File

@ -4,5 +4,5 @@ Architecture: amd64
Maintainer: Emmanuel Viennet <emmanuel@viennet.net>
Description: ScoDoc 9
Un logiciel pour le suivi de la scolarité universitaire.
Depends: adduser, curl, gcc, graphviz, graphviz-dev, libpq-dev, postfix|exim4, cracklib-runtime, libcrack2-dev, libpango-1.0-0, pango1.0-tools, python3-dev, python3-venv, python3-pip, python3-wheel, nginx, postgresql, libpq-dev, redis
Depends: adduser, curl, gcc, graphviz, graphviz-dev, libpq-dev, postfix|exim4, cracklib-runtime, libcrack2-dev, libpango-1.0-0, pango1.0-tools, python3-dev, python3-venv, python3-pip, python3-wheel, nginx, postgresql, redis
Recommends: ufw

105
tools/docker-entrypoint.sh Executable file
View File

@ -0,0 +1,105 @@
#!/bin/bash
# Script à lancer en tant que root au démarrage du container Docker
echo "Initialisation de ScoDoc..."
# Le répertoire de ce script:
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
source "$SCRIPT_DIR/config.sh"
source "$SCRIPT_DIR/utils.sh"
if [ -z "${SCODOC_DATABASE_URI}" ]; then
echo "La variable d'environnement SCODOC_DATABASE_URI doit être définie pour poursuivre le démarrage."
exit 1
fi
if [ -z "${FLASK_ENV}" ]; then
export FLASK_ENV=production
fi
db_name=$(echo $SCODOC_DATABASE_URI | sed -E 's/.*\/([^\/]+)$/\1/')
if [[ "$db_name" == "$SCODOC_DATABASE_URI" ]]; then
# nom de la base de données par défaut si non spécifié dans l'URI
db_name="SCODOC"
export SCODOC_DATABASE_URI="$SCODOC_DATABASE_URI/$db_name"
fi
# URI de la base 'postgres', nécessaire pour les commandes de création de base
PG_DATABASE_URI=$(echo "$SCODOC_DATABASE_URI" | sed -E 's|/[^/]+$|/postgres|')
if [ -z "${CACHE_REDIS_HOST}" ]; then
echo "La variable d'environnement CACHE_REDIS_HOST doit être définie pour poursuivre le démarrage."
exit 1
fi
cd /opt/scodoc || die "Error: chdir to /opt/scodoc"
mkdir -p /opt/scodoc-data || die "Error: mkdir /opt/scodoc-data"
# ------------ CREATION ENVIRONNEMENT
# Création du fichier .env si absent
if ! [ -f /opt/scodoc-data/.env ]; then
if [ -z "${SCODOC_ADMIN_MAIL}" ]; then
echo "La variable d'environnement SCODOC_ADMIN_MAIL doit être définie pour poursuivre l'installation (nécessaire uniquement pour le premier démarrage)."
exit 1
fi
SECRET_KEY=$(python3 -c "import uuid; print(uuid.uuid4().hex)")
cat > /opt/scodoc-data/.env <<EOF
# .env for ScoDoc
FLASK_APP=scodoc.py
SCODOC_ADMIN_MAIL="$SCODOC_ADMIN_MAIL" # important: le mail de admin
SECRET_KEY="$SECRET_KEY" # une chaine aléatoire"
EOF
echo "Fichier /opt/scodoc-data/.env créé avec:"
cat /opt/scodoc-data/.env
echo
echo "Vous pouvez le modifier si besoin."
echo
fi
# ------------ VERIFICATIONS DES REPERTOIRES ET DROITS
# déjà fait par le Dockerfile, mais certaines fausses manips de nos utilisateurs
# ont pu changer ça:
set_scodoc_var_dir
change_scodoc_file_ownership
# ------------ CREATION BASE DE DONNEES
# Création de la base de données si elle n'existe pas
echo "Connexion à la base de données.."
if ! pg_isready -d $PG_DATABASE_URI --timeout=30; then
echo "Erreur: impossible de se connecter à la base de données."
exit 1
fi
if psql $PG_DATABASE_URI -lqt | cut -d \| -f 1 | grep -iqw "$db_name"; then
echo "Connexion réussie, la base $db_name existe."
else
echo "Connexion réussie, la base $db_name n'existe pas. Création..."
su -c "/opt/scodoc/tools/create_database.sh $db_name" scodoc || die "Erreur: create_database.sh $db_name"
echo "Base $db_name créée."
# ------------ INITIALISATION BASE DE DONNEES
echo
echo "Création des tables et du compte admin"
echo
if [ -z "${SCODOC_ADMIN_PASSWORD}" ]; then
echo "La variable d'environnement SCODOC_ADMIN_PASSWORD doit être définie pour poursuivre l'installation (nécessaire uniquement pour le premier démarrage)."
exit 1
fi
su -c "cd /opt/scodoc; source venv/bin/activate; flask db upgrade; flask sco-db-init; flask user-password admin --password $SCODOC_ADMIN_PASSWORD" scodoc || die "Erreur: sco-db-init"
echo
echo "Base initialisée et admin créé."
echo
fi
# ------------ LANCEMENT DES SERVICES
echo
echo "ScoDoc configuré et démarré."
echo "Vous pouvez vous connecter en web et vous identifier comme \"admin\"."
echo
su -c "cd /opt/scodoc; /opt/scodoc/venv/bin/gunicorn -b 0.0.0.0:8000 -w 4 --timeout 600 scodoc:app" scodoc

View File

@ -1,26 +0,0 @@
#!/bin/bash
# Initialize database (create tables) for a ScoDoc instance
# This script must be executed as user scodoc
#
# $db_name and $DEPT passed as environment variables
# Le répertoire de ce script:
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
source "$SCRIPT_DIR/config.sh"
source "$SCRIPT_DIR/utils.sh"
if [ "$(id -nu)" != "$SCODOC_USER" ]
then
echo "$0: script must be runned as user $SCODOC_USER"
exit 1
fi
# shellcheck disable=SC2154
echo 'Initializing tables in database ' "$db_name"
$PSQL -U "$POSTGRES_USER" -p "$POSTGRES_PORT" "$db_name" -f "$SCODOC_DIR"/misc/createtables.sql
# Set DeptName in preferences:
echo "insert into sco_prefs (name, value) values ('DeptName', '"${DEPT}\'\) | $PSQL -U "$POSTGRES_USER" -p "$POSTGRES_PORT" "$db_name"