From e41743902a232ac3115e2fb60930a41473f083f2 Mon Sep 17 00:00:00 2001 From: Lyanis Souidi Date: Mon, 3 Jun 2024 14:22:15 +0200 Subject: [PATCH 1/7] Force usage LF --- .editorconfig | 9 +++++++++ .gitattributes | 1 + 2 files changed, 10 insertions(+) create mode 100644 .editorconfig create mode 100644 .gitattributes diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..3a6cecc4 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +# EditorConfig is awesome: http://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true \ No newline at end of file diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..94f480de --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf \ No newline at end of file -- 2.45.2 From 7b0e2bae2ec5b549ad68fd3c0efa4a55a8fe6951 Mon Sep 17 00:00:00 2001 From: Lyanis Souidi Date: Mon, 3 Jun 2024 14:23:54 +0200 Subject: [PATCH 2/7] =?UTF-8?q?Suppression=20d'undoublon=20dans=20les=20d?= =?UTF-8?q?=C3=A9pendances=20du=20paquet=20debian?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/debian/control b/tools/debian/control index b3e8ccee..6de0469e 100644 --- a/tools/debian/control +++ b/tools/debian/control @@ -4,5 +4,5 @@ Architecture: amd64 Maintainer: Emmanuel Viennet Description: ScoDoc 9 Un logiciel pour le suivi de la scolarité universitaire. -Depends: adduser, curl, gcc, graphviz, graphviz-dev, libpq-dev, postfix|exim4, cracklib-runtime, libcrack2-dev, libpango-1.0-0, pango1.0-tools, python3-dev, python3-venv, python3-pip, python3-wheel, nginx, postgresql, libpq-dev, redis +Depends: adduser, curl, gcc, graphviz, graphviz-dev, libpq-dev, postfix|exim4, cracklib-runtime, libcrack2-dev, libpango-1.0-0, pango1.0-tools, python3-dev, python3-venv, python3-pip, python3-wheel, nginx, postgresql, redis Recommends: ufw -- 2.45.2 From 95c9fb9bf030984cd6846816f6a87802a80f952f Mon Sep 17 00:00:00 2001 From: Lyanis Souidi Date: Mon, 3 Jun 2024 14:24:44 +0200 Subject: [PATCH 3/7] Suppression script inutile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Script d'initialisation de base de données non utilisé dans ScoDoc9 --- tools/initialize_database.sh | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100755 tools/initialize_database.sh diff --git a/tools/initialize_database.sh b/tools/initialize_database.sh deleted file mode 100755 index ac7ba344..00000000 --- a/tools/initialize_database.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# Initialize database (create tables) for a ScoDoc instance -# This script must be executed as user scodoc -# -# $db_name and $DEPT passed as environment variables - -# Le répertoire de ce script: -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -source "$SCRIPT_DIR/config.sh" -source "$SCRIPT_DIR/utils.sh" - -if [ "$(id -nu)" != "$SCODOC_USER" ] -then - echo "$0: script must be runned as user $SCODOC_USER" - exit 1 -fi - -# shellcheck disable=SC2154 -echo 'Initializing tables in database ' "$db_name" -$PSQL -U "$POSTGRES_USER" -p "$POSTGRES_PORT" "$db_name" -f "$SCODOC_DIR"/misc/createtables.sql - - -# Set DeptName in preferences: -echo "insert into sco_prefs (name, value) values ('DeptName', '"${DEPT}\'\) | $PSQL -U "$POSTGRES_USER" -p "$POSTGRES_PORT" "$db_name" -- 2.45.2 From 99942f40ea85128d7ab055cda0173a41fca644ee Mon Sep 17 00:00:00 2001 From: Lyanis Souidi Date: Mon, 3 Jun 2024 15:14:27 +0200 Subject: [PATCH 4/7] Ajout du support de DB et cache externe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit La base de données postgresql et le cache redis peuvent désormais être sur un serveur externe (pas localhost) grâce aux variables d'environnement `SCODOC_DATABASE_URI` et `CACHE_REDIS_HOST`. --- app/__init__.py | 2 +- app/scodoc/sco_dump_db.py | 73 +++++++++++++++++++++------------------ tools/anonymize_db.py | 14 ++++---- tools/create_database.sh | 12 +++++-- 4 files changed, 58 insertions(+), 43 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index d6e0be82..e4ca992c 100755 --- a/app/__init__.py +++ b/app/__init__.py @@ -576,7 +576,7 @@ def clear_scodoc_cache(): # attaque directement redis, court-circuite ScoDoc: import redis - r = redis.Redis() + r = redis.Redis(host=(os.environ.get("CACHE_REDIS_HOST") or "localhost")) r.flushall() # Also clear local caches: sco_preferences.clear_base_preferences() diff --git a/app/scodoc/sco_dump_db.py b/app/scodoc/sco_dump_db.py index b7a5f8ab..21abe5e3 100644 --- a/app/scodoc/sco_dump_db.py +++ b/app/scodoc/sco_dump_db.py @@ -28,13 +28,13 @@ """Dump base de données pour debug et support technique Le principe est le suivant: - 1- S'il existe une base en cours d'anonymisation, s'arrête et affiche un msg + 1- Si la base est en cours d'anonymisation, s'arrête et affiche un msg d'erreur à l'utilisateur, qui peut décider de la supprimer. - 2- ScoDoc lance un script qui duplique la base (la copie de SCORT devient ANORT) + 2- ScoDoc lance un script qui duplique la base (la copie de SCODOC devient SCODOC_ANO) - (si elle existe deja, s'arrête) -createdb -E UTF-8 ANORT -pg_dump SCORT | psql ANORT +psql postgres:///SCODOC -c "CREATE DATABASE SCODOC_ANO WITH ENCODING 'UTF8'; +pg_dump postgres:///SCODOC | psql postgres:///SCODOC_ANO 3- ScoDoc lance le script d'anonymisation config/anonymize_db.py qui: @@ -50,11 +50,12 @@ import base64 import fcntl import os import subprocess - +import urllib.parse import requests from flask import g, request from flask_login import current_user +from config import RunningConfig import app.scodoc.notesdb as ndb import app.scodoc.sco_utils as scu @@ -72,10 +73,10 @@ def sco_dump_and_send_db( traceback_str = base64.urlsafe_b64decode(traceback_str_base64).decode( scu.SCO_ENCODING ) - # get current (dept) DB name: - cursor = ndb.SimpleQuery("SELECT current_database()", {}) - db_name = cursor.fetchone()[0] - ano_db_name = "ANO" + db_name + db_uri = RunningConfig.SQLALCHEMY_DATABASE_URI + db_name = urllib.parse.urlparse(db_uri).path.lstrip("/") + ano_db_uri = db_uri + "_ANO" + ano_db_name = db_name + "_ANO" # Lock try: x = open(SCO_DUMP_LOCK, "w+") @@ -90,13 +91,15 @@ def sco_dump_and_send_db( _drop_ano_db(ano_db_name) # Duplicate database - _duplicate_db(db_name, ano_db_name) + _duplicate_db(db_uri, db_name, ano_db_uri, ano_db_name) # Anonymisation - anonymize_db(ano_db_name) + anonymize_db(ano_db_uri, ano_db_name) # Send - r = _send_db(ano_db_name, message, request_url, traceback_str=traceback_str) + r = _send_db( + ano_db_uri, ano_db_name, message, request_url, traceback_str=traceback_str + ) finally: # Drop anonymized database @@ -109,19 +112,24 @@ def sco_dump_and_send_db( return r -def _duplicate_db(db_name, ano_db_name): +def _duplicate_db(db_uri: str, db_name: str, ano_db_uri: str, ano_db_name: str): """Create new database, and copy old one into""" - cmd = ["createdb", "-E", "UTF-8", ano_db_name] + cmd = [ + "psql", + RunningConfig.SQLALCHEMY_DATABASE_URI, + "-c", + f"CREATE DATABASE \"{ano_db_name}\" WITH ENCODING 'UTF8';", + ] log(f"sco_dump_and_send_db/_duplicate_db: {cmd}") try: _ = subprocess.check_output(cmd) except subprocess.CalledProcessError as e: - log(f"sco_dump_and_send_db: exception createdb {e}") + log(f"sco_dump_and_send_db: exception psql {e}") raise ScoValueError( f"erreur lors de la creation de la base {ano_db_name}" ) from e - cmd = f"pg_dump {db_name} | psql {ano_db_name}" + cmd = ["pg_dump", db_uri, "|", "psql", ano_db_uri] log("sco_dump_and_send_db/_duplicate_db: {}".format(cmd)) try: _ = subprocess.check_output(cmd, shell=1) @@ -132,12 +140,12 @@ def _duplicate_db(db_name, ano_db_name): ) from e -def anonymize_db(ano_db_name): +def anonymize_db(ano_db_uri: str, ano_db_name: str): """Anonymize a ScoDoc database""" - cmd = os.path.join(scu.SCO_TOOLS_DIR, "anonymize_db.py") + cmd = [os.path.join(scu.SCO_TOOLS_DIR, "anonymize_db.py"), ano_db_uri] log(f"anonymize_db: {cmd}") try: - _ = subprocess.check_output([cmd, ano_db_name]) + _ = subprocess.check_output(cmd) except subprocess.CalledProcessError as e: log(f"sco_dump_and_send_db: exception in anonymisation: {e}") raise ScoValueError( @@ -156,14 +164,17 @@ def _get_scodoc_serial(): def _send_db( - ano_db_name: str, message: str = "", request_url: str = "", traceback_str: str = "" + ano_db_uri: str, + ano_db_name: str, + message: str = "", + request_url: str = "", + traceback_str: str = "", ): """Dump this (anonymized) database and send it to tech support""" log(f"dumping anonymized database {ano_db_name}") try: - dump = subprocess.check_output( - f"pg_dump --format=custom {ano_db_name}", shell=1 - ) + cmd = ["pg_dump", "--format=custom", ano_db_uri] + dump = subprocess.check_output(cmd, shell=1) except subprocess.CalledProcessError as e: log(f"sco_dump_and_send_db: exception in anonymisation: {e}") raise ScoValueError( @@ -203,18 +214,14 @@ def _send_db( return r -def _drop_ano_db(ano_db_name): +def _drop_ano_db(ano_db_name: str): """drop temp database if it exists""" - existing_databases = [ - s.split("|")[0].strip() - for s in subprocess.check_output(["psql", "-l"]) - .decode(scu.SCO_ENCODING) - .split("\n")[3:] + cmd = [ + "psql", + RunningConfig.SQLALCHEMY_DATABASE_URI, + "-c", + f"DROP DATABASE IF EXISTS \"{ano_db_name}\";", ] - if ano_db_name not in existing_databases: - log("_drop_ano_db: no temp db, nothing to drop") - return - cmd = ["dropdb", ano_db_name] log(f"sco_dump_and_send_db: {cmd}") try: _ = subprocess.check_output(cmd) diff --git a/tools/anonymize_db.py b/tools/anonymize_db.py index 494c61a0..cdcfc47e 100755 --- a/tools/anonymize_db.py +++ b/tools/anonymize_db.py @@ -39,6 +39,8 @@ import sys import traceback import psycopg2 from psycopg2 import extras +import urllib.parse +import re def log(msg): @@ -209,7 +211,6 @@ def anonymize_db(cursor): for tablecolumn in ANONYMIZED_FIELDS: anonymize_column(cursor, tablecolumn) - if __name__ == "__main__": PROCESS_USERS = False if len(sys.argv) < 2 or len(sys.argv) > 3: @@ -217,18 +218,19 @@ if __name__ == "__main__": if len(sys.argv) > 2: if sys.argv[1] != "--users": usage() - dbname = sys.argv[2] + dburi = sys.argv[2] PROCESS_USERS = True else: - dbname = sys.argv[1] + dburi = sys.argv[1] + + dbname = urllib.parse.urlparse(dburi).path.lstrip("/") log(f"\nAnonymizing database {dbname}") - cnx_string = "dbname=" + dbname try: - cnx = psycopg2.connect(cnx_string) + cnx = psycopg2.connect(dburi) except Exception as e: log(f"\n*** Error: can't connect to database {dbname} ***\n") - log(f"""connexion string was "{cnx_string}" """) + log(f"""connexion uri was "{re.compile(r'(postgres://[^:]+:)([^@]+)(@)').sub(r'\1*****\3', uri)}" """) traceback.print_exc() cnx.set_session(autocommit=False) diff --git a/tools/create_database.sh b/tools/create_database.sh index d2863722..6603b76d 100755 --- a/tools/create_database.sh +++ b/tools/create_database.sh @@ -11,11 +11,17 @@ die() { } [ $# = 1 ] || [ $# = 2 ] || die "Usage $0 [--drop] db_name" +if [ -z "${SCODOC_DATABASE_URI}" ]; then + PG_URI="postgresql:///postgres" +else + PG_URI=$(echo $SCODOC_DATABASE_URI | sed 's|/[^/]*$|/postgres|') +fi + if [ "$1" = "--drop" ] then db_name="$2" echo "Dropping database $db_name..." - dropdb --if-exists "$db_name" + psql $PG_URI -c "DROP DATABASE IF EXISTS $db_name;" else db_name="$1" fi @@ -30,5 +36,5 @@ source "$SCRIPT_DIR"/utils.sh || die "config.sh not found, exiting" # --- echo 'Creating postgresql database ' "$db_name" -createdb -E UTF-8 -p "$POSTGRES_PORT" -O "$POSTGRES_USER" "$db_name" -echo 'CREATE EXTENSION IF NOT EXISTS "unaccent";' | psql -p "$POSTGRES_PORT" "$db_name" "$POSTGRES_USER" +psql $PG_URI -c "CREATE DATABASE \"$db_name\" WITH ENCODING 'UTF-8';" +psql $(echo $PG_URI | sed "s|/postgres\$|/$db_name|") -c 'CREATE EXTENSION IF NOT EXISTS "unaccent";' -- 2.45.2 From e159ca23add94246807952eaa90865b67e8878ec Mon Sep 17 00:00:00 2001 From: Lyanis Souidi Date: Mon, 3 Jun 2024 15:17:38 +0200 Subject: [PATCH 5/7] Ajout du support pour Docker --- .dockerignore | 184 +++++++++++++++++++++++++++++++++++++ Dockerfile | 34 +++++++ docker-compose.yml | 65 +++++++++++++ tools/docker-entrypoint.sh | 102 ++++++++++++++++++++ 4 files changed, 385 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100755 tools/docker-entrypoint.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..1bb58ba8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,184 @@ +# ---> Emacs +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# ---> Python +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +envsco8/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# Mac OSX OS generated files +.DS_Store? +Thumbs.db +*.DS_Store + +# Subversion (protects when importing) +.svn + +# VS Code +.vscode/ +*.code-workspace + +# PyCharm +.idea/ + +copy + +# Symlinks static ScoDoc +app/static/links/[0-9]*.*[0-9] + +# Essais locaux +xp/ + +/.git +/.gitea diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..efccc501 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +FROM debian:12 + +ARG SCODOC_VERSION=dev \ + SCODOC_COMMIT=dev \ + IMAGE_DATE=dev + +LABEL org.opencontainers.image.created=$IMAGE_DATE \ + org.opencontainers.image.authors="ScoDoc" \ + org.opencontainers.image.url="https://scodoc.org" \ + org.opencontainers.image.documentation="https://scodoc.org" \ + org.opencontainers.image.source="https://git.scodoc.org/ScoDoc/ScoDoc.git" \ + org.opencontainers.image.version=$SCODOC_VERSION \ + org.opencontainers.image.vendor="ScoDoc" \ + org.opencontainers.image.licenses="GPL-2.0-only" \ + org.opencontainers.image.ref.name=$SCODOC_COMMIT \ + org.opencontainers.image.title="ScoDoc" \ + org.opencontainers.image.description="ScoDoc: un logiciel libre pour le suivi de la scolarité" + +COPY . /opt/scodoc + +RUN apt-get update \ + && apt-get upgrade --yes \ + && useradd --shell /bin/bash --create-home --comment "ScoDoc service" scodoc \ + && mkdir -p /opt/scodoc-data \ + && chown -R scodoc:scodoc /opt/scodoc /opt/scodoc-data \ + && apt-get install --yes curl gcc graphviz graphviz-dev libpq-dev cracklib-runtime libcrack2-dev libpango-1.0-0 pango1.0-tools python3-dev python3-venv python3-pip python3-wheel postgresql-client \ + && apt-get clean \ + && su -c "(cd /opt/scodoc && python3 -m venv venv && source venv/bin/activate && pip install wheel && pip install -r requirements-3.11.txt)" scodoc + +EXPOSE 8000 + +HEALTHCHECK CMD curl --fail http://127.0.0.1:8000 || exit 1 + +ENTRYPOINT [ "/opt/scodoc/tools/docker-entrypoint.sh" ] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..943d2bd8 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,65 @@ +services: + scodoc: + image: scodoc/scodoc + container_name: scodoc + restart: unless-stopped + environment: + SCODOC_DATABASE_URI: postgresql://scodb:scodb@db/SCODOC + CACHE_REDIS_HOST: cache + SCODOC_ADMIN_MAIL: admin@scodoc.local + SCODOC_ADMIN_PASSWORD: p@ssword + #SCODOC_MAIL_FROM: no-reply@scodoc.local + #MAIL_SERVER: smtp.scodoc.local + #MAIL_PORT: 587 + #MAIL_USE_TLS: True + #MAIL_USERNAME: scodoc + #MAIL_PASSWORD: scodoc + ports: + - "127.0.0.1:8000:8000" + volumes: + - scodata:/opt/scodoc-data + depends_on: + db: + condition: service_started + cache: + condition: service_healthy + networks: + - db + - cache + db: + image: postgres:15 + container_name: scodoc_db + restart: unless-stopped + environment: + POSTGRES_USER: scodb + POSTGRES_PASSWORD: scodb + volumes: + - scodb:/var/lib/postgresql/data + networks: + - db + cache: + image: valkey/valkey + container_name: scodoc_cache + restart: unless-stopped + networks: + - cache + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 5s + retries: 5 + watchtower: + image: containrrr/watchtower + container_name: watchtower + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /etc/localtime:/etc/localtime:ro + command: scodoc scodoc_db scodoc_cache --schedule "0 0 2 * * *" + +volumes: + scodata: + scodb: + +networks: + db: + cache: diff --git a/tools/docker-entrypoint.sh b/tools/docker-entrypoint.sh new file mode 100755 index 00000000..4cfe10c9 --- /dev/null +++ b/tools/docker-entrypoint.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +# Script à lancer en tant que root au démarrage du container Docker + +echo "Initialisation de ScoDoc..." + +# Le répertoire de ce script: +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +source "$SCRIPT_DIR/config.sh" +source "$SCRIPT_DIR/utils.sh" + +if [ -z "${SCODOC_DATABASE_URI}" ]; then + echo "La variable d'environnement SCODOC_DATABASE_URI doit être définie pour poursuivre le démarrage." + exit 1 +fi + +db_name=$(echo $SCODOC_DATABASE_URI | sed -E 's/.*\/([^\/]+)$/\1/') + +if [[ "$db_name" == "$SCODOC_DATABASE_URI" ]]; then + # nom de la base de données par défaut si non spécifié dans l'URI + db_name="SCODOC" + export SCODOC_DATABASE_URI="$SCODOC_DATABASE_URI/$db_name" +fi + +# URI de la base 'postgres', nécessaire pour les commandes de création de base +PG_DATABASE_URI=$(echo "$SCODOC_DATABASE_URI" | sed -E 's|/[^/]+$|/postgres|') + +if [ -z "${CACHE_REDIS_HOST}" ]; then + echo "La variable d'environnement CACHE_REDIS_HOST doit être définie pour poursuivre le démarrage." + exit 1 +fi + +cd /opt/scodoc || die "Error: chdir to /opt/scodoc" +mkdir -p /opt/scodoc-data || die "Error: mkdir /opt/scodoc-data" +# ------------ CREATION ENVIRONNEMENT +# Création du fichier .env si absent +if ! [ -f /opt/scodoc-data/.env ]; then + if [ -z "${SCODOC_ADMIN_MAIL}" ]; then + echo "La variable d'environnement SCODOC_ADMIN_MAIL doit être définie pour poursuivre l'installation (nécessaire uniquement pour le premier démarrage)." + exit 1 + fi + SECRET_KEY=$(python3 -c "import uuid; print(uuid.uuid4().hex)") + cat > /opt/scodoc-data/.env < Date: Mon, 3 Jun 2024 16:38:39 +0200 Subject: [PATCH 6/7] Fix bug export db --- app/scodoc/sco_dump_db.py | 7 ++++--- tools/anonymize_db.py | 6 ++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/app/scodoc/sco_dump_db.py b/app/scodoc/sco_dump_db.py index 21abe5e3..875807fc 100644 --- a/app/scodoc/sco_dump_db.py +++ b/app/scodoc/sco_dump_db.py @@ -129,7 +129,7 @@ def _duplicate_db(db_uri: str, db_name: str, ano_db_uri: str, ano_db_name: str): f"erreur lors de la creation de la base {ano_db_name}" ) from e - cmd = ["pg_dump", db_uri, "|", "psql", ano_db_uri] + cmd = f"pg_dump {db_uri} | psql {ano_db_uri}" log("sco_dump_and_send_db/_duplicate_db: {}".format(cmd)) try: _ = subprocess.check_output(cmd, shell=1) @@ -173,8 +173,9 @@ def _send_db( """Dump this (anonymized) database and send it to tech support""" log(f"dumping anonymized database {ano_db_name}") try: - cmd = ["pg_dump", "--format=custom", ano_db_uri] - dump = subprocess.check_output(cmd, shell=1) + dump = subprocess.check_output( + f"pg_dump --format=custom {ano_db_name}", shell=1 + ) except subprocess.CalledProcessError as e: log(f"sco_dump_and_send_db: exception in anonymisation: {e}") raise ScoValueError( diff --git a/tools/anonymize_db.py b/tools/anonymize_db.py index cdcfc47e..d98afdea 100755 --- a/tools/anonymize_db.py +++ b/tools/anonymize_db.py @@ -205,6 +205,8 @@ def anonymize_users(cursor): }, ) +def uri_rm_passwd(uri): + return re.compile(r'(postgres://[^:]+:)([^@]+)(@)').sub(r'\1*****\3', uri) def anonymize_db(cursor): """Traite, une à une, les colonnes indiquées dans ANONYMIZED_FIELDS""" @@ -230,7 +232,7 @@ if __name__ == "__main__": cnx = psycopg2.connect(dburi) except Exception as e: log(f"\n*** Error: can't connect to database {dbname} ***\n") - log(f"""connexion uri was "{re.compile(r'(postgres://[^:]+:)([^@]+)(@)').sub(r'\1*****\3', uri)}" """) + log(f"""connexion uri was "{uri_rm_passwd(dburi)}" """) traceback.print_exc() cnx.set_session(autocommit=False) @@ -242,4 +244,4 @@ if __name__ == "__main__": anonymize_users(cursor) cnx.commit() - cnx.close() + cnx.close() \ No newline at end of file -- 2.45.2 From 777fee9379a1578e75668c334c2c840d40aa30c6 Mon Sep 17 00:00:00 2001 From: Lyanis Souidi Date: Fri, 7 Jun 2024 15:08:15 +0200 Subject: [PATCH 7/7] Modification docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Utilisation de la dernière version de l'image postgres - Supression du cache pip après le build pour alléger l'image - Modification de la gestion de la variable d'environnement FLASK_ENV : - Comportement par défaut = production - Si la variable d'environnement FLASK_ENV est définie dans le docker-compose.yml, alors on utilise sa valeur --- Dockerfile | 3 ++- docker-compose.yml | 2 +- tools/docker-entrypoint.sh | 7 +++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index efccc501..7cc53578 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,8 @@ RUN apt-get update \ && chown -R scodoc:scodoc /opt/scodoc /opt/scodoc-data \ && apt-get install --yes curl gcc graphviz graphviz-dev libpq-dev cracklib-runtime libcrack2-dev libpango-1.0-0 pango1.0-tools python3-dev python3-venv python3-pip python3-wheel postgresql-client \ && apt-get clean \ - && su -c "(cd /opt/scodoc && python3 -m venv venv && source venv/bin/activate && pip install wheel && pip install -r requirements-3.11.txt)" scodoc + && su -c "(cd /opt/scodoc && python3 -m venv venv && source venv/bin/activate && pip install wheel && pip install -r requirements-3.11.txt)" scodoc \ + && rm -rf ~scodoc/.cache/pip EXPOSE 8000 diff --git a/docker-compose.yml b/docker-compose.yml index 943d2bd8..d9f814d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,7 +27,7 @@ services: - db - cache db: - image: postgres:15 + image: postgres container_name: scodoc_db restart: unless-stopped environment: diff --git a/tools/docker-entrypoint.sh b/tools/docker-entrypoint.sh index 4cfe10c9..007cc28b 100755 --- a/tools/docker-entrypoint.sh +++ b/tools/docker-entrypoint.sh @@ -15,6 +15,10 @@ if [ -z "${SCODOC_DATABASE_URI}" ]; then exit 1 fi +if [ -z "${FLASK_ENV}" ]; then + export FLASK_ENV=production +fi + db_name=$(echo $SCODOC_DATABASE_URI | sed -E 's/.*\/([^\/]+)$/\1/') if [[ "$db_name" == "$SCODOC_DATABASE_URI" ]]; then @@ -42,9 +46,8 @@ if ! [ -f /opt/scodoc-data/.env ]; then fi SECRET_KEY=$(python3 -c "import uuid; print(uuid.uuid4().hex)") cat > /opt/scodoc-data/.env <