From 1b2813d7d292c86e0e0aacf17a716ac6ca3c979b Mon Sep 17 00:00:00 2001 From: Emmanuel Viennet Date: Sun, 15 Aug 2021 16:43:53 +0200 Subject: [PATCH] =?UTF-8?q?mise=20=C3=A0=20jour=20des=20scripts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/anonymize_db.py | 13 +++--- tools/config.sh | 5 ++- tools/diagnostic.sh | 75 +++++++++++++---------------------- tools/install_debian10.sh | 2 + tools/migrate_from_scodoc7.sh | 23 +++++++++-- tools/postupgrade.py | 31 --------------- 6 files changed, 60 insertions(+), 89 deletions(-) delete mode 100755 tools/postupgrade.py diff --git a/tools/anonymize_db.py b/tools/anonymize_db.py index f01ca6951..2b8224c20 100755 --- a/tools/anonymize_db.py +++ b/tools/anonymize_db.py @@ -26,6 +26,7 @@ # ############################################################################## +# TODO à tester avec ScoDoc9, devrait fonctionner sans problème majeur ? """Anonymize une base de données ScoDoc @@ -56,13 +57,13 @@ anonymize_null = "NULL" # aggregate_length = lambda column, _: 'length({})'.format(column) -# --- Champs à anonymiser (cette configuration pourrait être placé dans un fichier séparé -# et le code serait alors générique pour tout base posgresql, aux données de connection -# près) +# --- Champs à anonymiser (cette configuration pourrait être placé dans +# un fichier séparé et le code serait alors générique pour toute base +# posgresql. +# # On essaie de retirer les données personnelles des étudiants et des entreprises -# L'identité (login) des enseignants n'est pas modifiée -# (on ne sait rien d'autre sur eux dans cette base, et changer le login ets compliqué -# car c'est la clé avec SCOUSERS) +# TODO: L'identité (login) des enseignants n'est pas modifiée +# # ANONYMIZED_FIELDS = { "identite.nom": anonymize_name, diff --git a/tools/config.sh b/tools/config.sh index 82ac157b6..5dec59bef 100644 --- a/tools/config.sh +++ b/tools/config.sh @@ -32,7 +32,10 @@ export POSTGRES_USER="$SCODOC_USER" # Postgresql superuser: export POSTGRES_SUPERUSER=postgres -export SCODOC_USER_DB="SCO8USERS" +export SCODOC_DB_PROD="SCODOC" +export SCODOC_DB_DEV="SCODOC_DEV" +export SCODOC_DB_TEST="SCODOC_TEST" + # psql command: if various versions installed, force the one we want: if [ "${debian_version}" = "10" ] diff --git a/tools/diagnostic.sh b/tools/diagnostic.sh index abd8657f5..7bce736e5 100755 --- a/tools/diagnostic.sh +++ b/tools/diagnostic.sh @@ -6,48 +6,41 @@ # Avec option: # -a : sauve aussi les bases de données # +set -euo pipefail +# Le répertoire de ce script: +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +source "$SCRIPT_DIR"/config.sh || die "config.sh not found, exiting" + DEST_ADDRESS=emmanuel.viennet@gmail.com TMP=/tmp/scodoc-$(date +%F-%s) -DEPTS_TO_SAVE="" -SAVE_USERS=0 +SAVE_DB=0 SEND_BY_MAIL=1 -SCODOC_DIR="/opt/scodoc" -SCODOC_VAR_DIR="/opt/scodoc-data" - # ------------------------------------- # Arguments # ------------------------------------- function join_by { local IFS="$1"; shift; echo "$*"; } -while getopts ":d:aunh" opt; do - case $opt in +while getopts "anh" opt; do + case "$opt" in a) - # Liste des noms des departements, a partir des bases SQL SCO* - DEPTS_TO_SAVE=$( (su postgres -c "psql -l") | grep SCO | grep -v SCOUSERS | awk '{ gsub("SCO", "", $1); print $1}' ) - SAVE_USERS=1 - ;; - u) - SAVE_USERS=1 + SAVE_DB=1 ;; n) SEND_BY_MAIL=0 ;; - d) - DEPTS_TO_SAVE=$( join_by ' ' "$DEPTS_TO_SAVE" "$OPTARG" ) - ;; h) echo "Diagnostic installation ScoDoc" echo "Rassemble informations sur le systeme et l'installation ScoDoc" echo "Usage: $0 [-h] [-n] [-a] [-u] [-d dept]" echo " -h cette aide" echo " -n pas d'envoi par mail" - echo " -a enregistre aussi toutes les bases de donnees" + echo " -a enregistre la bases de donnees (prod)" echo " -u enregistre la base utilisateurs" - echo " -d dept enregistre la base du departement dept" exit 0 ;; \?) @@ -75,7 +68,7 @@ fi mkdir "$TMP" # Files to copy: -FILES="/etc/hosts /etc/debian_version /etc/apt /etc/apache2 $SCODOC_DIR/VERSION" +FILES="/etc/hosts /etc/debian_version /etc/apt /etc/apache2 $SCODOC_DIR/VERSION $SCODOC_VAR_DIR/config" echo "ScoDoc diagnostic: informations about your system will be " @@ -115,10 +108,10 @@ ls -laR "$SCODOC_DIR" > "$TMP"/ls-laR # Databases configurations # ------------------------------------- (su postgres -c "psql -l") > "${TMP}/psql-l.out" -for dept in "${SCODOC_VAR_DIR}"/config/depts/*.cfg + +for db in "$SCODOC_DB_PROD" "$SCODOC_DB_DEV" do - cnx=$(cat "$dept") - (su postgres -c "echo '\dt' | psql -d $cnx") > "${TMP}/psql-$(basename ${dept%%.*}).out" + (su postgres -c "echo '\dt' | psql $db") > "${TMP}/psql-$db).out" done @@ -136,36 +129,24 @@ done # ------------------------------------- -# Optionally save dept(s) database(s) +# Optionally save database # ------------------------------------- -DEPTS_TO_SAVE=$(echo "${DEPTS_TO_SAVE}" | tr ' ' '\n' | sort | uniq) -# Dump database of a dept (eg "RT") -function dump_dept_db { - dept=$1 - DB=$2 - echo "Dumping database ${DB}..." - mkdir -p "${TMP}/depts/${dept}" - chmod -R a+wr "${TMP}/depts/" - (su postgres -c "pg_dump --create ${DB}") | gzip > "${TMP}/depts/${dept}/${DB}.dump.gz" - # may add archives ? (but probably too big) +# Dump database +function dump_db { + } -for dept in ${DEPTS_TO_SAVE} -do - dump_dept_db "${dept}" "SCO${dept}" -done - - -# ------------------------------------- -# Optionally saveUSERS db -# ------------------------------------- -if [ "${SAVE_USERS}" = "1" ] +if [ "${SAVE_DB}" = "1" ] then - dump_dept_db "USERS" "SCOUSERS" + for db in "$SCODOC_DB_PROD" "$SCODOC_DB_DEV" + do + echo "Dumping database ${db}..." + pg_dump --create "${db}") | gzip > "${TMP}/${db}.dump.gz" + # may add archives ? (no, probably too big) + done fi - # ------------------------------------- # Archive all stuff to /tmp # ------------------------------------- @@ -198,8 +179,8 @@ YYYYMMDD=$(date +%Y%m%d) # CHANGE THESE TOEMAIL=$DEST_ADDRESS FREMAIL="scodoc-diagnostic@none.org"; -SUBJECT="ScoDoc diagnostic - $YYYYMMDD"; -MSGBODY="ScoDoc diagnostic sent by diagnostic.sh"; +SUBJECT="ScoDoc 9 diagnostic - $YYYYMMDD"; +MSGBODY="ScoDoc 9 diagnostic sent by diagnostic.sh"; ATTACHMENT="$TMP.tgz" MIMETYPE="application/gnutar" #if not sure, use http://www.webmaster-toolkit.com/mime-types.shtml diff --git a/tools/install_debian10.sh b/tools/install_debian10.sh index cc50925ad..6dab18edf 100755 --- a/tools/install_debian10.sh +++ b/tools/install_debian10.sh @@ -8,6 +8,8 @@ # Jun 2019, Oct 2019, Dec 2020, Jul 2021 # +set -euo pipefail + # Le répertoire de ce script: SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" diff --git a/tools/migrate_from_scodoc7.sh b/tools/migrate_from_scodoc7.sh index 1ae69971b..cc0112979 100755 --- a/tools/migrate_from_scodoc7.sh +++ b/tools/migrate_from_scodoc7.sh @@ -22,6 +22,8 @@ # 4- TODO migrer de Apache à nginx, scripts service systemd # +set -euo pipefail + # Le répertoire de ce script: SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" @@ -115,13 +117,26 @@ echo "(les utilisateurs ScoDoc8 existants seront laissés inchangés)" echo "-------------------------------------------------------------" echo -su -c (cd "$SCODOC_DIR" && flask import-scodoc7-users) +su -c "(cd $SCODOC_DIR && flask import-scodoc7-users)" "$SCODOC_USER" # ----- Migration bases départements # les départements ScoDoc7 ont été déplacés dans /opt/scodoc-data/config/dept for f in "$SCODOC_VAR_DIR"/config/depts/*.cfg do dept=$(basename "${f%.*}") - # Liste des bases de données de département: - psql -l | awk '{print $1;}' | grep ^SCO | grep -v SCOUSERS | grep -v SCO8USERS -done \ No newline at end of file + echo + echo "----------------------------------------------" + echo "| MIGRATION DU DEPARTEMENT $dept" + echo "----------------------------------------------" + su -c "(cd $SCODOC_DIR && flask import-scodoc7-dept)" "$dept" + echo "restarting postgresql server..." + systemctl restart postgresql +done + +# Précaution a priori inutile (import-scodoc7-dept efface les caches) +systemctl restart redis + + +# Commande listant les nom des departement en DB: +# Liste des bases de données de département: +# dept_db=$(psql -l | awk '{print $1;}' | grep ^SCO | grep -v SCOUSERS | grep -v SCO8USERS | awk '{print substr($1,4);}') diff --git a/tools/postupgrade.py b/tools/postupgrade.py deleted file mode 100755 index c776254d9..000000000 --- a/tools/postupgrade.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/opt/zope213/bin/python -# -*- coding: utf-8 -*- - -""" -ScoDoc post-upgrade script. - -This script is launched by upgrade.sh after each SVN update. - -Run as "root" with Zope shutted down and postgresql up, -_before_ upgrading the database. - -E. Viennet, June 2008 -Mar 2017: suppress upgrade of very old Apache configs -Aug 2020: move photos to .../var/scodoc/ -Apr 2021: bug #70 -Jun 2021: update for ScoDoc8 -""" -import os -import sys -import glob -import shutil -from scodocutils import log, SCODOC_DIR, SCODOC_VAR_DIR, SCODOC_LOGOS_DIR, SCO_TMPDIR - -if os.getuid() != 0: - log("postupgrade.py: must be run as root") - sys.exit(1) - -# Continue here... - -# --- -sys.exit(0)