lyanis/ScoDoc
1
0
Fork 0
forked from ScoDoc/ScoDoc
Code Pull Requests Actions Packages Activity

Ajout du support de DB et cache externe

Browse Source 
La base de données postgresql et le cache redis peuvent désormais être sur un serveur externe (pas localhost) grâce aux variables d'environnement `SCODOC_DATABASE_URI` et `CACHE_REDIS_HOST`.
This commit is contained in:
Lyanis Souidi 2024-06-03 15:14:27 +02:00
parent 95c9fb9bf0
commit 99942f40ea
Signed by: lyanis
GPG Key ID: 202150AA0DAB9FAC
4 changed files with 58 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/__init__.py
View File

@ -576,7 +576,7 @@ def clear_scodoc_cache():
# attaque directement redis, court-circuite ScoDoc: # attaque directement redis, court-circuite ScoDoc:
import redis import redis
r = redis.Redis() r = redis.Redis(host=(os.environ.get("CACHE_REDIS_HOST") or "localhost"))
r.flushall() r.flushall()
# Also clear local caches: # Also clear local caches:
sco_preferences.clear_base_preferences() sco_preferences.clear_base_preferences()

73
app/scodoc/sco_dump_db.py
View File

@ -28,13 +28,13 @@
"""Dump base de données pour debug et support technique """Dump base de données pour debug et support technique
Le principe est le suivant: Le principe est le suivant:
1- S'il existe une base en cours d'anonymisation, s'arrête et affiche un msg 1- Si la base est en cours d'anonymisation, s'arrête et affiche un msg
d'erreur à l'utilisateur, qui peut décider de la supprimer. d'erreur à l'utilisateur, qui peut décider de la supprimer.
2- ScoDoc lance un script qui duplique la base (la copie de SCORT devient ANORT) 2- ScoDoc lance un script qui duplique la base (la copie de SCODOC devient SCODOC_ANO)
- (si elle existe deja, s'arrête) - (si elle existe deja, s'arrête)
createdb -E UTF-8 ANORT psql postgres:///SCODOC -c "CREATE DATABASE SCODOC_ANO WITH ENCODING 'UTF8';
pg_dump SCORT | psql ANORT pg_dump postgres:///SCODOC | psql postgres:///SCODOC_ANO
3- ScoDoc lance le script d'anonymisation config/anonymize_db.py qui: 3- ScoDoc lance le script d'anonymisation config/anonymize_db.py qui:
@ -50,11 +50,12 @@ import base64
import fcntl import fcntl
import os import os
import subprocess import subprocess
import urllib.parse
import requests import requests
from flask import g, request from flask import g, request
from flask_login import current_user from flask_login import current_user
from config import RunningConfig
import app.scodoc.notesdb as ndb import app.scodoc.notesdb as ndb
import app.scodoc.sco_utils as scu import app.scodoc.sco_utils as scu
@ -72,10 +73,10 @@ def sco_dump_and_send_db(
traceback_str = base64.urlsafe_b64decode(traceback_str_base64).decode( traceback_str = base64.urlsafe_b64decode(traceback_str_base64).decode(
scu.SCO_ENCODING scu.SCO_ENCODING
) )
# get current (dept) DB name: db_uri = RunningConfig.SQLALCHEMY_DATABASE_URI
cursor = ndb.SimpleQuery("SELECT current_database()", {}) db_name = urllib.parse.urlparse(db_uri).path.lstrip("/")
db_name = cursor.fetchone()[0] ano_db_uri = db_uri + "_ANO"
ano_db_name = "ANO" + db_name ano_db_name = db_name + "_ANO"
# Lock # Lock
try: try:
x = open(SCO_DUMP_LOCK, "w+") x = open(SCO_DUMP_LOCK, "w+")
@ -90,13 +91,15 @@ def sco_dump_and_send_db(
_drop_ano_db(ano_db_name) _drop_ano_db(ano_db_name)
# Duplicate database # Duplicate database
_duplicate_db(db_name, ano_db_name) _duplicate_db(db_uri, db_name, ano_db_uri, ano_db_name)
# Anonymisation # Anonymisation
anonymize_db(ano_db_name) anonymize_db(ano_db_uri, ano_db_name)
# Send # Send
r = _send_db(ano_db_name, message, request_url, traceback_str=traceback_str) r = _send_db(
ano_db_uri, ano_db_name, message, request_url, traceback_str=traceback_str
)
finally: finally:
# Drop anonymized database # Drop anonymized database
@ -109,19 +112,24 @@ def sco_dump_and_send_db(
return r return r
def _duplicate_db(db_name, ano_db_name): def _duplicate_db(db_uri: str, db_name: str, ano_db_uri: str, ano_db_name: str):
"""Create new database, and copy old one into""" """Create new database, and copy old one into"""
cmd = ["createdb", "-E", "UTF-8", ano_db_name] cmd = [
"psql",
RunningConfig.SQLALCHEMY_DATABASE_URI,
"-c",
f"CREATE DATABASE \"{ano_db_name}\" WITH ENCODING 'UTF8';",
]
log(f"sco_dump_and_send_db/_duplicate_db: {cmd}") log(f"sco_dump_and_send_db/_duplicate_db: {cmd}")
try: try:
_ = subprocess.check_output(cmd) _ = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
log(f"sco_dump_and_send_db: exception createdb {e}") log(f"sco_dump_and_send_db: exception psql {e}")
raise ScoValueError( raise ScoValueError(
f"erreur lors de la creation de la base {ano_db_name}" f"erreur lors de la creation de la base {ano_db_name}"
) from e ) from e
cmd = f"pg_dump {db_name} | psql {ano_db_name}" cmd = ["pg_dump", db_uri, "|", "psql", ano_db_uri]
log("sco_dump_and_send_db/_duplicate_db: {}".format(cmd)) log("sco_dump_and_send_db/_duplicate_db: {}".format(cmd))
try: try:
_ = subprocess.check_output(cmd, shell=1) _ = subprocess.check_output(cmd, shell=1)
@ -132,12 +140,12 @@ def _duplicate_db(db_name, ano_db_name):
) from e ) from e
def anonymize_db(ano_db_name): def anonymize_db(ano_db_uri: str, ano_db_name: str):
"""Anonymize a ScoDoc database""" """Anonymize a ScoDoc database"""
cmd = os.path.join(scu.SCO_TOOLS_DIR, "anonymize_db.py") cmd = [os.path.join(scu.SCO_TOOLS_DIR, "anonymize_db.py"), ano_db_uri]
log(f"anonymize_db: {cmd}") log(f"anonymize_db: {cmd}")
try: try:
_ = subprocess.check_output([cmd, ano_db_name]) _ = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
log(f"sco_dump_and_send_db: exception in anonymisation: {e}") log(f"sco_dump_and_send_db: exception in anonymisation: {e}")
raise ScoValueError( raise ScoValueError(
@ -156,14 +164,17 @@ def _get_scodoc_serial():
def _send_db( def _send_db(
ano_db_name: str, message: str = "", request_url: str = "", traceback_str: str = "" ano_db_uri: str,
ano_db_name: str,
message: str = "",
request_url: str = "",
traceback_str: str = "",
): ):
"""Dump this (anonymized) database and send it to tech support""" """Dump this (anonymized) database and send it to tech support"""
log(f"dumping anonymized database {ano_db_name}") log(f"dumping anonymized database {ano_db_name}")
try: try:
dump = subprocess.check_output( cmd = ["pg_dump", "--format=custom", ano_db_uri]
f"pg_dump --format=custom {ano_db_name}", shell=1 dump = subprocess.check_output(cmd, shell=1)
)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
log(f"sco_dump_and_send_db: exception in anonymisation: {e}") log(f"sco_dump_and_send_db: exception in anonymisation: {e}")
raise ScoValueError( raise ScoValueError(
@ -203,18 +214,14 @@ def _send_db(
return r return r
def _drop_ano_db(ano_db_name): def _drop_ano_db(ano_db_name: str):
"""drop temp database if it exists""" """drop temp database if it exists"""
existing_databases = [ cmd = [
s.split("|")[0].strip() "psql",
for s in subprocess.check_output(["psql", "-l"]) RunningConfig.SQLALCHEMY_DATABASE_URI,
.decode(scu.SCO_ENCODING) "-c",
.split("\n")[3:] f"DROP DATABASE IF EXISTS \"{ano_db_name}\";",
] ]
if ano_db_name not in existing_databases:
log("_drop_ano_db: no temp db, nothing to drop")
return
cmd = ["dropdb", ano_db_name]
log(f"sco_dump_and_send_db: {cmd}") log(f"sco_dump_and_send_db: {cmd}")
try: try:
_ = subprocess.check_output(cmd) _ = subprocess.check_output(cmd)

14
tools/anonymize_db.py
View File

@ -39,6 +39,8 @@ import sys
import traceback import traceback
import psycopg2 import psycopg2
from psycopg2 import extras from psycopg2 import extras
import urllib.parse
import re
def log(msg): def log(msg):
@ -209,7 +211,6 @@ def anonymize_db(cursor):
for tablecolumn in ANONYMIZED_FIELDS: for tablecolumn in ANONYMIZED_FIELDS:
anonymize_column(cursor, tablecolumn) anonymize_column(cursor, tablecolumn)
if __name__ == "__main__": if __name__ == "__main__":
PROCESS_USERS = False PROCESS_USERS = False
if len(sys.argv) < 2 or len(sys.argv) > 3: if len(sys.argv) < 2 or len(sys.argv) > 3:
@ -217,18 +218,19 @@ if __name__ == "__main__":
if len(sys.argv) > 2: if len(sys.argv) > 2:
if sys.argv[1] != "--users": if sys.argv[1] != "--users":
usage() usage()
dbname = sys.argv[2] dburi = sys.argv[2]
PROCESS_USERS = True PROCESS_USERS = True
else: else:
dbname = sys.argv[1] dburi = sys.argv[1]
dbname = urllib.parse.urlparse(dburi).path.lstrip("/")
log(f"\nAnonymizing database {dbname}") log(f"\nAnonymizing database {dbname}")
cnx_string = "dbname=" + dbname
try: try:
cnx = psycopg2.connect(cnx_string) cnx = psycopg2.connect(dburi)
except Exception as e: except Exception as e:
log(f"\n*** Error: can't connect to database {dbname} ***\n") log(f"\n*** Error: can't connect to database {dbname} ***\n")
log(f"""connexion string was "{cnx_string}" """) log(f"""connexion uri was "{re.compile(r'(postgres://[^:]+:)([^@]+)(@)').sub(r'\1*****\3', uri)}" """)
traceback.print_exc() traceback.print_exc()
cnx.set_session(autocommit=False) cnx.set_session(autocommit=False)

12
tools/create_database.sh
View File

@ -11,11 +11,17 @@ die() {
} }
[ $# = 1 ] || [ $# = 2 ] || die "Usage $0 [--drop] db_name" [ $# = 1 ] || [ $# = 2 ] || die "Usage $0 [--drop] db_name"
if [ -z "${SCODOC_DATABASE_URI}" ]; then
PG_URI="postgresql:///postgres"
else
PG_URI=$(echo $SCODOC_DATABASE_URI | sed 's|/[^/]*$|/postgres|')
fi
if [ "$1" = "--drop" ] if [ "$1" = "--drop" ]
then then
db_name="$2" db_name="$2"
echo "Dropping database $db_name..." echo "Dropping database $db_name..."
dropdb --if-exists "$db_name" psql $PG_URI -c "DROP DATABASE IF EXISTS $db_name;"
else else
db_name="$1" db_name="$1"
fi fi
@ -30,5 +36,5 @@ source "$SCRIPT_DIR"/utils.sh || die "config.sh not found, exiting"
# --- # ---
echo 'Creating postgresql database ' "$db_name" echo 'Creating postgresql database ' "$db_name"
createdb -E UTF-8 -p "$POSTGRES_PORT" -O "$POSTGRES_USER" "$db_name" psql $PG_URI -c "CREATE DATABASE \"$db_name\" WITH ENCODING 'UTF-8';"
echo 'CREATE EXTENSION IF NOT EXISTS "unaccent";' | psql -p "$POSTGRES_PORT" "$db_name" "$POSTGRES_USER" psql $(echo $PG_URI | sed "s|/postgres\$|/$db_name|") -c 'CREATE EXTENSION IF NOT EXISTS "unaccent";'