forked from ScoDoc/ScoDoc
Ajout du support de DB et cache externe
La base de données postgresql et le cache redis peuvent désormais être sur un serveur externe (pas localhost) grâce aux variables d'environnement `SCODOC_DATABASE_URI` et `CACHE_REDIS_HOST`.
This commit is contained in:
parent
95c9fb9bf0
commit
99942f40ea
GPG Key ID:
202150AA0DAB9FAC
@ -576,7 +576,7 @@ def clear_scodoc_cache():
|
||||
# attaque directement redis, court-circuite ScoDoc:
|
||||
import redis
|
||||
|
||||
r = redis.Redis()
|
||||
r = redis.Redis(host=(os.environ.get("CACHE_REDIS_HOST") or "localhost"))
|
||||
r.flushall()
|
||||
# Also clear local caches:
|
||||
sco_preferences.clear_base_preferences()
|
||||
|
||||
@ -28,13 +28,13 @@
|
||||
"""Dump base de données pour debug et support technique
|
||||
|
||||
Le principe est le suivant:
|
||||
1- S'il existe une base en cours d'anonymisation, s'arrête et affiche un msg
|
||||
1- Si la base est en cours d'anonymisation, s'arrête et affiche un msg
|
||||
d'erreur à l'utilisateur, qui peut décider de la supprimer.
|
||||
|
||||
2- ScoDoc lance un script qui duplique la base (la copie de SCORT devient ANORT)
|
||||
2- ScoDoc lance un script qui duplique la base (la copie de SCODOC devient SCODOC_ANO)
|
||||
- (si elle existe deja, s'arrête)
|
||||
createdb -E UTF-8 ANORT
|
||||
pg_dump SCORT | psql ANORT
|
||||
psql postgres:///SCODOC -c "CREATE DATABASE SCODOC_ANO WITH ENCODING 'UTF8';
|
||||
pg_dump postgres:///SCODOC | psql postgres:///SCODOC_ANO
|
||||
|
||||
|
||||
3- ScoDoc lance le script d'anonymisation config/anonymize_db.py qui:
|
||||
@ -50,11 +50,12 @@ import base64
|
||||
import fcntl
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
import urllib.parse
|
||||
import requests
|
||||
|
||||
from flask import g, request
|
||||
from flask_login import current_user
|
||||
from config import RunningConfig
|
||||
|
||||
import app.scodoc.notesdb as ndb
|
||||
import app.scodoc.sco_utils as scu
|
||||
@ -72,10 +73,10 @@ def sco_dump_and_send_db(
|
||||
traceback_str = base64.urlsafe_b64decode(traceback_str_base64).decode(
|
||||
scu.SCO_ENCODING
|
||||
)
|
||||
# get current (dept) DB name:
|
||||
cursor = ndb.SimpleQuery("SELECT current_database()", {})
|
||||
db_name = cursor.fetchone()[0]
|
||||
ano_db_name = "ANO" + db_name
|
||||
db_uri = RunningConfig.SQLALCHEMY_DATABASE_URI
|
||||
db_name = urllib.parse.urlparse(db_uri).path.lstrip("/")
|
||||
ano_db_uri = db_uri + "_ANO"
|
||||
ano_db_name = db_name + "_ANO"
|
||||
# Lock
|
||||
try:
|
||||
x = open(SCO_DUMP_LOCK, "w+")
|
||||
@ -90,13 +91,15 @@ def sco_dump_and_send_db(
|
||||
_drop_ano_db(ano_db_name)
|
||||
|
||||
# Duplicate database
|
||||
_duplicate_db(db_name, ano_db_name)
|
||||
_duplicate_db(db_uri, db_name, ano_db_uri, ano_db_name)
|
||||
|
||||
# Anonymisation
|
||||
anonymize_db(ano_db_name)
|
||||
anonymize_db(ano_db_uri, ano_db_name)
|
||||
|
||||
# Send
|
||||
r = _send_db(ano_db_name, message, request_url, traceback_str=traceback_str)
|
||||
r = _send_db(
|
||||
ano_db_uri, ano_db_name, message, request_url, traceback_str=traceback_str
|
||||
)
|
||||
|
||||
finally:
|
||||
# Drop anonymized database
|
||||
@ -109,19 +112,24 @@ def sco_dump_and_send_db(
|
||||
return r
|
||||
|
||||
|
||||
def _duplicate_db(db_name, ano_db_name):
|
||||
def _duplicate_db(db_uri: str, db_name: str, ano_db_uri: str, ano_db_name: str):
|
||||
"""Create new database, and copy old one into"""
|
||||
cmd = ["createdb", "-E", "UTF-8", ano_db_name]
|
||||
cmd = [
|
||||
"psql",
|
||||
RunningConfig.SQLALCHEMY_DATABASE_URI,
|
||||
"-c",
|
||||
f"CREATE DATABASE \"{ano_db_name}\" WITH ENCODING 'UTF8';",
|
||||
]
|
||||
log(f"sco_dump_and_send_db/_duplicate_db: {cmd}")
|
||||
try:
|
||||
_ = subprocess.check_output(cmd)
|
||||
except subprocess.CalledProcessError as e:
|
||||
log(f"sco_dump_and_send_db: exception createdb {e}")
|
||||
log(f"sco_dump_and_send_db: exception psql {e}")
|
||||
raise ScoValueError(
|
||||
f"erreur lors de la creation de la base {ano_db_name}"
|
||||
) from e
|
||||
|
||||
cmd = f"pg_dump {db_name} | psql {ano_db_name}"
|
||||
cmd = ["pg_dump", db_uri, "|", "psql", ano_db_uri]
|
||||
log("sco_dump_and_send_db/_duplicate_db: {}".format(cmd))
|
||||
try:
|
||||
_ = subprocess.check_output(cmd, shell=1)
|
||||
@ -132,12 +140,12 @@ def _duplicate_db(db_name, ano_db_name):
|
||||
) from e
|
||||
|
||||
|
||||
def anonymize_db(ano_db_name):
|
||||
def anonymize_db(ano_db_uri: str, ano_db_name: str):
|
||||
"""Anonymize a ScoDoc database"""
|
||||
cmd = os.path.join(scu.SCO_TOOLS_DIR, "anonymize_db.py")
|
||||
cmd = [os.path.join(scu.SCO_TOOLS_DIR, "anonymize_db.py"), ano_db_uri]
|
||||
log(f"anonymize_db: {cmd}")
|
||||
try:
|
||||
_ = subprocess.check_output([cmd, ano_db_name])
|
||||
_ = subprocess.check_output(cmd)
|
||||
except subprocess.CalledProcessError as e:
|
||||
log(f"sco_dump_and_send_db: exception in anonymisation: {e}")
|
||||
raise ScoValueError(
|
||||
@ -156,14 +164,17 @@ def _get_scodoc_serial():
|
||||
|
||||
|
||||
def _send_db(
|
||||
ano_db_name: str, message: str = "", request_url: str = "", traceback_str: str = ""
|
||||
ano_db_uri: str,
|
||||
ano_db_name: str,
|
||||
message: str = "",
|
||||
request_url: str = "",
|
||||
traceback_str: str = "",
|
||||
):
|
||||
"""Dump this (anonymized) database and send it to tech support"""
|
||||
log(f"dumping anonymized database {ano_db_name}")
|
||||
try:
|
||||
dump = subprocess.check_output(
|
||||
f"pg_dump --format=custom {ano_db_name}", shell=1
|
||||
)
|
||||
cmd = ["pg_dump", "--format=custom", ano_db_uri]
|
||||
dump = subprocess.check_output(cmd, shell=1)
|
||||
except subprocess.CalledProcessError as e:
|
||||
log(f"sco_dump_and_send_db: exception in anonymisation: {e}")
|
||||
raise ScoValueError(
|
||||
@ -203,18 +214,14 @@ def _send_db(
|
||||
return r
|
||||
|
||||
|
||||
def _drop_ano_db(ano_db_name):
|
||||
def _drop_ano_db(ano_db_name: str):
|
||||
"""drop temp database if it exists"""
|
||||
existing_databases = [
|
||||
s.split("|")[0].strip()
|
||||
for s in subprocess.check_output(["psql", "-l"])
|
||||
.decode(scu.SCO_ENCODING)
|
||||
.split("\n")[3:]
|
||||
cmd = [
|
||||
"psql",
|
||||
RunningConfig.SQLALCHEMY_DATABASE_URI,
|
||||
"-c",
|
||||
f"DROP DATABASE IF EXISTS \"{ano_db_name}\";",
|
||||
]
|
||||
if ano_db_name not in existing_databases:
|
||||
log("_drop_ano_db: no temp db, nothing to drop")
|
||||
return
|
||||
cmd = ["dropdb", ano_db_name]
|
||||
log(f"sco_dump_and_send_db: {cmd}")
|
||||
try:
|
||||
_ = subprocess.check_output(cmd)
|
||||
|
||||
@ -39,6 +39,8 @@ import sys
|
||||
import traceback
|
||||
import psycopg2
|
||||
from psycopg2 import extras
|
||||
import urllib.parse
|
||||
import re
|
||||
|
||||
|
||||
def log(msg):
|
||||
@ -209,7 +211,6 @@ def anonymize_db(cursor):
|
||||
for tablecolumn in ANONYMIZED_FIELDS:
|
||||
anonymize_column(cursor, tablecolumn)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
PROCESS_USERS = False
|
||||
if len(sys.argv) < 2 or len(sys.argv) > 3:
|
||||
@ -217,18 +218,19 @@ if __name__ == "__main__":
|
||||
if len(sys.argv) > 2:
|
||||
if sys.argv[1] != "--users":
|
||||
usage()
|
||||
dbname = sys.argv[2]
|
||||
dburi = sys.argv[2]
|
||||
PROCESS_USERS = True
|
||||
else:
|
||||
dbname = sys.argv[1]
|
||||
dburi = sys.argv[1]
|
||||
|
||||
dbname = urllib.parse.urlparse(dburi).path.lstrip("/")
|
||||
|
||||
log(f"\nAnonymizing database {dbname}")
|
||||
cnx_string = "dbname=" + dbname
|
||||
try:
|
||||
cnx = psycopg2.connect(cnx_string)
|
||||
cnx = psycopg2.connect(dburi)
|
||||
except Exception as e:
|
||||
log(f"\n*** Error: can't connect to database {dbname} ***\n")
|
||||
log(f"""connexion string was "{cnx_string}" """)
|
||||
log(f"""connexion uri was "{re.compile(r'(postgres://[^:]+:)([^@]+)(@)').sub(r'\1*****\3', uri)}" """)
|
||||
traceback.print_exc()
|
||||
|
||||
cnx.set_session(autocommit=False)
|
||||
|
||||
@ -11,11 +11,17 @@ die() {
|
||||
}
|
||||
[ $# = 1 ] || [ $# = 2 ] || die "Usage $0 [--drop] db_name"
|
||||
|
||||
if [ -z "${SCODOC_DATABASE_URI}" ]; then
|
||||
PG_URI="postgresql:///postgres"
|
||||
else
|
||||
PG_URI=$(echo $SCODOC_DATABASE_URI | sed 's|/[^/]*$|/postgres|')
|
||||
fi
|
||||
|
||||
if [ "$1" = "--drop" ]
|
||||
then
|
||||
db_name="$2"
|
||||
echo "Dropping database $db_name..."
|
||||
dropdb --if-exists "$db_name"
|
||||
psql $PG_URI -c "DROP DATABASE IF EXISTS $db_name;"
|
||||
else
|
||||
db_name="$1"
|
||||
fi
|
||||
@ -30,5 +36,5 @@ source "$SCRIPT_DIR"/utils.sh || die "config.sh not found, exiting"
|
||||
|
||||
# ---
|
||||
echo 'Creating postgresql database ' "$db_name"
|
||||
createdb -E UTF-8 -p "$POSTGRES_PORT" -O "$POSTGRES_USER" "$db_name"
|
||||
echo 'CREATE EXTENSION IF NOT EXISTS "unaccent";' | psql -p "$POSTGRES_PORT" "$db_name" "$POSTGRES_USER"
|
||||
psql $PG_URI -c "CREATE DATABASE \"$db_name\" WITH ENCODING 'UTF-8';"
|
||||
psql $(echo $PG_URI | sed "s|/postgres\$|/$db_name|") -c 'CREATE EXTENSION IF NOT EXISTS "unaccent";'
|
||||
|
||||
Loading…
Reference in New Issue
Block a user