ScoDoc-Lille/app/__init__.py

533 lines
18 KiB
Python
Raw Normal View History

2021-05-29 18:22:51 +02:00
# -*- coding: UTF-8 -*
# pylint: disable=invalid-name
2022-09-10 15:23:54 +02:00
import base64
import datetime
2021-05-29 18:22:51 +02:00
import os
2021-08-29 19:57:32 +02:00
import socket
import sys
2021-08-29 19:57:32 +02:00
import time
import traceback
2021-05-29 18:22:51 +02:00
import logging
2021-08-29 19:57:32 +02:00
from logging.handlers import SMTPHandler, WatchedFileHandler
from threading import Thread
2021-05-29 18:22:51 +02:00
from flask import current_app, g, request
2021-05-29 18:22:51 +02:00
from flask import Flask
2022-03-04 20:02:50 +01:00
from flask import abort, flash, has_request_context, jsonify
2021-07-28 08:12:57 +02:00
from flask import render_template
from flask.json import JSONEncoder
2021-08-29 22:42:38 +02:00
from flask.logging import default_handler
2021-05-29 18:22:51 +02:00
from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate
from flask_login import LoginManager, current_user
2021-05-29 18:22:51 +02:00
from flask_mail import Mail
from flask_bootstrap import Bootstrap
from flask_moment import Moment
2021-07-19 19:53:01 +02:00
from flask_caching import Cache
import sqlalchemy
2021-05-29 18:22:51 +02:00
from app.scodoc.sco_exceptions import (
AccessDenied,
ScoBugCatcher,
ScoGenError,
ScoValueError,
APIInvalidParams,
)
from config import DevConfig
2021-08-21 17:07:44 +02:00
import sco_version
2021-05-29 18:22:51 +02:00
db = SQLAlchemy()
migrate = Migrate(compare_type=True)
2021-05-29 18:22:51 +02:00
login = LoginManager()
login.login_view = "auth.login"
2021-09-11 15:59:06 +02:00
login.login_message = "Identifiez-vous pour accéder à cette page."
2021-05-29 18:22:51 +02:00
mail = Mail()
bootstrap = Bootstrap()
2021-05-29 18:22:51 +02:00
moment = Moment()
2022-03-12 09:39:28 +01:00
CACHE_TYPE = os.environ.get("CACHE_TYPE")
cache = Cache(
2021-07-27 14:33:11 +02:00
config={
# see https://flask-caching.readthedocs.io/en/latest/index.html#configuring-flask-caching
2022-03-12 09:39:28 +01:00
"CACHE_TYPE": CACHE_TYPE or "RedisCache",
# by default, never expire:
"CACHE_DEFAULT_TIMEOUT": os.environ.get("CACHE_DEFAULT_TIMEOUT") or 0,
2021-07-27 14:33:11 +02:00
}
)
2021-07-19 19:53:01 +02:00
2021-05-29 18:22:51 +02:00
2021-07-28 08:12:57 +02:00
def handle_sco_value_error(exc):
return render_template("sco_value_error.html", exc=exc), 404
def handle_access_denied(exc):
return render_template("error_access_denied.html", exc=exc), 403
def internal_server_error(exc):
"""Bugs scodoc, erreurs 500"""
2021-08-29 22:42:38 +02:00
# note that we set the 500 status explicitly
2022-09-10 15:23:54 +02:00
from app.scodoc import sco_utils as scu
return (
render_template(
"error_500.html",
SCOVERSION=sco_version.SCOVERSION,
date=datetime.datetime.now().isoformat(),
exc=exc,
2022-09-10 15:23:54 +02:00
traceback_str_base64=base64.urlsafe_b64encode(
traceback.format_exc().encode(scu.SCO_ENCODING)
).decode(scu.SCO_ENCODING),
request_url=request.url,
2022-09-10 15:23:54 +02:00
scu=scu,
),
500,
)
2021-08-29 22:42:38 +02:00
def handle_sco_bug(exc):
"""Un bug, en général rare, sur lequel les dev cherchent des
informations pour le corriger.
"""
Thread(
target=_async_dump, args=(current_app._get_current_object(), request.url)
).start()
return internal_server_error(exc)
def _async_dump(app, request_url: str):
from app.scodoc.sco_dump_db import sco_dump_and_send_db
with app.app_context():
ndb.open_db_connection()
try:
sco_dump_and_send_db("ScoBugCatcher", request_url=request_url)
except ScoValueError:
pass
2021-09-13 09:54:53 +02:00
def handle_invalid_usage(error):
response = jsonify(error.to_dict())
response.status_code = error.status_code
return response
# JSON ENCODING
class ScoDocJSONEncoder(JSONEncoder):
def default(self, o):
2022-10-23 14:51:57 +02:00
if isinstance(o, (datetime.datetime, datetime.date)):
return o.isoformat()
return super().default(o)
def render_raw_html(template_filename: str, **args) -> str:
"""Load and render an HTML file _without_ using Flask
Necessary for 503 error mesage, when DB is down and Flask may be broken.
"""
template_path = os.path.join(
current_app.config["SCODOC_DIR"],
"app",
current_app.template_folder,
template_filename,
)
with open(template_path) as f:
txt = f.read().format(**args)
return txt
def postgresql_server_error(e):
"""Erreur de connection au serveur postgresql (voir notesdb.open_db_connection)"""
return render_raw_html("error_503.html", SCOVERSION=sco_version.SCOVERSION), 503
class LogRequestFormatter(logging.Formatter):
2021-08-29 22:42:38 +02:00
"""Ajoute URL et remote_addr for logging"""
def format(self, record):
if has_request_context():
record.url = request.url
record.remote_addr = request.remote_addr
else:
record.url = None
record.remote_addr = None
record.sco_user = current_user
if has_request_context():
record.sco_admin_mail = current_app.config["SCODOC_ADMIN_MAIL"]
else:
record.sco_admin_mail = "(pas de requête)"
return super().format(record)
class LogExceptionFormatter(logging.Formatter):
"""Formatteur pour les exceptions: ajoute détails"""
def format(self, record):
if has_request_context():
record.url = request.url
record.remote_addr = request.environ.get(
"HTTP_X_FORWARDED_FOR", request.remote_addr
)
record.http_referrer = request.referrer
record.http_method = request.method
if request.method == "GET":
record.http_params = str(request.args)
else:
2021-09-27 22:58:05 +02:00
# rep = reprlib.Repr() # abbrège
record.http_params = str(request.form)[:2048]
else:
record.url = None
record.remote_addr = None
record.http_referrer = None
record.http_method = None
record.http_params = None
record.sco_user = current_user
if has_request_context():
record.sco_admin_mail = current_app.config["SCODOC_ADMIN_MAIL"]
else:
record.sco_admin_mail = "(pas de requête)"
2021-08-29 22:42:38 +02:00
return super().format(record)
2021-09-12 23:06:23 +02:00
class ScoSMTPHandler(SMTPHandler):
def getSubject(self, record: logging.LogRecord) -> str:
stack_summary = traceback.extract_tb(record.exc_info[2])
frame_summary = stack_summary[-1]
2021-09-13 09:54:53 +02:00
subject = f"ScoExc({sco_version.SCOVERSION}): {record.exc_info[0].__name__} in {frame_summary.name} {frame_summary.filename}"
2021-09-12 23:06:23 +02:00
return subject
class ReverseProxied(object):
"""Adaptateur wsgi qui nous permet d'avoir toutes les URL calculées en https
sauf quand on est en dev.
La variable HTTP_X_FORWARDED_PROTO est positionnée par notre config nginx"""
def __init__(self, app):
self.app = app
def __call__(self, environ, start_response):
scheme = environ.get("HTTP_X_FORWARDED_PROTO")
if scheme:
environ["wsgi.url_scheme"] = scheme # ou forcer à https ici ?
return self.app(environ, start_response)
def create_app(config_class=DevConfig):
app = Flask(__name__, static_url_path="/ScoDoc/static", static_folder="static")
app.wsgi_app = ReverseProxied(app.wsgi_app)
app.json_encoder = ScoDocJSONEncoder
2022-07-11 18:17:39 +02:00
app.logger.setLevel(logging.INFO)
# Evite de logguer toutes les requetes dans notre log
logging.getLogger("werkzeug").disabled = True
2021-05-29 18:22:51 +02:00
app.config.from_object(config_class)
# Vérifie/crée lien sym pour les URL statiques
link_filename = f"{app.root_path}/static/links/{sco_version.SCOVERSION}"
if not os.path.exists(link_filename):
app.logger.info(f"creating symlink {link_filename}")
os.symlink("..", link_filename)
2021-05-29 18:22:51 +02:00
db.init_app(app)
migrate.init_app(app, db)
login.init_app(app)
mail.init_app(app)
bootstrap.init_app(app)
moment.init_app(app)
2021-07-19 19:53:01 +02:00
cache.init_app(app)
sco_cache.CACHE = cache
2022-03-12 09:39:28 +01:00
if CACHE_TYPE: # non default
app.logger.info(f"CACHE_TYPE={CACHE_TYPE}")
2021-05-29 18:22:51 +02:00
app.register_error_handler(ScoGenError, handle_sco_value_error)
2021-07-28 08:12:57 +02:00
app.register_error_handler(ScoValueError, handle_sco_value_error)
app.register_error_handler(ScoBugCatcher, handle_sco_bug)
app.register_error_handler(AccessDenied, handle_access_denied)
2021-08-29 22:42:38 +02:00
app.register_error_handler(500, internal_server_error)
app.register_error_handler(503, postgresql_server_error)
2021-09-13 09:54:53 +02:00
app.register_error_handler(APIInvalidParams, handle_invalid_usage)
2021-07-28 08:12:57 +02:00
2021-05-29 18:22:51 +02:00
from app.auth import bp as auth_bp
app.register_blueprint(auth_bp, url_prefix="/auth")
2021-12-23 19:28:25 +01:00
from app.entreprises import bp as entreprises_bp
2022-02-21 19:25:38 +01:00
2021-12-23 19:28:25 +01:00
app.register_blueprint(entreprises_bp, url_prefix="/ScoDoc/entreprises")
2021-07-04 12:32:13 +02:00
from app.views import scodoc_bp
from app.views import scolar_bp
2021-05-29 18:22:51 +02:00
from app.views import notes_bp
from app.views import users_bp
from app.views import absences_bp
from app.api import api_bp
from app.api import api_web_bp
2021-05-29 18:22:51 +02:00
2021-07-04 12:32:13 +02:00
# https://scodoc.fr/ScoDoc
app.register_blueprint(scodoc_bp)
# https://scodoc.fr/ScoDoc/RT/Scolarite/...
app.register_blueprint(scolar_bp, url_prefix="/ScoDoc/<scodoc_dept>/Scolarite")
# https://scodoc.fr/ScoDoc/RT/Scolarite/Notes/...
app.register_blueprint(notes_bp, url_prefix="/ScoDoc/<scodoc_dept>/Scolarite/Notes")
# https://scodoc.fr/ScoDoc/RT/Scolarite/Users/...
app.register_blueprint(users_bp, url_prefix="/ScoDoc/<scodoc_dept>/Scolarite/Users")
# https://scodoc.fr/ScoDoc/RT/Scolarite/Absences/...
app.register_blueprint(
absences_bp, url_prefix="/ScoDoc/<scodoc_dept>/Scolarite/Absences"
)
2021-09-09 12:49:23 +02:00
app.register_blueprint(api_bp, url_prefix="/ScoDoc/api")
app.register_blueprint(api_web_bp, url_prefix="/ScoDoc/<scodoc_dept>/api")
2022-07-26 09:00:48 +02:00
scodoc_log_formatter = LogRequestFormatter(
"[%(asctime)s] %(sco_user)s@%(remote_addr)s requested %(url)s\n"
"%(levelname)s: %(message)s"
)
# les champs additionnels sont définis dans LogRequestFormatter
scodoc_exc_formatter = LogExceptionFormatter(
"[%(asctime)s] %(sco_user)s@%(remote_addr)s requested %(url)s\n"
"%(levelname)s: %(message)s\n"
"Referrer: %(http_referrer)s\n"
"Method: %(http_method)s\n"
"Params: %(http_params)s\n"
"Admin mail: %(sco_admin_mail)s\n"
2021-08-29 22:42:38 +02:00
)
if not app.testing:
2021-08-29 22:42:38 +02:00
if not app.debug:
# --- Config logs pour PRODUCTION
# On supprime le logguer par défaut qui va vers stderr et pollue les logs systemes
app.logger.removeHandler(default_handler)
# --- Mail des messages ERROR et CRITICAL
if app.config["MAIL_SERVER"]:
auth = None
if app.config["MAIL_USERNAME"] or app.config["MAIL_PASSWORD"]:
auth = (app.config["MAIL_USERNAME"], app.config["MAIL_PASSWORD"])
secure = None
if app.config["MAIL_USE_TLS"]:
secure = ()
host_name = socket.gethostname()
2021-09-12 23:06:23 +02:00
mail_handler = ScoSMTPHandler(
2021-08-29 22:42:38 +02:00
mailhost=(app.config["MAIL_SERVER"], app.config["MAIL_PORT"]),
fromaddr=app.config["SCODOC_MAIL_FROM"],
2021-08-29 22:42:38 +02:00
toaddrs=["exception@scodoc.org"],
2021-09-13 10:06:25 +02:00
subject="ScoDoc Exception", # unused see ScoSMTPHandler
2021-08-29 22:42:38 +02:00
credentials=auth,
secure=secure,
)
mail_handler.setFormatter(scodoc_exc_formatter)
mail_handler.setLevel(logging.ERROR)
app.logger.addHandler(mail_handler)
else:
# Pour logs en DEV uniquement:
default_handler.setFormatter(scodoc_log_formatter)
2021-08-29 22:42:38 +02:00
# Config logs pour DEV et PRODUCTION
# Configuration des logs (actifs aussi en mode development)
2021-08-29 19:57:32 +02:00
# usually /opt/scodoc-data/log/scodoc.log:
# rotated by logrotate
file_handler = WatchedFileHandler(
app.config["SCODOC_LOG_FILE"], encoding="utf-8"
2021-05-29 18:22:51 +02:00
)
file_handler.setFormatter(scodoc_log_formatter)
2021-05-29 18:22:51 +02:00
file_handler.setLevel(logging.INFO)
app.logger.addHandler(file_handler)
# Log pour les erreurs (exceptions) uniquement:
# usually /opt/scodoc-data/log/scodoc_exc.log
file_handler = WatchedFileHandler(
app.config["SCODOC_ERR_FILE"], encoding="utf-8"
)
file_handler.setFormatter(scodoc_exc_formatter)
file_handler.setLevel(logging.ERROR)
app.logger.addHandler(file_handler)
2021-05-29 18:22:51 +02:00
2021-08-29 19:57:32 +02:00
# app.logger.setLevel(logging.INFO)
2021-08-21 17:07:44 +02:00
app.logger.info(f"{sco_version.SCONAME} {sco_version.SCOVERSION} startup")
2021-08-29 19:57:32 +02:00
app.logger.info(
f"create_app({config_class.__name__}, {config_class.SQLALCHEMY_DATABASE_URI})"
)
# ---- INITIALISATION SPECIFIQUES A SCODOC
from app.scodoc import sco_bulletins_generator
2021-08-29 19:57:32 +02:00
from app.scodoc.sco_bulletins_legacy import BulletinGeneratorLegacy
from app.scodoc.sco_bulletins_standard import BulletinGeneratorStandard
2022-02-21 19:25:38 +01:00
from app.but.bulletin_but_pdf import BulletinGeneratorStandardBUT
2021-08-29 19:57:32 +02:00
from app.scodoc.sco_bulletins_ucac import BulletinGeneratorUCAC
2022-02-21 19:25:38 +01:00
# l'ordre est important, le premier sera le "défaut" pour les nouveaux départements.
2021-08-29 19:57:32 +02:00
sco_bulletins_generator.register_bulletin_class(BulletinGeneratorStandard)
2022-02-21 19:25:38 +01:00
sco_bulletins_generator.register_bulletin_class(BulletinGeneratorStandardBUT)
sco_bulletins_generator.register_bulletin_class(BulletinGeneratorLegacy)
2021-08-29 19:57:32 +02:00
sco_bulletins_generator.register_bulletin_class(BulletinGeneratorUCAC)
if app.testing or app.debug:
from app.scodoc.sco_bulletins_example import BulletinGeneratorExample
sco_bulletins_generator.register_bulletin_class(BulletinGeneratorExample)
2021-09-09 08:03:43 +02:00
2021-05-29 18:22:51 +02:00
return app
2022-07-26 09:00:48 +02:00
def set_sco_dept(scodoc_dept: str, open_cnx=True):
2021-08-13 00:34:58 +02:00
"""Set global g object to given dept and open db connection if needed"""
# Check that dept exists
try:
dept = Departement.query.filter_by(acronym=scodoc_dept).first()
except sqlalchemy.exc.OperationalError:
abort(503)
2021-08-13 00:34:58 +02:00
if not dept:
raise ScoValueError(f"Invalid dept: {scodoc_dept}")
g.scodoc_dept = scodoc_dept # l'acronyme
g.scodoc_dept_id = dept.id # l'id
2022-07-26 09:00:48 +02:00
if open_cnx and not hasattr(g, "db_conn"):
2021-08-13 00:34:58 +02:00
ndb.open_db_connection()
2021-10-16 19:20:36 +02:00
if not hasattr(g, "stored_get_formsemestre"):
g.stored_get_formsemestre = {}
2021-08-13 00:34:58 +02:00
def user_db_init():
"""Initialize the users database.
Check that basic roles and admin user exist.
"""
from app.auth.models import User, Role
current_app.logger.info("Init User's db")
# Create roles:
Role.reset_standard_roles_permissions()
current_app.logger.info("created initial roles")
# Ensure that admin exists
admin_mail = current_app.config.get("SCODOC_ADMIN_MAIL")
if admin_mail:
admin_user_name = current_app.config["SCODOC_ADMIN_LOGIN"]
user = User.query.filter_by(user_name=admin_user_name).first()
if not user:
user = User(user_name=admin_user_name, email=admin_mail)
try:
db.session.add(user)
db.session.commit()
except:
db.session.rollback()
raise
current_app.logger.info(
"created initial admin user, login: {u.user_name}, email: {u.email}".format(
u=user
)
)
def sco_db_insert_constants():
"""Initialize Sco database: insert some constants (modalités, ...)."""
from app import models
current_app.logger.info("Init Sco db")
# Modalités:
models.FormationModalite.insert_modalites()
def initialize_scodoc_database(erase=False, create_all=False):
"""Initialize the database for unit tests
Starts from an existing database and create all necessary
SQL tables and functions.
If erase is True, _erase_ all database content.
"""
from app import models
# - ERASE (the truncation sql function has been defined above)
if erase:
truncate_database()
# - Create all tables
if create_all:
# managed by migrations, except for TESTS
db.create_all()
# - Insert initial roles and create super-admin user
user_db_init()
# - Insert some constant values (modalites, ...)
sco_db_insert_constants()
2021-08-10 12:57:38 +02:00
# - Flush cache
clear_scodoc_cache()
def truncate_database():
"""Erase content of all tables (including users !) from
the current database.
"""
# use a stored SQL function, see createtables.sql
try:
db.session.execute("SELECT truncate_tables('scodoc');")
db.session.commit()
except:
db.session.rollback()
raise
2021-08-10 12:57:38 +02:00
def clear_scodoc_cache():
"""Clear ScoDoc cache
This cache (currently Redis) is persistent between invocation
and it may be necessary to clear it during developement or tests.
"""
# attaque directement redis, court-circuite ScoDoc:
import redis
r = redis.Redis()
r.flushall()
2021-08-13 00:34:58 +02:00
# Also clear local caches:
sco_preferences.clear_base_preferences()
2021-08-10 12:57:38 +02:00
2021-08-29 19:57:32 +02:00
# --------- Logging
2021-08-30 23:28:15 +02:00
def log(msg: str, silent_test=True):
2021-08-29 19:57:32 +02:00
"""log a message.
If Flask app, use configured logger, else stderr.
"""
if silent_test and current_app and current_app.config["TESTING"]:
2021-08-30 23:28:15 +02:00
return
2021-08-29 19:57:32 +02:00
try:
dept = getattr(g, "scodoc_dept", "")
msg = f" ({dept}) {msg}"
except RuntimeError:
# Flask Working outside of application context.
pass
2022-09-07 13:06:26 +02:00
if current_app and not current_app.config["DEBUG"]:
2021-08-29 19:57:32 +02:00
current_app.logger.info(msg)
else:
sys.stdout.flush()
sys.stderr.write(
2022-09-07 13:06:26 +02:00
f"""[{time.strftime("%a %b %d %H:%M:%S %Y")}] scodoc: {msg}\n"""
2021-08-29 19:57:32 +02:00
)
sys.stderr.flush()
# Debug: log call stack
def log_call_stack():
log("Call stack:\n" + "\n".join(x.strip() for x in traceback.format_stack()[:-1]))
# Alarms by email:
def send_scodoc_alarm(subject, txt):
from app.scodoc import sco_preferences
from app import email
sender = sco_preferences.get_preference("email_from_addr")
email.send_email(subject, sender, ["exception@scodoc.org"], txt)
2021-08-13 00:34:58 +02:00
from app.models import Departement
from app.scodoc import notesdb as ndb, sco_preferences
from app.scodoc import sco_cache
2021-08-10 12:57:38 +02:00
2022-03-04 20:02:50 +01:00
def scodoc_flash_status_messages():
"""Should be called on each page: flash messages indicating specific ScoDoc status"""
email_test_mode_address = sco_preferences.get_preference("email_test_mode_address")
if email_test_mode_address:
flash(
f"Mode test: mails redirigés vers {email_test_mode_address}",
category="warning",
)