From 71e4f6c9c7662a281b5abb41a5801259bd1352a6 Mon Sep 17 00:00:00 2001 From: Emmanuel Viennet Date: Sat, 18 Sep 2021 17:38:52 +0200 Subject: [PATCH] Portage for ScoDoc 9 --- .flaskenv | 1 + .gitignore | 173 +++++++++++++++++++++++++++++++++++++++++++++++ README.md | 16 +++++ app/__init__.py | 17 +++++ app/email.py | 30 ++++++++ app/routes.py | 161 +++++++++++++++++++++++++++++++++++++++++++ installmgr.py | 5 ++ requirements.txt | 16 +++++ 8 files changed, 419 insertions(+) create mode 100644 .flaskenv create mode 100644 .gitignore create mode 100644 README.md create mode 100644 app/__init__.py create mode 100644 app/email.py create mode 100644 app/routes.py create mode 100644 installmgr.py create mode 100644 requirements.txt diff --git a/.flaskenv b/.flaskenv new file mode 100644 index 0000000..71ffb6e --- /dev/null +++ b/.flaskenv @@ -0,0 +1 @@ +FLASK_APP=installmgr.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9a2e411 --- /dev/null +++ b/.gitignore @@ -0,0 +1,173 @@ +# ---> Emacs +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# ---> Python +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +envsco8/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# Mac OSX OS generated files +.DS_Store? +Thumbs.db +*.DS_Store + +# Subversion (protects when importing) +.svn + +# VS Code +.vscode/ +*.code-workspace + + +copy diff --git a/README.md b/README.md new file mode 100644 index 0000000..49aafbb --- /dev/null +++ b/README.md @@ -0,0 +1,16 @@ +# InstallMgr + +Mini app Flask remplaçant les CGI Sciript de `scodoc.iutv`. + +## API + + - last_stable_version : numéro de la dernière release "officielle" + + - upload_dump : réception (POST) d'un fichier de dump + + - version?mode=${mode}\&release=${SCODOC_RELEASE}\&sn=${SN}" + mode = install | upgrade + release = current client release + sn = client serial number + returns: serial number + diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..0360dc3 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1,17 @@ +import logging +import os + +from flask import Flask +from flask import g, current_app +from flask_mail import Mail + +mail = Mail() + + +def create_app(): + from app.routes import bp + + app = Flask(__name__) + mail.init_app(app) + app.register_blueprint(bp) + return app diff --git a/app/email.py b/app/email.py new file mode 100644 index 0000000..226429d --- /dev/null +++ b/app/email.py @@ -0,0 +1,30 @@ +# -*- coding: UTF-8 -* +from threading import Thread +from flask import current_app +from flask_mail import Message +from app import mail + + +def send_async_email(app, msg): + with app.app_context(): + mail.send(msg) + + +def send_email( + subject: str, sender: str, recipients: list, text_body: str, html_body="" +): + """ + Send an email + If html_body is specified, build a multipart message with HTML content, + else send a plain text email. + """ + msg = Message(subject, sender=sender, recipients=recipients) + msg.body = text_body + msg.html = html_body + send_message(msg) + + +def send_message(msg): + Thread( + target=send_async_email, args=(current_app._get_current_object(), msg) + ).start() diff --git a/app/routes.py b/app/routes.py new file mode 100644 index 0000000..77e3def --- /dev/null +++ b/app/routes.py @@ -0,0 +1,161 @@ +import json, datetime, fcntl, os, re, socket, subprocess, time + +from flask import request, abort +from flask import Blueprint +from app import email + +bp = Blueprint("routes", __name__) + +# -------------------------------------------------------------- +DIR = "/opt/installmgr/" +REPOSIT_DIR = "/opt/installmgr/incoming_dumps" +MAX_REPOSIT_SIZE = 100 * 20 * 1024 # kB (here, max 100 dumps of 20MB) + +ALERT_MAIL_FROM = "root@scodoc.iutv.univ-paris13.fr" +ALERT_MAIL_TO = "emmanuel.viennet@gmail.com" + +LOG_FILENAME = os.path.join(DIR, "upload-dump-errors.log") +UPLOAD_LOG_FILENAME = os.path.join(DIR, "upload-dump-log.json") +DEBUG = False # if false, don't publish error messages + + +@bp.route("/last_stable_version") +def last_stable_version(): + # LAST_RELEASE_TAG=$(curl "$GITEA_RELEASE_URL" | jq ".[].tag_name" | tr -d -c "0-9.\n" | sort --version-sort | tail -1) + return "9.0.30" + + +@bp.route("/upload_dump", methods=["POST"]) +def upload_dump(): + """ + Réception d'un fichier de dump uploadé + """ + log = open(LOG_FILENAME, "a") + now = datetime.datetime.now() + fulltime = now.isoformat() + # shorttime= now.replace(microsecond=0).isoformat() + remote_addr = request.remote_addr # client addr + log.write("{} request from {}\n".format(fulltime, remote_addr)) + # Avec seulement alphanum et tiret: + clean_deptname = re.sub(r"[^A-Za-z-]", "", request.form["dept_name"]) + the_file = request.files["file"] + filename = the_file.filename + data = the_file.file.read() + + try: + remote_host = socket.gethostbyaddr(remote_addr)[0] + except: + log.write("reverse DNS lookup failed for {}".format(remote_addr)) + remote_host = "" + + D = { + "dept_name": request.form["dept_name"], + "serial": request.form["serial"], + "sco_user": request.form["sco_user"], + "sent_by": request.form["sent_by"], + "sco_version": request.form["sco_version"], + "sco_subversion": request.form["sco_subversion"], + "dump_filename": fulltime + "_" + clean_deptname + ".gz", + "dump_size": len(data), + "remote_addr": remote_addr, + "remote_host": remote_host, + } + + log.write("received data ({} bytes)\n".format(D["dump_size"])) + json_descr = json.dumps(D, sort_keys=True, indent=4) + # --- Check disk space + + cur_size = int(subprocess.check_output(["du", "-skx", REPOSIT_DIR]).split("\t")[0]) + + if (cur_size + len(data) / 1024) > MAX_REPOSIT_SIZE: + # out of space ! + log.write( + "Out of space: cur_size={}kB, limit={}\n".format(cur_size, MAX_REPOSIT_SIZE) + ) + # Send alert + try: + email.send_email( + "[upload-dump] Out of space !", + ALERT_MAIL_FROM, + [ALERT_MAIL_TO], + "Out space !\nNew upload was canceled:\n" + json_descr, + ) + except: + log.write("exception while sending email !\n") + abort(507, "Insufficient Storage") + else: + log.write("writing dump to {}\n".format(D["dump_filename"])) + # dump: + f = open(os.path.join(REPOSIT_DIR, D["dump_filename"]), "wb") + f.write(data) + f.close() + uplog = open(UPLOAD_LOG_FILENAME, "a") + uplog.write(json_descr) + uplog.write("\n,\n") # separator + uplog.close() + + # Send notification + try: + log.write("sending notification to {}\n".format(ALERT_MAIL_TO)) + email.send_email( + f"[upload-dump] new dump {D['dept_name']} from {D['remote_addr']} ({D['remote_host']})", + ALERT_MAIL_FROM, + [ALERT_MAIL_TO], + "New upload:\n" + json_descr, + ) + except: + log.write("exception while sending email !\n") + return ("", 204) # ok empty response + + +# Lock for counter +class Lock: + def acquire(self): + self.f = open("lock", "w") + fcntl.flock(self.f.fileno(), fcntl.LOCK_EX) + + def release(self): + self.f.close() + + +def increment(): + L = Lock() + L.acquire() + try: + try: + val = int(open(DIR + "/counter").read()) + except: + val = 0 + val += 1 + open("counter", "w").write("%d" % val) + finally: + L.release() + return val + + +@bp.route("/version", methods=["GET"]) +def version(): + """ + echo -e "DATE\tIP\tSVN\tSERIAL\tOP" > installs.log; chown www-data installs.log + """ + mode = request.args.get("mode", "?") + sn = request.args.get("sn", "") # serial number + svn = request.args.get("svn", "") # installed subversion + commit = request.args.get("commit", "") # installed git commit + if mode == "install" or not sn: + serial = increment() + else: + serial = sn + + f = open(DIR + "installs.log", "a") + f.write( + "%s\t%s\t%s\t%s\t%s\n" + % ( + time.strftime("%Y-%m-%d %H:%M:%S"), + request.remote_addr, + svn or commit, + serial, + mode, + ) + ) + f.close() diff --git a/installmgr.py b/installmgr.py new file mode 100644 index 0000000..f302d2c --- /dev/null +++ b/installmgr.py @@ -0,0 +1,5 @@ +"""Application Flask: ScoDoc +""" +from app import create_app + +app = create_app() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..aab7e37 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,16 @@ +black==21.9b0 +blinker==1.4 +click==8.0.1 +Flask==2.0.1 +Flask-Mail==0.9.1 +itsdangerous==2.0.1 +Jinja2==3.0.1 +MarkupSafe==2.0.1 +mypy-extensions==0.4.3 +pathspec==0.9.0 +platformdirs==2.3.0 +python-dotenv==0.19.0 +regex==2021.8.28 +tomli==1.2.1 +typing-extensions==3.10.0.2 +Werkzeug==2.0.1