From ac5f298c7d9e8e4aca41d08c90021543868c94cd Mon Sep 17 00:00:00 2001 From: boyska Date: Tue, 15 Dec 2020 14:38:44 +0100 Subject: [PATCH] adapt to fastapi + reformat --- techrec/cli.py | 14 +- techrec/db.py | 10 +- techrec/forge.py | 8 +- techrec/maint.py | 3 +- techrec/server.py | 660 +++++++++++++++++++--------------------------- 5 files changed, 294 insertions(+), 401 deletions(-) diff --git a/techrec/cli.py b/techrec/cli.py index 49107c9..c190cb4 100644 --- a/techrec/cli.py +++ b/techrec/cli.py @@ -1,9 +1,12 @@ +import logging import os import os.path import sys -from argparse import ArgumentParser, Action +from argparse import Action, ArgumentParser from datetime import datetime -import logging + +from . import forge, maint, server +from .config_manager import get_config logging.basicConfig(stream=sys.stdout) logger = logging.getLogger("cli") @@ -11,11 +14,6 @@ logger = logging.getLogger("cli") CWD = os.getcwd() -from . import forge -from . import maint -from .config_manager import get_config -from . import server - def pre_check_permissions(): def is_writable(d): @@ -75,7 +73,7 @@ def common_pre(): logger.warn("Configuration file '%s' does not exist; skipping" % path) continue configs.append(path) - if getattr(sys, 'frozen', False): + if getattr(sys, "frozen", False): os.chdir(sys._MEIPASS) else: os.chdir(os.path.dirname(os.path.realpath(__file__))) diff --git a/techrec/db.py b/techrec/db.py index 1938e27..09ca1db 100644 --- a/techrec/db.py +++ b/techrec/db.py @@ -2,15 +2,15 @@ This module contains DB logic """ from __future__ import print_function + import logging +import sys from datetime import datetime, timedelta - -import sys - -from sqlalchemy import create_engine, Column, Integer, String, DateTime, inspect -from sqlalchemy.orm import sessionmaker +from sqlalchemy import (Column, DateTime, Integer, String, create_engine, + inspect) from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker from .config_manager import get_config diff --git a/techrec/forge.py b/techrec/forge.py index 3da65cd..a73b819 100644 --- a/techrec/forge.py +++ b/techrec/forge.py @@ -1,8 +1,8 @@ -from datetime import datetime, timedelta -from time import sleep -import os -from subprocess import Popen import logging +import os +from datetime import datetime, timedelta +from subprocess import Popen +from time import sleep from .config_manager import get_config diff --git a/techrec/maint.py b/techrec/maint.py index b7cd211..8f86ad9 100644 --- a/techrec/maint.py +++ b/techrec/maint.py @@ -1,6 +1,7 @@ from __future__ import print_function -import sys + import logging +import sys from sqlalchemy import inspect diff --git a/techrec/server.py b/techrec/server.py index c5f78e3..cada3d9 100644 --- a/techrec/server.py +++ b/techrec/server.py @@ -1,23 +1,26 @@ -import os -import sys -from datetime import datetime -import logging -from functools import partial -import unicodedata +#!/usr/bin/env python3 -from bottle import Bottle, request, static_file, redirect, abort, response -import bottle +import logging +import os +import unicodedata +from datetime import datetime +from functools import partial + +from fastapi import FastAPI, HTTPException, Response +from fastapi.responses import RedirectResponse + +from .cli import common_pre +from .config_manager import get_config +from .db import Rec, RecDB +from .forge import create_mp3 +from .processqueue import get_process_queue logger = logging.getLogger("server") -botlog = logging.getLogger("bottle") -botlog.setLevel(logging.INFO) -botlog.addHandler(logging.StreamHandler(sys.stdout)) -bottle._stderr = lambda x: botlog.info(x.strip()) -from .db import Rec, RecDB -from .processqueue import get_process_queue -from .forge import create_mp3 -from .config_manager import get_config +common_pre() +app = FastAPI() +pq = get_process_queue() +db = RecDB(get_config()["DB_URI"]) def date_read(s): @@ -35,384 +38,275 @@ def rec_sanitize(rec): return d -class DateApp(Bottle): - """ - This application will expose some date-related functions; it is intended to - be used when you need to know the server's time on the browser - """ - - def __init__(self): - Bottle.__init__(self) - self.route("/help", callback=self.help) - self.route("/date", callback=self.date) - self.route("/custom", callback=self.custom) - - def date(self): - n = datetime.now() - return { - "unix": n.strftime("%s"), - "isoformat": n.isoformat(), - "ctime": n.ctime(), - } - - def custom(self): - n = datetime.now() - if "strftime" not in request.query: - abort(400, 'Need argument "strftime"') - response.content_type = "text/plain" - return n.strftime(request.query["strftime"]) - - def help(self): - response.content_type = "text/plain" - return ( - "/date : get JSON dict containing multiple formats of now()\n" - + "/custom?strftime=FORMAT : get now().strftime(FORMAT)" - ) +@app.get("/date/date") +def date(): + n = datetime.now() + return {"unix": n.strftime("%s"), "isoformat": n.isoformat(), "ctime": n.ctime()} -class RecAPI(Bottle): - def __init__(self, app): - Bottle.__init__(self) - self._route() - self._app = app - self.db = RecDB(get_config()["DB_URI"]) - - def _route(self): - self.post("/create", callback=self.create) - self.post("/delete", callback=self.delete) - self.post("/update/", callback=self.update) - self.post("/generate", callback=self.generate) - self.get("/help", callback=self.help) - self.get("/", callback=self.help) - self.get("/get/search", callback=self.search) - self.get("/get/ongoing", callback=self.get_ongoing) - self.get("/get/archive", callback=self.get_archive) - self.get("/jobs", callback=self.running_jobs) - self.get("/jobs/", callback=self.check_job) - - def create(self): - req = dict(request.POST.decode().allitems()) - ret = {} - logger.debug("Create request %s " % req) - - now = datetime.now() - start = date_read(req["starttime"]) if "starttime" in req else now - name = req["name"] if "name" in req else u"" - end = date_read(req["endtime"]) if "endtime" in req else now - - rec = Rec(name=name, starttime=start, endtime=end) - ret = self.db.add(rec) - - return self.rec_msg( - "Nuova registrazione creata! (id:%d)" % ret.id, rec=rec_sanitize(rec) - ) - - def delete(self): - req = dict(request.POST.decode().allitems()) - logging.info("Server: request delete %s " % (req)) - if "id" not in req: - return self.rec_err("No valid ID") - - if self.db.delete(req["id"]): - return self.rec_msg("DELETE OK") - else: - return self.rec_err("DELETE error: %s" % (self.db.get_err())) - - def update(self, recid): - req = dict(request.POST.decode().allitems()) - - newrec = {} - now = datetime.now() - if "starttime" not in req: - newrec["starttime"] = now - else: - newrec["starttime"] = date_read(req["starttime"]) - if "endtime" not in req: - newrec["endtime"] = now - else: - newrec["endtime"] = date_read(req["endtime"]) - if "name" in req: - newrec["name"] = req["name"] - - try: - logger.info("prima di update") - result_rec = self.db.update(recid, newrec) - logger.info("dopo update") - except Exception as exc: - return self.rec_err("Errore Aggiornamento", exception=exc) - return self.rec_msg("Aggiornamento completato!", rec=rec_sanitize(result_rec)) - - def generate(self): - # prendiamo la rec in causa - recid = dict(request.POST.decode().allitems())["id"] - rec = self.db._search(_id=recid)[0] - if rec.filename is not None and os.path.exists(rec.filename): - return { - "status": "ready", - "message": "The file has already been generated at %s" % rec.filename, - "rec": rec, - } - if ( - get_config()["FORGE_MAX_DURATION"] > 0 - and (rec.endtime - rec.starttime).total_seconds() - > get_config()["FORGE_MAX_DURATION"] - ): - response.status = 400 - return { - "status": "error", - "message": "The requested recording is too long" - + " (%d seconds)" % (rec.endtime - rec.starttime).total_seconds(), - } - rec.filename = get_config()["AUDIO_OUTPUT_FORMAT"] % { - "time": rec.starttime.strftime( - "%y%m%d_%H%M" - ), # kept for retrocompatibility, should be dropped - "endtime": rec.endtime.strftime("%H%M"), - "startdt": rec.starttime.strftime("%y%m%d_%H%M"), - "enddt": rec.endtime.strftime("%y%m%d_%H%M"), - "name": "".join( - filter( - lambda c: c.isalpha(), - unicodedata.normalize("NFKD", rec.name) - .encode("ascii", "ignore") - .decode("ascii"), - ) - ), - } - self.db.get_session(rec).commit() - job_id = self._app.pq.submit( - create_mp3, - start=rec.starttime, - end=rec.endtime, - outfile=os.path.join(get_config()["AUDIO_OUTPUT"], rec.filename), - options={ - "title": rec.name, - "license_uri": get_config()["TAG_LICENSE_URI"], - "extra_tags": get_config()["TAG_EXTRA"], - }, - ) - logger.debug("SUBMITTED: %d" % job_id) - return self.rec_msg( - "Aggiornamento completato!", - job_id=job_id, - result="/output/" + rec.filename, - rec=rec_sanitize(rec), - ) - - def check_job(self, job_id): - try: - job = self._app.pq.check_job(job_id) - except ValueError: - abort(400, "job_id not valid") - - def ret(status): - return {"job_status": status, "job_id": job_id} - - if job is True: - return ret("DONE") - if job is False: - abort(404, "No such job has ever been spawned") - else: - if job.ready(): - try: - res = job.get() - return res - except Exception as exc: - r = ret("FAILED") - r["exception"] = str(exc) - import traceback - - tb = traceback.format_exc() - logger.warning(tb) - if get_config()["DEBUG"]: - r["exception"] = "%s: %s" % (str(exc), tb) - r["traceback"] = tb - - return r - return ret("WIP") - - def running_jobs(self): - res = {} - res["last_job_id"] = self._app.pq.last_job_id - res["running"] = self._app.pq.jobs.keys() - return res - - def search(self, args=None): - req = dict() - req.update(request.GET.allitems()) - logger.debug("Search request: %s" % (req)) - - values = self.db._search(**req) - from pprint import pprint - - logger.debug("Returned Values %s" % pprint([r.serialize() for r in values])) - - ret = {} - for rec in values: - ret[rec.id] = rec_sanitize(rec) - - logging.info("Return: %s" % ret) - return ret - - def get_ongoing(self): - return {rec.id: rec_sanitize(rec) for rec in self.db.get_ongoing()} - - def get_archive(self): - return {rec.id: rec_sanitize(rec) for rec in self.db.get_archive_recent()} - - # @route('/help') - def help(self): - return "

help

\ -

/get, /get/, /get/

\ -

Get Info about rec identified by ID

\ - \ -

/search, /search/, /search//

\ -

Search rec that match key/value (or get all)

\ - \ -

/delete/

\ -

Delete rec identified by ID

\ -

/update

\ -

Not implemented.

" - - # JSON UTILS - - def rec_msg(self, msg, status=True, **kwargs): - d = {"message": msg, "status": status} - d.update(kwargs) - return d - - def rec_err(self, msg, **kwargs): - return self.rec_msg(msg, status=False, **kwargs) +def TextResponse(text: str): + return Response(content=text, media_type="text/plain") -class RecServer: - def __init__(self): - self._app = Bottle() - self._app.pq = get_process_queue() - self._route() - - self.db = RecDB(get_config()["DB_URI"]) - - def _route(self): - # Static part of the site - self._app.route( - "/output/", - callback=lambda filepath: static_file( - filepath, root=get_config()["AUDIO_OUTPUT"], download=True - ), - ) - - self._app.route( - "/static/", - callback=lambda filepath: static_file( - filepath, root=get_config()["STATIC_FILES"] - ), - ) - self._app.route("/", callback=lambda: redirect("/new.html")) - self._app.route( - "/new.html", - callback=partial( - static_file, "new.html", root=get_config()["STATIC_PAGES"] - ), - ) - self._app.route( - "/old.html", - callback=partial( - static_file, "old.html", root=get_config()["STATIC_PAGES"] - ), - ) - self._app.route( - "/archive.html", - callback=partial( - static_file, "archive.html", root=get_config()["STATIC_PAGES"] - ), - ) +def abort(code, text): + raise HTTPException(status_code=code, detail=text) -class DebugAPI(Bottle): - """ - This application is useful for testing the webserver itself - """ - - def __init__(self): - Bottle.__init__(self) - self.route("/sleep/:milliseconds", callback=self.sleep) - self.route("/cpusleep/:howmuch", callback=self.cpusleep) - self.route("/big/:exponent", callback=self.big) - - def sleep(self, milliseconds): - import time - - time.sleep(int(milliseconds) / 1000.0) - return "ok" - - def cpusleep(self, howmuch): - out = "" - for i in xrange(int(howmuch) * (10 ** 3)): - if i % 11234 == 0: - out += "a" - return out - - def big(self, exponent): - """ - returns a 2**n -1 string - """ - for i in xrange(int(exponent)): - yield str(i) * (2 ** i) - - def help(self): - response.content_type = "text/plain" - return """ - /sleep/ : sleep, than say "ok" - /cpusleep/ : busysleep, than say "ok" - /big/ : returns a 2**n -1 byte content - """ +@app.get("/date/custom") +def custom(strftime: str = ""): + n = datetime.now() + if not strftime: + abort(400, 'Need argument "strftime"') + return TextResponse(n.strftime(strftime)) -class PasteLoggingServer(bottle.PasteServer): - def run(self, handler): # pragma: no cover - from paste import httpserver - from paste.translogger import TransLogger - - handler = TransLogger(handler, **self.options["translogger_opts"]) - del self.options["translogger_opts"] - httpserver.serve(handler, host=self.host, port=str(self.port), **self.options) - - -bottle.server_names["pastelog"] = PasteLoggingServer - - -def main_cmd(*args): - """meant to be called from argparse""" - c = RecServer() - c._app.mount("/date", DateApp()) - c._app.mount("/api", RecAPI(c._app)) - if get_config()["DEBUG"]: - c._app.mount("/debug", DebugAPI()) - - server = get_config()["WSGI_SERVER"] - if server == "pastelog": - from paste.translogger import TransLogger - - get_config()["WSGI_SERVER_OPTIONS"]["translogger_opts"] = get_config()[ - "TRANSLOGGER_OPTS" - ] - - c._app.run( - server=server, - host=get_config()["HOST"], - port=get_config()["PORT"], - debug=get_config()["DEBUG"], - quiet=True, # this is to hide access.log style messages - **get_config()["WSGI_SERVER_OPTIONS"] +@app.get("/date/help") +def help(): + return TextResponse( + "/date : get JSON dict containing multiple formats of now()\n" + + "/custom?strftime=FORMAT : get now().strftime(FORMAT)" ) -if __name__ == "__main__": - from cli import common_pre +@app.get("/api/create") +def create(): + req = dict(request.POST.decode().allitems()) + ret = {} + logger.debug("Create request %s " % req) - common_pre() - logger.warn("Usage of server.py is deprecated; use cli.py") - main_cmd() + now = datetime.now() + start = date_read(req["starttime"]) if "starttime" in req else now + name = req["name"] if "name" in req else u"" + end = date_read(req["endtime"]) if "endtime" in req else now + + rec = Rec(name=name, starttime=start, endtime=end) + ret = db.add(rec) + + return self.rec_msg( + "Nuova registrazione creata! (id:%d)" % ret.id, rec=rec_sanitize(rec) + ) + + +@app.get("/api/delete") +def delete(): + req = dict(request.POST.decode().allitems()) + logging.info("Server: request delete %s " % (req)) + if "id" not in req: + return self.rec_err("No valid ID") + + if db.delete(req["id"]): + return self.rec_msg("DELETE OK") + else: + return self.rec_err("DELETE error: %s" % (db.get_err())) + + +@app.get("/api/update/{recid}") +def update(recid: int): + req = dict(request.POST.decode().allitems()) + + newrec = {} + now = datetime.now() + if "starttime" not in req: + newrec["starttime"] = now + else: + newrec["starttime"] = date_read(req["starttime"]) + if "endtime" not in req: + newrec["endtime"] = now + else: + newrec["endtime"] = date_read(req["endtime"]) + if "name" in req: + newrec["name"] = req["name"] + + try: + logger.info("prima di update") + result_rec = db.update(recid, newrec) + logger.info("dopo update") + except Exception as exc: + return self.rec_err("Errore Aggiornamento", exception=exc) + return self.rec_msg("Aggiornamento completato!", rec=rec_sanitize(result_rec)) + + +@app.get("/api/generate") +def generate(): + # prendiamo la rec in causa + recid = dict(request.POST.decode().allitems())["id"] + rec = db._search(_id=recid)[0] + if rec.filename is not None and os.path.exists(rec.filename): + return { + "status": "ready", + "message": "The file has already been generated at %s" % rec.filename, + "rec": rec, + } + if ( + get_config()["FORGE_MAX_DURATION"] > 0 + and (rec.endtime - rec.starttime).total_seconds() + > get_config()["FORGE_MAX_DURATION"] + ): + response.status = 400 + return { + "status": "error", + "message": "The requested recording is too long" + + " (%d seconds)" % (rec.endtime - rec.starttime).total_seconds(), + } + rec.filename = get_config()["AUDIO_OUTPUT_FORMAT"] % { + "time": rec.starttime.strftime( + "%y%m%d_%H%M" + ), # kept for retrocompatibility, should be dropped + "endtime": rec.endtime.strftime("%H%M"), + "startdt": rec.starttime.strftime("%y%m%d_%H%M"), + "enddt": rec.endtime.strftime("%y%m%d_%H%M"), + "name": "".join( + filter( + lambda c: c.isalpha(), + unicodedata.normalize("NFKD", rec.name) + .encode("ascii", "ignore") + .decode("ascii"), + ) + ), + } + db.get_session(rec).commit() + job_id = self._app.pq.submit( + create_mp3, + start=rec.starttime, + end=rec.endtime, + outfile=os.path.join(get_config()["AUDIO_OUTPUT"], rec.filename), + options={ + "title": rec.name, + "license_uri": get_config()["TAG_LICENSE_URI"], + "extra_tags": get_config()["TAG_EXTRA"], + }, + ) + logger.debug("SUBMITTED: %d" % job_id) + return self.rec_msg( + "Aggiornamento completato!", + job_id=job_id, + result="/output/" + rec.filename, + rec=rec_sanitize(rec), + ) + + +@app.get("/api/jobs/{job_id}") +def check_job(job_id: int): + try: + job = pq.check_job(job_id) + except ValueError: + abort(400, "job_id not valid") + + def ret(status): + return {"job_status": status, "job_id": job_id} + + if job is True: + return ret("DONE") + if job is False: + abort(404, "No such job has ever been spawned") + else: + if job.ready(): + try: + res = job.get() + return res + except Exception as exc: + r = ret("FAILED") + r["exception"] = str(exc) + import traceback + + tb = traceback.format_exc() + logger.warning(tb) + if get_config()["DEBUG"]: + r["exception"] = "%s: %s" % (str(exc), tb) + r["traceback"] = tb + + return r + return ret("WIP") + + +@app.get("/api/jobs") +def running_jobs(): + res = {} + res["last_job_id"] = self._app.pq.last_job_id + res["running"] = self._app.pq.jobs.keys() + return res + + +@app.get("/api/get/search") +def search(args=None): + req = dict() + req.update(request.GET.allitems()) + logger.debug("Search request: %s" % (req)) + + values = db._search(**req) + from pprint import pprint + + logger.debug("Returned Values %s" % pprint([r.serialize() for r in values])) + + ret = {} + for rec in values: + ret[rec.id] = rec_sanitize(rec) + + logging.info("Return: %s" % ret) + return ret + + +@app.get("/api/get/ongoing") +def get_ongoing(): + return {rec.id: rec_sanitize(rec) for rec in db.get_ongoing()} + + +@app.get("/api/get/archive") +def get_archive(): + return {rec.id: rec_sanitize(rec) for rec in db.get_archive_recent()} + + +@app.get("/api/help") +@app.get("/api") +def help(): + return Response( + media_type="text/html", + content=""" +

help

+

/get, /get/, /get/{id}

+

Get Info about rec identified by ID

+ +

/search, /search/, /search/{key}/{value}

+

Search rec that match key/value (or get all)

+ +

/delete/{id}

+

Delete rec identified by ID

+

/update/{id}

+

Not implemented.

+ """, + ) + + +# JSON UTILS + + +def rec_msg(msg, status=True, **kwargs): + d = {"message": msg, "status": status} + d.update(kwargs) + return d + + +def rec_err(msg, **kwargs): + return self.rec_msg(msg, status=False, **kwargs) + + +# TODO: serve /output/ reading from get_config()['AUDIO_OUTPUT'] +# TODO: serve /static/ reading from get_config()['STATIC_FILES'] +# TODO: self._app.route("/", callback=lambda: redirect("/new.html")) +@app.get("/") +def home(): + return RedirectResponse("/new.html") + + +for page in ("new.html", "old.html", "archive.html"): + # route f"/{page}" → get_config()["STATIC_PAGES"] + page + pass + + +if __name__ == "__main__": + logger.warn("Usage of server.py is not supported anymore; use cli.py") + import sys + + sys.exit(1) # vim: set ts=4 sw=4 et ai ft=python: