readme about testing

let's hope this works; installation should be much faster (wheel)

.gitignore

@@ -5,3 +5,5 @@ build/
 dist/
 rec/
 *.egg-info/
+/venv
+/docker/output/*

.gitlab-ci.yml

@@ -0,0 +1,21 @@
+image: python:3.7
+
+stages:
+  - static
+  - test
+
+mypy:
+  stage: static
+  before_script:
+    - pip install mypy
+  script:
+    - mypy techrec
+
+test:
+  stage: test
+  before_script:
+    - pip install pytest pytest-asyncio
+    - pip install -r requirements.txt
+    - mkdir techrec/output
+  script:
+    - pytest

Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.7
+
+ARG hostuid=1000
+ARG hostgid=1000
+
+ENV TECHREC_CONFIG=/src/techrec/docker/config.py
+ENV DEBIAN_FRONTEND=noninteractive
+
+WORKDIR /src
+COPY . /src/techrec
+
+RUN groupadd -g ${hostgid} techrec \
+    && useradd -g techrec -u ${hostuid} -m techrec \
+    && mkdir -p /src/techrec \
+    && mkdir -p /src/db \
+    && chown -R techrec:techrec /src \
+    && apt-get -qq update \
+    && apt-get install -qq -y ffmpeg \
+ && rm -rf /var/lib/apt/lists/*
+
+USER techrec
+
+
+RUN python -m venv ./venv \
+    && ./venv/bin/python -m pip install wheel \
+ && ./venv/bin/python -m pip install -e ./techrec
+
+VOLUME ["/src/db"]
+EXPOSE 8000
+
+ENTRYPOINT ["/src/venv/bin/techrec"]
+CMD ["-vv", "serve"]

Makefile

@@ -0,0 +1,61 @@
+DOCKER := docker
+DOCKERC := docker-compose
+PORT := 8000
+VENV := venv
+CONFIG := dev_config.py
+PY := python
+OWNER := ${USER}
+
+docker-build:
+	$(DOCKERC) build \
+		--no-cache \
+		--build-arg=hostgid=$(shell id -g) \
+		--build-arg=hostuid=$(shell id -u) \
+		--build-arg=audiogid=$(shell cat /etc/group | grep audio | awk -F: '{print $3}')
+
+docker-build-liquidsoap:
+	$(DOCKER) pull savonet/liquidsoap:main
+	$(DOCKERC) build \
+		--no-cache \
+		--build-arg=audiogid=$(shell cat /etc/group | grep audio | awk -F: '{print $3}') \
+		liquidsoap
+
+docker-build-techrec:
+	$(DOCKERC) build \
+		--no-cache \
+		--build-arg=hostgid=$(shell id -g ${OWNER}) \
+		--build-arg=hostuid=$(shell id -u ${OWNER}) \
+		techrec
+
+docker-stop:
+	$(DOCKERC) down -v
+
+docker-run:
+	$(DOCKERC) run --rm --service-ports techrec
+
+docker-shell-techrec:
+	$(eval CONTAINER = $(shell docker ps|grep techrec_run|awk '{print $$12}'))
+	$(DOCKER) exec -ti $(CONTAINER) bash
+
+docker-shell-storage:
+	$(DOCKERC) exec storage bash
+
+docker-shell-liquidsoap:
+	$(eval CONTAINER = $(shell docker ps|grep liquidsoap|awk '{print $$12}'))
+	$(DOCKER) exec -ti $(CONTAINER) bash
+
+docker-logs-storage:
+	$(DOCKERC) logs -f storage
+
+docker-logs-liquidsoap:
+	$(DOCKERC) logs -f liquidsoap
+
+local-install:
+	$(PY) -m venv $(VENV)
+	./$(VENV)/bin/pip install -e .
+
+local-serve:
+	env TECHREC_CONFIG=$(CONFIG) ./$(VENV)/bin/techrec -vv serve
+
+
+.PHONY: docker-build docker-build-liquidsoap docker-build-techrec docker-stop docker-run docker-shell-techrec docker-shell-storage docker-shell-liquidsoap docker-logs-storage docker-logs-liquidsoap local-install local-serve

README.md

@@ -1,7 +1,7 @@
 TechRec
 =======
 
-A Python2/Python3 web application that assist radio speakers in recording their shows.
+A Python3 web application that assist radio speakers in recording their shows.
 Meant to be simple to install and to maintain.
 
 It basically takes a directory with the continuous recording and create new
@@ -10,7 +10,7 @@ files "cutting/pasting" with ffmpeg.
 Features
 =========
 
-* little system dependencies: python and ffmpeg
+* little system dependencies: python3 and ffmpeg
 * The interface is extremely simple to use
 * Supports nested recording (ie: to record an interview inside of a whole
   show)
@@ -32,16 +32,25 @@ parts of them. This can boil down to something like
 ffmpeg -i concat:2014-20-01-00-00.mp3|2014-20-01-00-01.mp3 -acodec copy -ss 160 -t 1840 foo.mp3
 ```
 
+How to run
+===========
+
+```sh
+pip install .
+env TECHREC_CONFIG=yourconfig.py techrec serve
+```
+
 
 Implementation details
 ======================
 
-It is based on bottle, to get a minimal framework. Simple APIs are offered
-through it, and the static site uses them.
+It is based on [fastapi](https://fastapi.tiangolo.com/), a really nice
+framework. Simple APIs are offered through it, and the static site uses them
+through JS.
 
-Jobs are not dispatched using stuff like celery, but with a thin wrapper over
-`multiprocessing.Pool`; this is just to keep the installation as simple as
-possible.
+Jobs are not dispatched using stuff like celery, but just using
+[`BackgroundTasks`](https://fastapi.tiangolo.com/tutorial/background-tasks/),
+in order to keep the installation as simple as possible.
 
 The encoding part is delegated to `ffmpeg`, but the code is really modular so
 changing this is a breeze. To be quicker and avoid the quality issues related
@@ -51,7 +60,11 @@ have the same format.
 testing
 -----------
 
-unit tests can be run with `python setup.py test`
+```
+gitlab-runner exec docker test
+```
+
+(or, `pytest-3`, assuming you have a properly configured system)
 
 screenshots
 --------------

contrib/docker/Dockerfile

@@ -9,7 +9,7 @@ python-pip
 RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends
 virtualenvwrapper
 
-RUN git clone https://github.com/boyska/techrec.git /opt/techrec
+RUN git clone https://git.lattuga.net/techbloc/techrec.git /opt/techrec
 RUN virtualenv --python=python2 /opt/virtualenv
 RUN /opt/virtualenv/bin/pip install -r /opt/techrec/server/requirements.txt
 RUN mkdir /opt/db

docker-compose.yaml

@@ -0,0 +1,49 @@
+version: "3"
+
+services:
+  liquidsoap:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile.liquidsoap
+    volumes:
+      - ./docker/run.liq:/run.liq
+      - ./docker/run.sh:/run.sh
+      - rec:/rec
+    devices:
+      - /dev/snd:/dev/snd
+    entrypoint: /run.sh
+    depends_on:
+      - storageprepare
+
+  storage:
+    image: nginx
+    volumes:
+      - rec:/var/www/rec
+      - ./docker/storage.conf:/etc/nginx/conf.d/default.conf:ro
+    ports:
+      - 18080:80
+    depends_on:
+      - storageprepare
+
+  storageprepare:
+    image: bash
+    volumes:
+      - rec:/rec
+    command: chmod 777 /rec
+
+  techrec:
+    build: .
+    volumes:
+      - .:/src/techrec
+      - rec:/rec
+      - ./docker/output:/src/output
+      - db:/src/db
+    ports:
+      - 8000:8000
+    depends_on:
+      - liquidsoap
+      - storage
+
+volumes:
+  rec:
+  db:

docker/Dockerfile.liquidsoap

@@ -0,0 +1,10 @@
+FROM savonet/liquidsoap:main
+
+ENV audiogid=995
+
+USER root
+
+RUN groupadd -g ${audiogid} hostaudio \
+ && usermod -a -G hostaudio liquidsoap
+
+USER liquidsoap

docker/config.py

@@ -0,0 +1,9 @@
+DB_URI = "sqlite:////src/db/techrec.db"
+AUDIO_INPUT = "http://storage"
+# decomment this if you want to test with local audio source
+# AUDIO_INPUT = "/rec"
+AUDIO_OUTPUT = "/src/output"
+DEBUG = True
+HOST = "0.0.0.0"
+PORT = 8000
+FFMPEG_OPTIONS = ["-loglevel", "warning"]

docker/run.liq

@@ -0,0 +1,26 @@
+#!/usr/bin/liquidsoap
+
+settings.log.stdout.set(true);
+settings.log.file.set(false);
+settings.log.level.set(3);
+
+# settings.server.telnet.set(true);
+# settings.server.telnet.bind_addr.set("127.0.0.1");
+# settings.server.telnet.port.set(6666);
+
+rorinput = input.alsa(device="default", bufferize=true);
+#rorinput = input.pulseaudio( );
+
+# rorinput = insert_metadata(id="trx",rorinput);
+rorinput = rewrite_metadata([("artist","Radio OndaRossa")],rorinput);
+
+# ESCPOST
+output.file(
+  id="rorrec",
+  reopen_when={0m},
+  %mp3(bitrate=80, samplerate=44100, stereo=true,stereo_mode="joint_stereo"),
+  "/rec/%Y-%m/%d/rec-%Y-%m-%d-%H-%M-%S.mp3",
+  # %vorbis(quality=0.3, samplerate=44100, channels=2),
+  # "/rec/%Y-%m/%d/rec-%Y-%m-%d-%H-%M-%S.ogg",
+  rorinput
+);

docker/run.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -xueo pipefail
+
+FILEPATH="/rec/$(date +%Y-%m)/$(date +%d)/rec-$(date +%Y-%m-%d-%H)-00-00.mp3"
+mkdir -p $(dirname ${FILEPATH})
+if ! [[ -f ${FILEPATH} ]]; then
+    ffmpeg -f lavfi -i anullsrc=r=11025:cl=mono -t 3600 -acodec mp3 ${FILEPATH}
+fi
+
+/run.liq

docker/storage.conf

@@ -0,0 +1,9 @@
+server {
+    listen 80 default_server;
+    server_name storage;
+
+    location / {
+        root /var/www/rec;
+        autoindex on;
+    }
+}

requirements.txt

@@ -0,0 +1,10 @@
+aiofiles==0.6.0
+aiohttp==3.7.4
+click==7.1.2
+fastapi==0.62.0
+h11==0.11.0
+pydantic==1.7.3
+SQLAlchemy==1.4.25
+starlette==0.13.6
+typing-extensions==3.7.4.3
+uvicorn==0.13.1

server/pages/archive.html

@@ -1,102 +0,0 @@
-<!DOCTYPE html>
-<html>
-
-<head>
-	<title>TechREC</title>
-	<link rel="icon" href="/static/img/icon.ico" />
-	<link rel="stylesheet" type="text/css" href="/static/css/pure-min.css" />
-	<link rel="stylesheet" type="text/css" href="/static/css/pure-skin-porpora.css" />
-	<link rel="stylesheet" type="text/css" href="/static/css/jquery-ui.min.css" />
-	<link rel="stylesheet" type="text/css" href="/static/css/techrec.css">
-	<link rel="stylesheet" type="text/css" href="/static/css/font-awesome.css" />
-
-	<script src="/static/js/jquery-1.9.1.min.js"></script>
-	<script src="/static/js/jquery-ui.min.js"></script>
-	<script src="/static/js/jquery.ui.datepicker-it.min.js"></script>
-	<script src="/static/js/rec.js"></script>
-	<script>
-function delta(end, start) {
-	//end, start are unix timestamps
-	diff = parseInt(end, 10) - parseInt(start, 10); //diff is in seconds
-	msec = diff*1000;
-	var hh = Math.floor(msec / 1000 / 60 / 60);
-	msec -= hh * 1000 * 60 * 60;
-	var mm = Math.floor(msec / 1000 / 60);
-	msec -= mm * 1000 * 60;
-	var ss = Math.floor(msec / 1000);
-	msec -= ss * 1000;
-
-	if(hh === 0) {
-		if(mm === 0) {
-			return ss + 's';
-		}
-		return mm + 'min ' + ss + 's';
-	}
-	return hh + 'h ' + mm + 'm ' + ss + 's';
-}
-$(function() {
-		"use strict";
-		RecAPI.get_archive().success(function(archive) {
-			/* To get sorted traversal, we need to do an array containing keys */
-			var keys = [];
-			for(var prop in archive) {
-				keys.push(prop);
-			}
-			keys.sort(function(a,b) { return b - a; }); //descending
-
-			/* ok, now we can traverse the objects */
-			for(var i =0; i < keys.length; i++) {
-				var rec = archive[keys[i]];
-				console.log(rec);
-				var name = $('<td/>').text(rec.name);
-				var start = $('<td/>').text(config.date_read(
-						parseInt(rec.starttime, 10)).toLocaleString()
-					);
-				var duration = $('<td/>').text(delta(rec.endtime, rec.starttime));
-				var dl_text = $('<span/>').text(" Scarica").addClass('pure-hidden-phone');
-				var fn = $("<td/>").append($("<a/>").prop("href", "/output/" +
-						rec.filename).addClass("pure-button pure-button-small")
-					.html( $("<i/>").addClass("fa fa-download").css("color", "green"))
-					.append(dl_text));
-				var row = $('<tr/>').append(name).append(start).append(duration).append(fn);
-				$('#ongoing-recs-table tbody').append(row);
-			}
-			});
-		});
-	</script>
-</head>
-
-<body class="pure-skin-porpora">
-
-<div class="pure-menu pure-menu-open pure-menu-horizontal">
-	<a href="#" class="pure-menu-heading">TechRec</a>
-	<ul>
-		<li><a href="new.html">Diretta</a></li>
-		<li><a href="old.html">Vecchie</a></li>
-		<li class="pure-menu-selected"><a href="archive.html">Archivio</a></li>
-	</ul>
-</div>
-
-<h1>Registrazioni gi&agrave; completate</h1>
-<div id="rec-normal" class="pure-g-r">
-	<div class="pure-u-1-8"></div>
-	<div class="pure-u-3-4">
-		<table width="100%" class="pure-table pure-table-horizontal pure-table-striped"
-			id="ongoing-recs-table" style="margin-top: 3em;">
-			<tbody>
-			<tr>
-				<th>Nome</th>
-				<th>Inizio</th>
-				<th>Durata</th>
-				<th>File</th>
-				</tr>
-			</tbody>
-		</table>
-	</div>
-	<div class="pure-u-1-8"></div>
-</div>
-
-</body>
-</html>
-
-<!-- vim: set ts=2 sw=2 noet: -->

server/processqueue.py

@@ -1,80 +0,0 @@
-import multiprocessing
-
-
-class JobQueue(object):
-    def __init__(self):
-        self.pool = multiprocessing.Pool(processes=1)
-        self.last_job_id = 0
-        self.jobs = {}  # job_id: AsyncResult
-
-    def submit(self, function, *args, **kwargs):
-        self.last_job_id += 1
-        job_id = self.last_job_id
-
-        def clean_jobs(res):
-            """this callback will remove the job from the queue"""
-            del self.jobs[job_id]
-
-        self.jobs[job_id] = self.pool.apply_async(function, args, kwargs, clean_jobs)
-        return job_id
-
-    def check_job(self, job_id):
-        """
-        If the job is running, return the asyncResult.
-        If it has already completed, returns True.
-        If no such job_id exists at all, returns False
-        """
-        if job_id <= 0:
-            raise ValueError("non-valid job_id")
-        if self.last_job_id < job_id:
-            return False
-        if job_id in self.jobs:
-            return self.jobs[job_id]
-        return True
-
-    def join(self):
-        self.pool.close()
-        self.pool.join()
-        self.pool = None
-
-
-def simulate_long_job(recid=None, starttime=None, endtime=None, name="", filename=None):
-    from time import sleep
-
-    print("evviva " + name)
-    sleep(2)
-    print("lavoro su " + name)
-    sleep(2)
-    print("done su " + name)
-
-
-_queue = None
-
-
-def get_process_queue():
-    global _queue
-    if _queue is None:
-        _queue = JobQueue()
-    return _queue
-
-
-if __name__ == "__main__":
-    from datetime import datetime
-
-    n = datetime.now()
-
-    def sleep(n):
-        import time
-
-        print("Inizio %d" % n)
-        time.sleep(n)
-        print("Finisco %d" % n)
-        return n
-
-    get_process_queue().submit(sleep, 3)
-    get_process_queue().submit(sleep, 3)
-    get_process_queue().join()
-    print(get_process_queue().jobs)
-    delta = (datetime.now() - n).total_seconds()
-    print(delta)
-    assert 5 < delta < 7

server/requirements.txt

@@ -1,4 +0,0 @@
-Paste==1.7.5.1
-SQLAlchemy==0.8.3
-bottle==0.11.6
-wsgiref==0.1.2

server/server.py

@@ -1,418 +0,0 @@
-import os
-import sys
-from datetime import datetime
-import logging
-from functools import partial
-import unicodedata
-
-from bottle import Bottle, request, static_file, redirect, abort, response
-import bottle
-
-logger = logging.getLogger("server")
-botlog = logging.getLogger("bottle")
-botlog.setLevel(logging.INFO)
-botlog.addHandler(logging.StreamHandler(sys.stdout))
-bottle._stderr = lambda x: botlog.info(x.strip())
-
-from .db import Rec, RecDB
-from .processqueue import get_process_queue
-from .forge import create_mp3
-from .config_manager import get_config
-
-
-def date_read(s):
-    return datetime.fromtimestamp(int(s))
-
-
-def date_write(dt):
-    return dt.strftime("%s")
-
-
-def rec_sanitize(rec):
-    d = rec.serialize()
-    d["starttime"] = date_write(d["starttime"])
-    d["endtime"] = date_write(d["endtime"])
-    return d
-
-
-class DateApp(Bottle):
-    """
-    This application will expose some date-related functions; it is intended to
-    be used when you need to know the server's time on the browser
-    """
-
-    def __init__(self):
-        Bottle.__init__(self)
-        self.route("/help", callback=self.help)
-        self.route("/date", callback=self.date)
-        self.route("/custom", callback=self.custom)
-
-    def date(self):
-        n = datetime.now()
-        return {
-            "unix": n.strftime("%s"),
-            "isoformat": n.isoformat(),
-            "ctime": n.ctime(),
-        }
-
-    def custom(self):
-        n = datetime.now()
-        if "strftime" not in request.query:
-            abort(400, 'Need argument "strftime"')
-        response.content_type = "text/plain"
-        return n.strftime(request.query["strftime"])
-
-    def help(self):
-        response.content_type = "text/plain"
-        return (
-            "/date : get JSON dict containing multiple formats of now()\n"
-            + "/custom?strftime=FORMAT : get now().strftime(FORMAT)"
-        )
-
-
-class RecAPI(Bottle):
-    def __init__(self, app):
-        Bottle.__init__(self)
-        self._route()
-        self._app = app
-        self.db = RecDB(get_config()["DB_URI"])
-
-    def _route(self):
-        self.post("/create", callback=self.create)
-        self.post("/delete", callback=self.delete)
-        self.post("/update/<recid:int>", callback=self.update)
-        self.post("/generate", callback=self.generate)
-        self.get("/help", callback=self.help)
-        self.get("/", callback=self.help)
-        self.get("/get/search", callback=self.search)
-        self.get("/get/ongoing", callback=self.get_ongoing)
-        self.get("/get/archive", callback=self.get_archive)
-        self.get("/jobs", callback=self.running_jobs)
-        self.get("/jobs/<job_id:int>", callback=self.check_job)
-
-    def create(self):
-        req = dict(request.POST.decode().allitems())
-        ret = {}
-        logger.debug("Create request %s " % req)
-
-        now = datetime.now()
-        start = date_read(req["starttime"]) if "starttime" in req else now
-        name = req["name"] if "name" in req else u""
-        end = date_read(req["endtime"]) if "endtime" in req else now
-
-        rec = Rec(name=name, starttime=start, endtime=end)
-        ret = self.db.add(rec)
-
-        return self.rec_msg(
-            "Nuova registrazione creata! (id:%d)" % ret.id, rec=rec_sanitize(rec)
-        )
-
-    def delete(self):
-        req = dict(request.POST.decode().allitems())
-        logging.info("Server: request delete %s " % (req))
-        if "id" not in req:
-            return self.rec_err("No valid ID")
-
-        if self.db.delete(req["id"]):
-            return self.rec_msg("DELETE OK")
-        else:
-            return self.rec_err("DELETE error: %s" % (self.db.get_err()))
-
-    def update(self, recid):
-        req = dict(request.POST.decode().allitems())
-
-        newrec = {}
-        now = datetime.now()
-        if "starttime" not in req:
-            newrec["starttime"] = now
-        else:
-            newrec["starttime"] = date_read(req["starttime"])
-        if "endtime" not in req:
-            newrec["endtime"] = now
-        else:
-            newrec["endtime"] = date_read(req["endtime"])
-        if "name" in req:
-            newrec["name"] = req["name"]
-
-        try:
-            logger.info("prima di update")
-            result_rec = self.db.update(recid, newrec)
-            logger.info("dopo update")
-        except Exception as exc:
-            return self.rec_err("Errore Aggiornamento", exception=exc)
-        return self.rec_msg("Aggiornamento completato!", rec=rec_sanitize(result_rec))
-
-    def generate(self):
-        # prendiamo la rec in causa
-        recid = dict(request.POST.decode().allitems())["id"]
-        rec = self.db._search(_id=recid)[0]
-        if rec.filename is not None and os.path.exists(rec.filename):
-            return {
-                "status": "ready",
-                "message": "The file has already been generated at %s" % rec.filename,
-                "rec": rec,
-            }
-        if (
-            get_config()["FORGE_MAX_DURATION"] > 0
-            and (rec.endtime - rec.starttime).total_seconds()
-            > get_config()["FORGE_MAX_DURATION"]
-        ):
-            response.status = 400
-            return {
-                "status": "error",
-                "message": "The requested recording is too long"
-                + " (%d seconds)" % (rec.endtime - rec.starttime).total_seconds(),
-            }
-        rec.filename = get_config()["AUDIO_OUTPUT_FORMAT"] % {
-            "time": rec.starttime.strftime(
-                "%y%m%d_%H%M"
-            ),  # kept for retrocompatibility, should be dropped
-            "endtime": rec.endtime.strftime("%H%M"),
-            "startdt": rec.starttime.strftime("%y%m%d_%H%M"),
-            "enddt": rec.endtime.strftime("%y%m%d_%H%M"),
-            "name": "".join(
-                filter(
-                    lambda c: c.isalpha(),
-                    unicodedata.normalize("NFKD", rec.name)
-                    .encode("ascii", "ignore")
-                    .decode("ascii"),
-                )
-            ),
-        }
-        self.db.get_session(rec).commit()
-        job_id = self._app.pq.submit(
-            create_mp3,
-            start=rec.starttime,
-            end=rec.endtime,
-            outfile=os.path.join(get_config()["AUDIO_OUTPUT"], rec.filename),
-            options={
-                "title": rec.name,
-                "license_uri": get_config()["TAG_LICENSE_URI"],
-                "extra_tags": get_config()["TAG_EXTRA"],
-            },
-        )
-        logger.debug("SUBMITTED: %d" % job_id)
-        return self.rec_msg(
-            "Aggiornamento completato!",
-            job_id=job_id,
-            result="/output/" + rec.filename,
-            rec=rec_sanitize(rec),
-        )
-
-    def check_job(self, job_id):
-        try:
-            job = self._app.pq.check_job(job_id)
-        except ValueError:
-            abort(400, "job_id not valid")
-
-        def ret(status):
-            return {"job_status": status, "job_id": job_id}
-
-        if job is True:
-            return ret("DONE")
-        if job is False:
-            abort(404, "No such job has ever been spawned")
-        else:
-            if job.ready():
-                try:
-                    res = job.get()
-                    return res
-                except Exception as exc:
-                    r = ret("FAILED")
-                    r["exception"] = str(exc)
-                    import traceback
-
-                    tb = traceback.format_exc()
-                    logger.warning(tb)
-                    if get_config()["DEBUG"]:
-                        r["exception"] = "%s: %s" % (str(exc), tb)
-                        r["traceback"] = tb
-
-                    return r
-            return ret("WIP")
-
-    def running_jobs(self):
-        res = {}
-        res["last_job_id"] = self._app.pq.last_job_id
-        res["running"] = self._app.pq.jobs.keys()
-        return res
-
-    def search(self, args=None):
-        req = dict()
-        req.update(request.GET.allitems())
-        logger.debug("Search request: %s" % (req))
-
-        values = self.db._search(**req)
-        from pprint import pprint
-
-        logger.debug("Returned Values %s" % pprint([r.serialize() for r in values]))
-
-        ret = {}
-        for rec in values:
-            ret[rec.id] = rec_sanitize(rec)
-
-        logging.info("Return: %s" % ret)
-        return ret
-
-    def get_ongoing(self):
-        return {rec.id: rec_sanitize(rec) for rec in self.db.get_ongoing()}
-
-    def get_archive(self):
-        return {rec.id: rec_sanitize(rec) for rec in self.db.get_archive_recent()}
-
-    # @route('/help')
-    def help(self):
-        return "<h1>help</h1><hr/>\
-        <h2>/get, /get/, /get/<id> </h2>\
-        <h3>Get Info about rec identified by ID </h3>\
-        \
-        <h2>/search, /search/, /search/<key>/<value></h2>\
-        <h3>Search rec that match key/value (or get all)</h3>\
-        \
-        <h2>/delete/<id> </h2>\
-        <h3>Delete rec identified by ID </h3>\
-        <h2>/update </h2>\
-        <h3>Not implemented.</h3>"
-
-    # JSON UTILS
-
-    def rec_msg(self, msg, status=True, **kwargs):
-        d = {"message": msg, "status": status}
-        d.update(kwargs)
-        return d
-
-    def rec_err(self, msg, **kwargs):
-        return self.rec_msg(msg, status=False, **kwargs)
-
-
-class RecServer:
-    def __init__(self):
-        self._app = Bottle()
-        self._app.pq = get_process_queue()
-        self._route()
-
-        self.db = RecDB(get_config()["DB_URI"])
-
-    def _route(self):
-        # Static part of the site
-        self._app.route(
-            "/output/<filepath:path>",
-            callback=lambda filepath: static_file(
-                filepath, root=get_config()["AUDIO_OUTPUT"], download=True
-            ),
-        )
-
-        self._app.route(
-            "/static/<filepath:path>",
-            callback=lambda filepath: static_file(
-                filepath, root=get_config()["STATIC_FILES"]
-            ),
-        )
-        self._app.route("/", callback=lambda: redirect("/new.html"))
-        self._app.route(
-            "/new.html",
-            callback=partial(
-                static_file, "new.html", root=get_config()["STATIC_PAGES"]
-            ),
-        )
-        self._app.route(
-            "/old.html",
-            callback=partial(
-                static_file, "old.html", root=get_config()["STATIC_PAGES"]
-            ),
-        )
-        self._app.route(
-            "/archive.html",
-            callback=partial(
-                static_file, "archive.html", root=get_config()["STATIC_PAGES"]
-            ),
-        )
-
-
-class DebugAPI(Bottle):
-    """
-    This application is useful for testing the webserver itself
-    """
-
-    def __init__(self):
-        Bottle.__init__(self)
-        self.route("/sleep/:milliseconds", callback=self.sleep)
-        self.route("/cpusleep/:howmuch", callback=self.cpusleep)
-        self.route("/big/:exponent", callback=self.big)
-
-    def sleep(self, milliseconds):
-        import time
-
-        time.sleep(int(milliseconds) / 1000.0)
-        return "ok"
-
-    def cpusleep(self, howmuch):
-        out = ""
-        for i in xrange(int(howmuch) * (10 ** 3)):
-            if i % 11234 == 0:
-                out += "a"
-        return out
-
-    def big(self, exponent):
-        """
-        returns a 2**n -1 string
-        """
-        for i in xrange(int(exponent)):
-            yield str(i) * (2 ** i)
-
-    def help(self):
-        response.content_type = "text/plain"
-        return """
-    /sleep/<int:milliseconds> : sleep, than say "ok"
-    /cpusleep/<int:howmuch> : busysleep, than say "ok"
-    /big/<int:exponent> : returns a 2**n -1 byte content
-    """
-
-
-class PasteLoggingServer(bottle.PasteServer):
-    def run(self, handler):  # pragma: no cover
-        from paste import httpserver
-        from paste.translogger import TransLogger
-
-        handler = TransLogger(handler, **self.options["translogger_opts"])
-        del self.options["translogger_opts"]
-        httpserver.serve(handler, host=self.host, port=str(self.port), **self.options)
-
-
-bottle.server_names["pastelog"] = PasteLoggingServer
-
-
-def main_cmd(*args):
-    """meant to be called from argparse"""
-    c = RecServer()
-    c._app.mount("/date", DateApp())
-    c._app.mount("/api", RecAPI(c._app))
-    if get_config()["DEBUG"]:
-        c._app.mount("/debug", DebugAPI())
-
-    server = get_config()["WSGI_SERVER"]
-    if server == "pastelog":
-        from paste.translogger import TransLogger
-
-        get_config()["WSGI_SERVER_OPTIONS"]["translogger_opts"] = get_config()[
-            "TRANSLOGGER_OPTS"
-        ]
-
-    c._app.run(
-        server=server,
-        host=get_config()["HOST"],
-        port=get_config()["PORT"],
-        debug=get_config()["DEBUG"],
-        quiet=True,  # this is to hide access.log style messages
-        **get_config()["WSGI_SERVER_OPTIONS"]
-    )
-
-
-if __name__ == "__main__":
-    from cli import common_pre
-
-    common_pre()
-    logger.warn("Usage of server.py is deprecated; use cli.py")
-    main_cmd()
-
-# vim: set ts=4 sw=4 et ai ft=python:

setup.cfg

@@ -0,0 +1,8 @@
+[flake8]
+max-line-length=89
+ignore=D
+
+[mypy]
+show_error_codes = True
+python_version = 3.7
+pretty = True

setup.py

@@ -1,28 +1,25 @@
 #!/usr/bin/env python
 
-from setuptools import setup
+from distutils.core import setup
+
+with open("requirements.txt") as buf:
+    REQUIREMENTS = [line.strip() for line in buf if line.strip()]
 
 setup(
     name="techrec",
-    version="1.2.0",
-    description="A Python2 web application "
+    version="2.0.0a1.dev1",
+    description="A Python3 web application "
     "that assist radio speakers in recording their shows",
     long_description=open("README.md").read(),
     long_description_content_type="text/markdown",
     author="boyska",
     author_email="piuttosto@logorroici.org",
     packages=["techrec"],
-    package_dir={"techrec": "server"},
-    install_requires=["Paste~=3.2", "SQLAlchemy==0.8.3", "bottle~=0.12"],
-    classifiers=[
-        "Programming Language :: Python :: 2.7",
-        "Programming Language :: Python :: 3.7",
-    ],
+    package_dir={"techrec": "techrec"},
+    install_requires=REQUIREMENTS,
+    classifiers=["Programming Language :: Python :: 3.7"],
     entry_points={"console_scripts": ["techrec = techrec.cli:main"]},
     zip_safe=False,
     install_package_data=True,
     package_data={"techrec": ["static/**/*", "pages/*.html"]},
-    test_suite="nose.collector",
-    setup_requires=["nose>=1.0"],
-    tests_requires=["nose>=1.0"],
 )

techrec/cli.py

@@ -1,32 +1,44 @@
+import logging
 import os
 import os.path
 import sys
-from argparse import ArgumentParser, Action
+from argparse import Action, ArgumentParser
 from datetime import datetime
-import logging
+import urllib.request
+
+from . import forge, maint, server
+from .config_manager import get_config
 
-logging.basicConfig(stream=sys.stdout)
 logger = logging.getLogger("cli")
 
-
 CWD = os.getcwd()
+OK_CODES = [200, 301, 302]
 
-from . import forge
-from . import maint
-from .config_manager import get_config
-from . import server
+
+def is_writable(d):
+    return os.access(d, os.W_OK)
+
+
+def check_remote_store(url: str) -> None:
+    try:
+        with urllib.request.urlopen(url) as req:
+            if req.code not in OK_CODES:
+                logger.warn(f"Audio input {url} not responding")
+    except Exception as e:
+        logger.warn(f"Audio input {url} not accessible: {e}")
 
 
 def pre_check_permissions():
-    def is_writable(d):
-        return os.access(d, os.W_OK)
-
-    if is_writable(get_config()["AUDIO_INPUT"]):
-        yield "Audio input '%s' writable" % get_config()["AUDIO_INPUT"]
-    if not os.access(get_config()["AUDIO_INPUT"], os.R_OK):
-        yield "Audio input '%s' unreadable" % get_config()["AUDIO_INPUT"]
-        sys.exit(10)
-    if is_writable(os.getcwd()):
+    audio_input = get_config()["AUDIO_INPUT"]
+    if audio_input.startswith("http://") or audio_input.startswith("https://"):
+        check_remote_store(audio_input)
+    else:
+        if is_writable(audio_input):
+            yield "Audio input '%s' writable" % audio_input
+        if not os.access(audio_input, os.R_OK):
+            yield "Audio input '%s' unreadable" % audio_input
+            sys.exit(10)
+    if is_writable(CWD):
         yield "Code writable"
     if not is_writable(get_config()["AUDIO_OUTPUT"]):
         yield "Audio output '%s' not writable" % get_config()["AUDIO_OUTPUT"]
@@ -63,8 +75,14 @@ class DateTimeAction(Action):
         setattr(namespace, self.dest, parsed_val)
 
 
-def common_pre():
-    prechecks = [pre_check_user, pre_check_permissions, pre_check_ffmpeg]
+code_dir = os.path.dirname(os.path.realpath(__file__))
+
+
+def common_pre(nochecks=False):
+    if nochecks:
+        prechecks = []
+    else:
+        prechecks = [pre_check_user, pre_check_permissions, pre_check_ffmpeg]
     configs = ["default_config.py"]
     if "TECHREC_CONFIG" in os.environ:
         for conf in os.environ["TECHREC_CONFIG"].split(":"):
@@ -72,13 +90,14 @@ def common_pre():
                 continue
             path = os.path.realpath(conf)
             if not os.path.exists(path):
-                logger.warn("Configuration file '%s' does not exist; skipping" % path)
+                logger.warn(
+                    "Configuration file '%s' does not exist; skipping" % path)
                 continue
             configs.append(path)
-    if getattr(sys, 'frozen', False):
+    if getattr(sys, "frozen", False):
         os.chdir(sys._MEIPASS)
     else:
-        os.chdir(os.path.dirname(os.path.realpath(__file__)))
+        os.chdir(code_dir)
     for conf in configs:
         get_config().from_pyfile(conf)
 

techrec/db.py

@@ -2,15 +2,22 @@
 This module contains DB logic
 """
 from __future__ import print_function
+
 import logging
+import sys
 from datetime import datetime, timedelta
 
-
-import sys
-
-from sqlalchemy import create_engine, Column, Integer, String, DateTime, inspect
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy import (
+    Column,
+    DateTime,
+    Boolean,
+    Integer,
+    String,
+    create_engine,
+    inspect,
+)
 from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
 
 from .config_manager import get_config
 
@@ -28,6 +35,8 @@ class Rec(Base):
     starttime = Column(DateTime, nullable=True)
     endtime = Column(DateTime, nullable=True)
     filename = Column(String, nullable=True)
+    ready = Column(Boolean, default=False)
+    error = Column(String, nullable=True, default=None)
 
     def __init__(self, name="", starttime=None, endtime=None, filename=None):
         self.name = name
@@ -43,6 +52,7 @@ class Rec(Base):
             "starttime": self.starttime,
             "endtime": self.endtime,
             "filename": self.filename,
+            "ready": self.ready,
         }
 
     def __repr__(self):
@@ -56,6 +66,14 @@ class Rec(Base):
             contents += ",Filename: '%s'" % self.filename
         return "<Rec(%s)>" % contents
 
+    @property
+    def status(self) -> str:
+        if self.error is not None:
+            return 'ERROR'
+        if self.ready:
+            return 'DONE'
+        return 'WIP'
+
 
 class RecDB:
     def __init__(self, uri):
@@ -64,7 +82,8 @@ class RecDB:
         self.log = logging.getLogger(name=self.__class__.__name__)
 
         logging.getLogger("sqlalchemy.engine").setLevel(logging.FATAL)
-        logging.getLogger("sqlalchemy.engine.base.Engine").setLevel(logging.FATAL)
+        logging.getLogger(
+            "sqlalchemy.engine.base.Engine").setLevel(logging.FATAL)
         logging.getLogger("sqlalchemy.dialects").setLevel(logging.FATAL)
         logging.getLogger("sqlalchemy.pool").setLevel(logging.FATAL)
         logging.getLogger("sqlalchemy.orm").setLevel(logging.FATAL)
@@ -169,7 +188,7 @@ class RecDB:
         return query.filter(Rec.filename == None)
 
     def _query_saved(self, query=None):
-        """Still not saved"""
+        """saved, regardless of status"""
         if query is None:
             query = self.get_session().query(Rec)
         return query.filter(Rec.filename != None)

techrec/default_config.py

@@ -4,24 +4,18 @@ import sys
 
 HOST = "localhost"
 PORT = "8000"
-# pastelog is just "paste", but customized to accept logging options
-WSGI_SERVER = "pastelog"
-# these are pastelog-specific options for logging engine
-TRANSLOGGER_OPTS = {
-    "logger_name": "accesslog",
-    "set_logger_level": logging.WARNING,
-    "setup_console_handler": False,
-}
-WSGI_SERVER_OPTIONS = {}
 
 DEBUG = True
 DB_URI = "sqlite:///techrec.db"
 AUDIO_OUTPUT = "output/"
 AUDIO_INPUT = "rec/"
+AUDIO_INPUT_BASICAUTH = None  # Could be a ("user", "pass") tuple instead
 AUDIO_INPUT_FORMAT = "%Y-%m/%d/rec-%Y-%m-%d-%H-%M-%S.mp3"
 AUDIO_OUTPUT_FORMAT = "techrec-%(startdt)s-%(endtime)s-%(name)s.mp3"
 FORGE_TIMEOUT = 20
 FORGE_MAX_DURATION = 3600 * 5
+FORGE_VERIFY = False
+FORGE_VERIFY_THRESHOLD = 3
 FFMPEG_OUT_CODEC = ["-acodec", "copy"]
 FFMPEG_OPTIONS = ["-loglevel", "warning", "-n"]
 FFMPEG_PATH = "ffmpeg"
@@ -33,7 +27,7 @@ TAG_LICENSE_URI = None
 # defaults
 STATIC_FILES = "static/"
 STATIC_PAGES = "pages/"
-if getattr(sys, 'frozen', False):  # pyinstaller
+if getattr(sys, "frozen", False):  # pyinstaller
     STATIC_FILES = os.path.join(sys._MEIPASS, STATIC_FILES)
     STATIC_PAGES = os.path.join(sys._MEIPASS, STATIC_PAGES)
 else:

techrec/forge.py

@@ -1,31 +1,48 @@
-from datetime import datetime, timedelta
-from time import sleep
-import os
-from subprocess import Popen
+import asyncio
 import logging
+import tempfile
+import os
+from datetime import datetime, timedelta
+from subprocess import Popen
+from time import sleep
+from typing import Callable, Optional
 
 from .config_manager import get_config
+from .http_retriever import download
+
+logger = logging.getLogger("forge")
+Validator = Callable[[datetime, datetime, str], bool]
 
 
-def get_timefile_exact(time):
+async def get_timefile_exact(time) -> str:
     """
     time is of type `datetime`; it is not "rounded" to match the real file;
     that work is done in get_timefile(time)
     """
-    return os.path.join(
-        get_config()["AUDIO_INPUT"], time.strftime(get_config()["AUDIO_INPUT_FORMAT"])
+    repo = get_config()["AUDIO_INPUT"]
+    path = os.path.join(
+        repo, time.strftime(get_config()["AUDIO_INPUT_FORMAT"])
     )
+    if path.startswith("http://") or path.startswith("https://"):
+        logger.info(f"downloading: {path}")
+        local = await download(
+            path,
+            basic_auth=get_config()['AUDIO_INPUT_BASICAUTH'],
+        )
+        return local
+    return path
 
 
-def round_timefile(exact):
+def round_timefile(exact: datetime) -> datetime:
     """
     This will round the datetime, so to match the file organization structure
     """
     return datetime(exact.year, exact.month, exact.day, exact.hour)
 
 
-def get_timefile(exact):
-    return get_timefile_exact(round_timefile(exact))
+async def get_timefile(exact: datetime) -> str:
+    file = await get_timefile_exact(round_timefile(exact))
+    return file
 
 
 def get_files_and_intervals(start, end, rounder=round_timefile):
@@ -65,13 +82,16 @@ def mp3_join(named_intervals):
     for (filename, start_cut, end_cut) in named_intervals:
         # this happens only one time, and only at the first iteration
         if start_cut:
-            assert startskip is None
+            if startskip is not None:
+                raise Exception("error in first cut iteration")
             startskip = start_cut
-        # this happens only one time, and only at the first iteration
+        # this happens only one time, and only at the last iteration
         if end_cut:
-            assert endskip is None
+            if endskip is not None:
+                raise Exception("error in last iteration")
             endskip = end_cut
-        assert "|" not in filename
+        if "|" in filename:
+            raise Exception(f"'|' in {filename}")
         files.append(filename)
 
     cmdline = [ffmpeg, "-i", "concat:%s" % "|".join(files)]
@@ -85,16 +105,32 @@ def mp3_join(named_intervals):
     return cmdline
 
 
-def create_mp3(start, end, outfile, options={}, **kwargs):
-    intervals = [
-        (get_timefile(begin), start_cut, end_cut)
-        for begin, start_cut, end_cut in get_files_and_intervals(start, end)
-    ]
+async def create_mp3(
+    start: datetime,
+    end: datetime,
+    outfile: str,
+    options={},
+    validator: Optional[Validator] = None,
+    **kwargs,
+):
+    if validator is None:
+
+        def validator(s, e, f):
+            return True
+
+    intervals = []
+    for begin, start_cut, end_cut in get_files_and_intervals(start, end):
+        try:
+            filename = await get_timefile(begin)
+        except Exception as e:
+            raise ValueError("Error while retrieving file: %s" % e) from e
+        intervals.append((filename, start_cut, end_cut))
     if os.path.exists(outfile):
         raise OSError("file '%s' already exists" % outfile)
     for path, _s, _e in intervals:
         if not os.path.exists(path):
-            raise OSError("file '%s' does not exist; recording system broken?" % path)
+            raise OSError(
+                "file '%s' does not exist; recording system broken?" % path)
 
     # metadata date/time formatted according to
     # https://wiki.xiph.org/VorbisComment#Date_and_time
@@ -102,10 +138,11 @@ def create_mp3(start, end, outfile, options={}, **kwargs):
     if outfile.endswith(".mp3"):
         metadata["TRDC"] = start.replace(microsecond=0).isoformat()
         metadata["RECORDINGTIME"] = metadata["TRDC"]
-        metadata["ENCODINGTIME"] = datetime.now().replace(microsecond=0).isoformat()
+        metadata["ENCODINGTIME"] = datetime.now().replace(
+            microsecond=0).isoformat()
     else:
         metadata["DATE"] = start.replace(microsecond=0).isoformat()
-    metadata["ENCODER"] = "https://github.com/boyska/techrec"
+    metadata["ENCODER"] = "https://git.lattuga.net/techbloc/techrec"
     if "title" in options:
         metadata["TITLE"] = options["title"]
     if options.get("license_uri", None) is not None:
@@ -116,14 +153,28 @@ def create_mp3(start, end, outfile, options={}, **kwargs):
     metadata_list = []
     for tag, value in metadata.items():
         if "=" in tag:
-            logging.error('Received a tag with "=" inside, skipping')
+            logger.error('Received a tag with "=" inside, skipping')
             continue
         metadata_list.append("-metadata")
         metadata_list.append("%s=%s" % (tag, value))
 
-    p = Popen(
-        mp3_join(intervals) + metadata_list + get_config()["FFMPEG_OPTIONS"] + [outfile]
+    prefix, suffix = os.path.basename(outfile).split(".", 1)
+    tmp_file = tempfile.NamedTemporaryFile(
+        suffix=".%s" % suffix,
+        prefix="forge-%s" % prefix,
+        delete=False,
+        # This is needed to avoid errors with the rename across different mounts
+        dir=os.path.dirname(outfile),
     )
+    cmd = (
+        mp3_join(intervals)
+        + metadata_list
+        + ["-y"]
+        + get_config()["FFMPEG_OPTIONS"]
+        + [tmp_file.name]
+    )
+    logger.info("Running %s", " ".join(cmd))
+    p = Popen(cmd)
     if get_config()["FORGE_TIMEOUT"] == 0:
         p.wait()
     else:
@@ -137,12 +188,16 @@ def create_mp3(start, end, outfile, options={}, **kwargs):
     if p.returncode is None:
         os.kill(p.pid, 15)
         try:
-            os.remove(outfile)
-        except:
+            os.remove(tmp_file.name)
+        except Exception:
             pass
         raise Exception("timeout")  # TODO: make a specific TimeoutError
     if p.returncode != 0:
         raise OSError("return code was %d" % p.returncode)
+    if not validator(start, end, tmp_file.name):
+        os.unlink(tmp_file.name)
+        return False
+    os.rename(tmp_file.name, outfile)
     return True
 
 
@@ -150,4 +205,4 @@ def main_cmd(options):
     log = logging.getLogger("forge_main")
     outfile = os.path.abspath(os.path.join(options.cwd, options.outfile))
     log.debug("will forge an mp3 into %s" % (outfile))
-    create_mp3(options.starttime, options.endtime, outfile)
+    asyncio.run(create_mp3(options.starttime, options.endtime, outfile))

techrec/http_retriever.py

@@ -0,0 +1,52 @@
+# -*- encoding: utf-8 -*-
+import os
+from typing import Optional, Tuple
+from tempfile import mkdtemp
+from logging import getLogger
+
+import aiohttp  # type: ignore
+
+CHUNK_SIZE = 2 ** 12
+
+log = getLogger("http")
+
+
+async def download(
+    remote: str,
+    staging: Optional[str] = None,
+    basic_auth: Optional[Tuple[str, str]] = None,
+) -> str:
+    """
+    This will download to AUDIO_STAGING the remote file and return the local
+    path of the downloaded file
+    """
+    _, filename = os.path.split(remote)
+    if staging:
+        base = staging
+    else:
+        # if no staging is specified, and you want to clean the storage
+        # used by techrec: rm -rf /tmp/techrec*
+        base = mkdtemp(prefix="techrec-", dir="/tmp")
+    local = os.path.join(base, filename)
+
+    session_args = {}
+    if basic_auth is not None:
+        session_args["auth"] = aiohttp.BasicAuth(
+            login=basic_auth[0], password=basic_auth[1], encoding="utf-8"
+        )
+
+    log.debug("Downloading %s with %s options", remote, ",".join(session_args.keys()))
+    async with aiohttp.ClientSession(**session_args) as session:
+        async with session.get(remote) as resp:
+            if resp.status != 200:
+                raise ValueError(
+                    "Could not download %s: error %d" % (remote, resp.status)
+                )
+            with open(local, "wb") as f:
+                while True:
+                    chunk = await resp.content.read(CHUNK_SIZE)
+                    if not chunk:
+                        break
+                    f.write(chunk)
+    log.debug("Downloading %s complete", remote)
+    return local

techrec/maint.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
-import sys
+
 import logging
+import sys
 
 from sqlalchemy import inspect
 

techrec/pages/archive.html

@@ -0,0 +1,53 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+	<title>TechREC</title>
+	<link rel="icon" href="/static/img/icon.ico" />
+	<link rel="stylesheet" type="text/css" href="/static/css/pure-min.css" />
+	<link rel="stylesheet" type="text/css" href="/static/css/pure-skin-porpora.css" />
+	<link rel="stylesheet" type="text/css" href="/static/css/jquery-ui.min.css" />
+	<link rel="stylesheet" type="text/css" href="/static/css/techrec.css">
+	<link rel="stylesheet" type="text/css" href="/static/css/font-awesome.css" />
+
+	<script src="/static/js/jquery-1.9.1.min.js"></script>
+	<script src="/static/js/jquery-ui.min.js"></script>
+	<script src="/static/js/jquery.ui.datepicker-it.min.js"></script>
+	<script src="/static/js/rec.js"></script>
+	<script src="/static/js/archive.js"></script>
+</head>
+
+<body class="pure-skin-porpora">
+
+<div class="pure-menu pure-menu-open pure-menu-horizontal">
+	<a href="#" class="pure-menu-heading">TechRec</a>
+	<ul>
+		<li><a href="new.html">Diretta</a></li>
+		<li><a href="old.html">Vecchie</a></li>
+		<li class="pure-menu-selected"><a href="archive.html">Archivio</a></li>
+	</ul>
+</div>
+
+<h1>Registrazioni gi&agrave; completate</h1>
+<div id="rec-normal" class="pure-g-r">
+	<div class="pure-u-1-8"></div>
+	<div class="pure-u-3-4">
+		<table width="100%" class="pure-table pure-table-horizontal pure-table-striped"
+			id="ongoing-recs-table" style="margin-top: 3em;">
+			<tbody>
+			<tr>
+				<th>Nome</th>
+				<th>Inizio</th>
+				<th>Durata</th>
+				<th>File</th>
+				</tr>
+			</tbody>
+		</table>
+	</div>
+	<div class="pure-u-1-8"></div>
+</div>
+
+</body>
+</html>
+
+<!-- vim: set ts=2 sw=2 noet: -->

techrec/requirements.txt

@@ -0,0 +1,8 @@
+SQLAlchemy==0.8.3
+click==7.1.2
+fastapi==0.62.0
+h11==0.11.0
+pydantic==1.7.3
+starlette==0.13.6
+typing-extensions==3.7.4.3
+uvicorn==0.13.1

techrec/server.py

@@ -0,0 +1,380 @@
+#!/usr/bin/env python3
+
+import sys
+import logging
+import time
+import os
+import unicodedata
+from datetime import datetime
+from typing import Optional
+from subprocess import check_output
+
+from fastapi import FastAPI, HTTPException, Request, Response, BackgroundTasks
+from fastapi.responses import FileResponse, RedirectResponse, JSONResponse
+from fastapi.staticfiles import StaticFiles
+from pydantic import BaseModel, Field
+
+from .cli import common_pre
+from .config_manager import get_config
+from .db import Rec, RecDB
+from .forge import create_mp3, Validator
+
+logger = logging.getLogger("server")
+
+common_pre(nochecks=('pytest' in sys.argv[0]))
+app = FastAPI()
+db = None
+
+
+def date_read(s):
+    return datetime.fromtimestamp(int(s))
+
+
+def date_write(dt):
+    return dt.strftime("%s")
+
+
+def rec_sanitize(rec):
+    d = rec.serialize()
+    d["starttime"] = date_write(d["starttime"])
+    d["endtime"] = date_write(d["endtime"])
+    return d
+
+
+@app.on_event("startup")
+async def startup_event():
+    global db
+    common_pre()
+    if get_config()["DEBUG"]:
+        logging.basicConfig(level=logging.DEBUG)
+    db = RecDB(get_config()["DB_URI"])
+
+
+@app.get("/date/date")
+def date():
+    n = datetime.now()
+    return {"unix": n.strftime("%s"), "isoformat": n.isoformat(), "ctime": n.ctime()}
+
+
+def TextResponse(text: str):
+    return Response(content=text, media_type="text/plain")
+
+
+def abort(code, text):
+    raise HTTPException(status_code=code, detail=text)
+
+
+@app.get("/date/custom")
+def custom(strftime: str = ""):
+    n = datetime.now()
+    if not strftime:
+        abort(400, 'Need argument "strftime"')
+    return TextResponse(n.strftime(strftime))
+
+
+@app.get("/date/help")
+def help():
+    return TextResponse(
+        "/date : get JSON dict containing multiple formats of now()\n"
+        + "/custom?strftime=FORMAT : get now().strftime(FORMAT)"
+    )
+
+
+class CreateInfo(BaseModel):
+    starttime: Optional[str] = None
+    endtime: Optional[str] = None
+    name: str = ""
+
+
+@app.post("/api/create")
+async def create(req: CreateInfo = None):
+    ret = {}
+    logger.debug("Create request %s " % req)
+
+    if req is None:
+        req = CreateInfo()
+    now = datetime.now()
+    start = date_read(req.starttime) if req.starttime is not None else now
+    name = req.name
+    end = date_read(req.endtime) if req.endtime is not None else now
+
+    rec = Rec(name=name, starttime=start, endtime=end)
+    ret = db.add(rec)
+
+    return rec_msg(
+        "Nuova registrazione creata! (id:%d)" % ret.id, rec=rec_sanitize(rec)
+    )
+
+
+class DeleteInfo(BaseModel):
+    id: int
+
+
+@app.post("/api/delete")
+def delete(req: DeleteInfo):
+    if db.delete(req.id):
+        return rec_msg("DELETE OK")
+    else:
+        return rec_err("DELETE error: %s" % (db.get_err()))
+
+
+def timefield_factory():
+    return int(time.time())
+
+
+TimeField = Field(default_factory=timefield_factory)
+
+
+class UpdateInfo(BaseModel):
+    name: str = ""
+    starttime: int = Field(default_factory=timefield_factory)
+    endtime: int = Field(default_factory=timefield_factory)
+    filename: Optional[str] = None
+
+
+@app.post("/api/update/{recid}")
+async def update(recid: int, req: UpdateInfo):
+    newrec = {}
+    newrec["starttime"] = date_read(req.starttime)
+    newrec["endtime"] = date_read(req.endtime)
+    if req.name:
+        newrec["name"] = req.name
+
+    try:
+        logger.info("prima di update")
+        result_rec = db.update(recid, newrec)
+        logger.info("dopo update")
+    except Exception as exc:
+        return rec_err("Errore Aggiornamento", exception=exc)
+    return rec_msg("Aggiornamento completato!", rec=rec_sanitize(result_rec))
+
+
+class GenerateInfo(BaseModel):
+    id: int
+
+
+class GenerateResponse(BaseModel):
+    status: str
+    message: str
+
+
+@app.post("/api/generate/{recid}")
+async def generate(recid: int, response: Response, background_tasks: BackgroundTasks):
+    # prendiamo la rec in causa
+    rec = db._search(_id=recid)[0]
+    if rec.ready:
+        return {
+            "status": "ready",
+            "message": "The file has already been generated at %s" % rec.filename,
+            "rec": rec,
+        }
+    if (
+        get_config()["FORGE_MAX_DURATION"] > 0
+        and (rec.endtime - rec.starttime).total_seconds()
+        > get_config()["FORGE_MAX_DURATION"]
+    ):
+        return JSONResponse(
+            status_code=400,
+            status="error",
+            message="The requested recording is too long"
+            + " (%d seconds)" % (rec.endtime - rec.starttime).total_seconds(),
+        )
+    rec.filename = get_config()["AUDIO_OUTPUT_FORMAT"] % {
+        "time": rec.starttime.strftime(
+            "%y%m%d_%H%M"
+        ),  # kept for retrocompatibility, should be dropped
+        "endtime": rec.endtime.strftime("%H%M"),
+        "startdt": rec.starttime.strftime("%y%m%d_%H%M"),
+        "enddt": rec.endtime.strftime("%y%m%d_%H%M"),
+        "name": "".join(
+            filter(
+                lambda c: c.isalpha(),
+                unicodedata.normalize("NFKD", rec.name)
+                .encode("ascii", "ignore")
+                .decode("ascii"),
+            )
+        ),
+    }
+    db.get_session(rec).commit()
+    background_tasks.add_task(
+        generate_mp3,
+        db_id=recid,
+        start=rec.starttime,
+        end=rec.endtime,
+        outfile=os.path.join(get_config()["AUDIO_OUTPUT"], rec.filename),
+        options={
+            "title": rec.name,
+            "license_uri": get_config()["TAG_LICENSE_URI"],
+            "extra_tags": get_config()["TAG_EXTRA"],
+        },
+    )
+    logger.debug("SUBMITTED: %d" % recid)
+    return rec_msg(
+        "Aggiornamento completato!",
+        job_id=rec.id,
+        result="/output/" + rec.filename,
+        rec=rec_sanitize(rec),
+    )
+
+
+def get_duration(fname) -> float:
+    lineout = check_output(
+        [
+            "ffprobe",
+            "-v",
+            "error",
+            "-show_entries",
+            "format=duration",
+            "-i",
+            fname,
+        ]
+    ).split(b"\n")
+    duration = next(l for l in lineout if l.startswith(b"duration="))
+    value = duration.split(b"=")[1]
+    return float(value)
+
+
+def get_validator(expected_duration_s: float, error_threshold_s: float) -> Validator:
+    def validator(start, end, fpath):
+        try:
+            duration = get_duration(fpath)
+        except Exception as exc:
+            logger.exception("Error determining duration of %s", fpath)
+            return False
+        logger.debug(
+            "expect %s to be %.1f±%.1fs, is %.1f",
+            fpath,
+            expected_duration_s,
+            error_threshold_s,
+            duration,
+        )
+        if duration > expected_duration_s + error_threshold_s:
+            return False
+        if duration < expected_duration_s - error_threshold_s:
+            return False
+        return True
+
+    return validator
+
+
+async def generate_mp3(db_id: int, **kwargs):
+    """creates and mark it as ready in the db"""
+    if get_config()["FORGE_VERIFY"]:
+        validator = get_validator(
+            (kwargs["end"] - kwargs["start"]).total_seconds(),
+            get_config()["FORGE_VERIFY_THRESHOLD"],
+        )
+        retries = 10
+    else:
+        validator = None
+        retries = 1
+
+    for i in range(retries):
+        try:
+            result = await create_mp3(validator=validator, **kwargs)
+        except Exception as exc:
+            logger.error("Error creating audio for %d -> %s", db_id, str(exc))
+            rec = db._search(_id=db_id)[0]
+            rec.error = str(exc)
+            db.get_session(rec).commit()
+            return False
+        logger.debug("Create mp3 for %d -> %s", db_id, result)
+        if result:
+            break
+        elif i < retries - 1:
+            logger.debug("waiting %d", i + 1)
+            time.sleep(i + 1)  # waiting time increases at each retry
+    else:
+        logger.warning("Could not create mp3 for %d: validation failed", db_id)
+        return False
+
+    rec = db._search(_id=db_id)[0]
+    rec.ready = True
+    db.get_session(rec).commit()
+    return True
+
+
+@app.get("/api/ready/{recid}")
+def check_job(recid: int):
+    rec = db._search(_id=recid)[0]
+
+    out = {"job_id": recid, "job_status": rec.status}
+
+    return out
+
+
+@app.get("/api/get/ongoing")
+def get_ongoing():
+    return {rec.id: rec_sanitize(rec) for rec in db.get_ongoing()}
+
+
+@app.get("/api/get/archive")
+def get_archive():
+    return {rec.id: rec_sanitize(rec) for rec in db.get_archive_recent()}
+
+
+@app.get("/api/help")
+@app.get("/api")
+def api_help():
+    return Response(
+        media_type="text/html",
+        content="""
+<h1>help</h1><hr/>
+<h2>/get, /get/, /get/{id} </h2>
+<h3>Get Info about rec identified by ID </h3>
+
+<h2>/search, /search/, /search/{key}/{value}</h2>
+<h3>Search rec that match key/value (or get all)</h3>
+
+<h2>/delete/{id} </h2>
+<h3>Delete rec identified by ID </h3>
+<h2>/update/{id} </h2>
+<h3>Not implemented.</h3>
+        """,
+    )
+
+
+# JSON UTILS
+
+
+def rec_msg(msg, status=True, **kwargs):
+    d = {"message": msg, "status": status}
+    d.update(kwargs)
+    return d
+
+
+def rec_err(msg, **kwargs):
+    return rec_msg(msg, status=False, **kwargs)
+
+
+app.mount("/output", StaticFiles(directory=get_config()["AUDIO_OUTPUT"]))
+app.mount("/static", StaticFiles(directory=get_config()["STATIC_FILES"]))
+
+
+@app.get("/")
+def home():
+    return RedirectResponse("/new.html")
+
+
+@app.route("/new.html")
+@app.route("/old.html")
+@app.route("/archive.html")
+def serve_pages(request: Request):
+    page = request.url.path[1:]
+    fpath = os.path.join(get_config()["STATIC_PAGES"], page)
+    return FileResponse(fpath)
+
+
+def main_cmd(options):
+    import uvicorn
+
+    uvicorn.run(app, host=get_config()["HOST"], port=int(get_config()["PORT"]))
+
+
+if __name__ == "__main__":
+    logger.warn("Usage of server.py is not supported anymore; use cli.py")
+    import sys
+
+    sys.exit(1)
+
+# vim: set ts=4 sw=4 et ai ft=python:

techrec/static/js/archive.js

@@ -0,0 +1,55 @@
+function delta(end, start) {
+	//end, start are unix timestamps
+	diff = parseInt(end, 10) - parseInt(start, 10); //diff is in seconds
+	msec = diff*1000;
+	var hh = Math.floor(msec / 1000 / 60 / 60);
+	msec -= hh * 1000 * 60 * 60;
+	var mm = Math.floor(msec / 1000 / 60);
+	msec -= mm * 1000 * 60;
+	var ss = Math.floor(msec / 1000);
+	msec -= ss * 1000;
+
+	if(hh === 0) {
+		if(mm === 0) {
+			return ss + 's';
+		}
+		return mm + 'min ' + ss + 's';
+	}
+	return hh + 'h ' + mm + 'm ' + ss + 's';
+}
+$(function() {
+		"use strict";
+		RecAPI.get_archive().success(function(archive) {
+			/* To get sorted traversal, we need to do an array containing keys */
+			var keys = [];
+			for(var prop in archive) {
+				keys.push(prop);
+			}
+			keys.sort(function(a,b) { return b - a; }); //descending
+
+			/* ok, now we can traverse the objects */
+			for(var i =0; i < keys.length; i++) {
+				var rec = archive[keys[i]];
+				console.log(rec);
+				var name = $('<td/>').text(rec.name);
+				var start = $('<td/>').text(config.date_read(
+						parseInt(rec.starttime, 10)).toLocaleString()
+					);
+				var duration = $('<td/>').text(delta(rec.endtime, rec.starttime));
+				var dl_text = $('<span/>').text(" Scarica").addClass('pure-hidden-phone');
+				var fn = $("<td/>")
+				if(rec.ready) {
+					fn.append($("<a/>").prop("href", "/output/" + rec.filename)
+						.addClass("pure-button pure-button-small")
+						.html( $("<i/>").addClass("fa fa-download").css("color", "green"))
+						.append(dl_text));
+				} else {
+					fn.html("<small>File not found</small>")
+				}
+				var row = $('<tr/>').append(name).append(start).append(duration).append(fn);
+				row.data('id', rec.id)
+				$('#ongoing-recs-table tbody').append(row);
+			}
+			});
+		});
+

techrec/static/js/rec.js

@@ -19,31 +19,46 @@ var RecAPI = {
   create: function () {
     return $.ajax('/api/create', {
       method: 'POST',
+      contentType: 'application/json',
+      data: "{}",
       dataType: 'json'
     })
   },
   stop: function (rec) {
-    return $.post('/api/update/' + rec.id, {
-      starttime: rec.starttime
-    })
+    return $.ajax('/api/update/' + rec.id,
+	    {
+		    method: 'POST',
+		    contentType: 'application/json',
+		    data: JSON.stringify({
+			    starttime: parseInt(rec.starttime, 10)
+		    })
+	    })
   },
   update: function (id, data) {
-    return $.post('/api/update/' + id, data)
+    return $.ajax(
+      '/api/update/' + data.id, {
+        method: 'POST',
+        dataType: 'json',
+        contentType: 'application/json',
+        data: JSON.stringify(data)
+      })
   },
   fullcreate: function (name, start, end) {
     return $.ajax(
       '/api/create', {
         method: 'POST',
         dataType: 'json',
-        data: { name: name,
+        contentType: 'application/json',
+        data: JSON.stringify({ name: name,
           starttime: config.date_write(start),
           endtime: config.date_write(end)
-        }
+        })
       })
   },
   generate: function (rec) {
-    return $.post('/api/generate', {
-      id: rec.id
+    return $.ajax('/api/generate/' + rec.id, {
+        method: 'POST',
+        dataType: 'json',
     })
   },
   get_archive: function () {
@@ -55,7 +70,7 @@ var RecAPI = {
 }
 
 function poll_job (job_id, callback) {
-  $.getJSON('/api/jobs/' + job_id)
+  $.getJSON('/api/ready/' + job_id)
   .done(function (data) {
     if (data.job_status !== 'WIP') {
       console.log('polling completed for job[' + job_id + ']', data)

techrec/test_forge.py

@@ -1,6 +1,7 @@
 from datetime import datetime, timedelta
 
-from nose.tools import raises, eq_
+import pytest
+from pytest import raises
 
 from .forge import (
     get_files_and_intervals,
@@ -21,6 +22,10 @@ get_config()["FFMPEG_PATH"] = "ffmpeg"
 get_config()["FFMPEG_OUT_CODEC"] = ["-acodec", "copy"]
 
 
+def eq_(a, b):
+    assert a == b, "%r != %r" % (a, b)
+
+
 def minutes(n):
     return timedelta(minutes=n)
 
@@ -32,8 +37,9 @@ def seconds(n):
 # timefile
 
 
-def test_timefile_exact():
-    eq_(get_timefile_exact(eight), "2014-05/30/2014-05-30-20-00-00.mp3")
+@pytest.mark.asyncio
+async def test_timefile_exact():
+    eq_(await get_timefile_exact(eight), "2014-05/30/2014-05-30-20-00-00.mp3")
 
 
 # Rounding
@@ -52,25 +58,27 @@ def test_rounding_value():
 # Rounding + timefile
 
 
-def test_timefile_alreadyround():
-    eq_(get_timefile(eight), "2014-05/30/2014-05-30-20-00-00.mp3")
+@pytest.mark.asyncio
+async def test_timefile_alreadyround():
+    eq_(await get_timefile(eight), "2014-05/30/2014-05-30-20-00-00.mp3")
 
 
-def test_timefile_toround():
-    eq_(get_timefile(eight + minutes(20)), "2014-05/30/2014-05-30-20-00-00.mp3")
+@pytest.mark.asyncio
+async def test_timefile_toround():
+    eq_(await get_timefile(eight + minutes(20)), "2014-05/30/2014-05-30-20-00-00.mp3")
 
 
 # Intervals
 
 
-@raises(ValueError)
 def test_intervals_same():
-    tuple(get_files_and_intervals(eight, eight))
+    with raises(ValueError):
+        tuple(get_files_and_intervals(eight, eight))
 
 
-@raises(ValueError)
 def test_intervals_before():
-    tuple(get_files_and_intervals(nine, eight))
+    with raises(ValueError):
+        tuple(get_files_and_intervals(nine, eight))
 
 
 def test_intervals_full_1():