Fix startup checks

.gitignore

@@ -5,3 +5,4 @@ build/
 dist/
 rec/
 *.egg-info/
+/venv

Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.7
+
+ARG hostuid=1000
+ARG hostgid=1000
+
+ENV TECHREC_CONFIG=/src/techrec/docker/config.py
+ENV DEBIAN_FRONTEND=noninteractive
+
+WORKDIR /src
+COPY . /src/techrec
+
+RUN groupadd -g ${hostgid} techrec \
+    && useradd -g techrec -u ${hostuid} -m techrec \
+    && mkdir -p /src/techrec \
+    && chown -R techrec:techrec /src \
+    && apt-get update \
+    && apt-get install -y ffmpeg \
+ && rm -rf /var/lib/apt/lists/*
+
+USER techrec
+
+
+RUN python -m venv ./venv \
+    && ./venv/bin/python -m pip install wheel \
+ && ./venv/bin/python -m pip install -e ./techrec
+
+ENTRYPOINT ["/src/venv/bin/techrec"]
+CMD ["-vv", "serve"]

Makefile

@@ -0,0 +1,54 @@
+DOCKER := docker
+DOCKERC := docker-compose
+PORT := 8000
+VENV := venv
+CONFIG := dev_config.py
+PY := python
+
+docker-build:
+	$(DOCKERC) build \
+		--no-cache \
+		--build-arg=hostgid=$(shell id -g) \
+		--build-arg=hostuid=$(shell id -u) \
+		--build-arg=audiogid=$(shell cat /etc/group | grep audio | awk -F: '{print $3}')
+
+docker-build-liquidsoap:
+	$(DOCKER) pull savonet/liquidsoap:main
+	$(DOCKERC) build \
+		--no-cache \
+		--build-arg=audiogid=$(shell cat /etc/group | grep audio | awk -F: '{print $3}') \
+		liquidsoap
+
+docker-build-techrec:
+	$(DOCKERC) build \
+		--no-cache \
+		--build-arg=hostgid=$(shell id -g) \
+		--build-arg=hostuid=$(shell id -u) \
+		techrec
+
+docker-stop:
+	$(DOCKERC) down
+
+docker-run:
+	$(DOCKERC) run --rm --service-ports techrec
+
+docker-shell-techrec:
+	$(eval CONTAINER = $(shell docker ps|grep techrec_run|awk '{print $$12}'))
+	$(DOCKER) exec -ti $(CONTAINER) bash
+
+docker-shell-storage:
+	$(DOCKERC) exec storage bash
+
+docker-shell-liquidsoap:
+	$(eval CONTAINER = $(shell docker ps|grep liquidsoap|awk '{print $$12}'))
+	$(DOCKER) exec -ti $(CONTAINER) bash
+
+local-install:
+	$(PY) -m venv $(VENV)
+	./$(VENV)/bin/pip install -e .
+
+local-serve:
+	env TECHREC_CONFIG=$(CONFIG) ./$(VENV)/bin/techrec -vv serve
+
+
+.PHONY: docker-build docker-build-liquidsoap docker-build-techrec docker-stop docker-run docker-shell-techrec docker-shell-storage docker-shell-liquidsoap local-install local-serve

docker-compose.yaml

@@ -0,0 +1,46 @@
+version: "3"
+
+services:
+  liquidsoap:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile.liquidsoap
+    volumes:
+      - ./docker/run.liq:/run.liq
+      - ./docker/ror.sh:/ror.sh
+      - rec:/rec
+    devices:
+      - /dev/snd:/dev/snd
+    entrypoint: /run.liq
+    depends_on:
+      - storageprepare
+
+  storage:
+    image: nginx
+    volumes:
+      - rec:/var/www/rec
+      - ./docker/storage.conf:/etc/nginx/conf.d/default.conf:ro
+    ports:
+      - 18080:80
+    depends_on:
+      - storageprepare
+
+  storageprepare:
+    image: bash
+    volumes:
+      - rec:/rec
+    command: chmod 777 /rec
+
+  techrec:
+    build: .
+    volumes:
+      - .:/src/techrec
+      - ./docker/output:/src/output
+    ports:
+      - 8000:8000
+    depends_on:
+      - liquidsoap
+      - storage
+
+volumes:
+  rec:

docker/Dockerfile.liquidsoap

@@ -0,0 +1,10 @@
+FROM savonet/liquidsoap:main
+
+ENV audiogid=995
+
+USER root
+
+RUN groupadd -g ${audiogid} hostaudio \
+ && usermod -a -G hostaudio liquidsoap
+
+USER liquidsoap

docker/config.py

@@ -0,0 +1,12 @@
+import logging
+
+AUDIO_INPUT = "http://storage/ror"
+AUDIO_OUTPUT = "/src/output"
+DEBUG = True
+HOST = "0.0.0.0"
+PORT = 8000
+TRANSLOGGER_OPTS = {
+    "logger_name": "accesslog",
+    "set_logger_level": logging.INFO,
+    "setup_console_handler": True,
+}

docker/run.liq

@@ -0,0 +1,26 @@
+#!/usr/bin/liquidsoap
+
+settings.log.stdout.set(true);
+settings.log.file.set(false);
+settings.log.level.set(3);
+
+# settings.server.telnet.set(true);
+# settings.server.telnet.bind_addr.set("127.0.0.1");
+# settings.server.telnet.port.set(6666);
+
+rorinput = input.alsa(device="default", bufferize=true);
+#rorinput = input.pulseaudio( );
+
+# rorinput = insert_metadata(id="trx",rorinput);
+rorinput = rewrite_metadata([("artist","Radio OndaRossa")],rorinput);
+
+# ESCPOST
+output.file(
+  id="rorrec",
+  reopen_when={0m},
+  %mp3(bitrate=80, samplerate=44100, stereo=true,stereo_mode="joint_stereo"),
+  "/rec/ror/%Y-%m/%d/rec-%Y-%m-%d-%H-%M-%S.mp3",
+  # %vorbis(quality=0.3, samplerate=44100, channels=2),
+  # "/rec/ror/%Y-%m/%d/rec-%Y-%m-%d-%H-%M-%S.ogg",
+  rorinput
+);

docker/storage.conf

@@ -0,0 +1,9 @@
+server {
+    listen 80 default_server;
+    server_name storage;
+
+    location / {
+        root /var/www/rec;
+        autoindex on;
+    }
+}

setup.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python
 
 from distutils.core import setup
-import setuptools
 
 REQUIREMENTS = [
     "SQLAlchemy==0.8.3",
     "aiofiles==0.6.0",
+    "aiohttp==3.7.4",
     "click==7.1.2",
     "fastapi==0.62.0",
     "h11==0.11.0",

techrec/cli.py

@@ -4,6 +4,7 @@ import os.path
 import sys
 from argparse import Action, ArgumentParser
 from datetime import datetime
+import urllib.request
 
 from . import forge, maint, server
 from .config_manager import get_config
@@ -12,18 +13,27 @@ logging.basicConfig(stream=sys.stdout)
 logger = logging.getLogger("cli")
 
 CWD = os.getcwd()
+OK_CODES = [200, 301, 302]
+
+
+def is_writable(d):
+    return os.access(d, os.W_OK)
 
 
 def pre_check_permissions():
-    def is_writable(d):
+    audio_input = get_config()["AUDIO_INPUT"]
-        return os.access(d, os.W_OK)
+    if audio_input.startswith("http://") or audio_input.startswith("https://"):
-
+        with urllib.request.urlopen(audio_input) as req:
-    if is_writable(get_config()["AUDIO_INPUT"]):
+            if req.code not in OK_CODES:
-        yield "Audio input '%s' writable" % get_config()["AUDIO_INPUT"]
+                yield f"Audio input {audio_input} not accessible"
-    if not os.access(get_config()["AUDIO_INPUT"], os.R_OK):
+                sys.exit(10)
-        yield "Audio input '%s' unreadable" % get_config()["AUDIO_INPUT"]
+    else:
-        sys.exit(10)
+        if is_writable(audio_input):
-    if is_writable(os.getcwd()):
+            yield "Audio input '%s' writable" % audio_input
+        if not os.access(audio_input, os.R_OK):
+            yield "Audio input '%s' unreadable" % audio_input
+            sys.exit(10)
+    if is_writable(CWD):
         yield "Code writable"
     if not is_writable(get_config()["AUDIO_OUTPUT"]):
         yield "Audio output '%s' not writable" % get_config()["AUDIO_OUTPUT"]
@@ -72,7 +82,8 @@ def common_pre():
                 continue
             path = os.path.realpath(conf)
             if not os.path.exists(path):
-                logger.warn("Configuration file '%s' does not exist; skipping" % path)
+                logger.warn(
+                    "Configuration file '%s' does not exist; skipping" % path)
                 continue
             configs.append(path)
     if getattr(sys, "frozen", False):

techrec/forge.py

@@ -1,3 +1,4 @@
+import asyncio
 import logging
 import tempfile
 import os
@@ -7,19 +8,27 @@ from time import sleep
 from typing import Callable, Optional
 
 from .config_manager import get_config
+from .http_retriever import download
 
 logger = logging.getLogger("forge")
 Validator = Callable[[datetime, datetime, str], bool]
 
 
-def get_timefile_exact(time) -> str:
+async def get_timefile_exact(time) -> str:
     """
     time is of type `datetime`; it is not "rounded" to match the real file;
     that work is done in get_timefile(time)
     """
-    return os.path.join(
+    remote_repo = get_config()["AUDIO_INPUT"]
-        get_config()["AUDIO_INPUT"], time.strftime(get_config()["AUDIO_INPUT_FORMAT"])
+    remote_path = os.path.join(
+        remote_repo, time.strftime(get_config()["AUDIO_INPUT_FORMAT"])
     )
+    if remote_path.startswith("http://") or remote_path.startswith("https://"):
+        logger.info(f"downloading {remote_path}")
+        print(f"DOWNLOADING -> {remote_path}")
+        local = await download(remote_path)
+        return local
+    return local_path
 
 
 def round_timefile(exact: datetime) -> datetime:
@@ -29,8 +38,9 @@ def round_timefile(exact: datetime) -> datetime:
     return datetime(exact.year, exact.month, exact.day, exact.hour)
 
 
-def get_timefile(exact: datetime) -> str:
+async def get_timefile(exact: datetime) -> str:
-    return get_timefile_exact(round_timefile(exact))
+    file = await get_timefile_exact(round_timefile(exact))
+    return file
 
 
 def get_files_and_intervals(start, end, rounder=round_timefile):
@@ -93,7 +103,7 @@ def mp3_join(named_intervals):
     return cmdline
 
 
-def create_mp3(
+async def create_mp3(
     start: datetime,
     end: datetime,
     outfile: str,
@@ -106,10 +116,10 @@ def create_mp3(
         def validator(s, e, f):
             return True
 
-    intervals = [
+    intervals = []
-        (get_timefile(begin), start_cut, end_cut)
+    for begin, start_cut, end_cut in get_files_and_intervals(start, end):
-        for begin, start_cut, end_cut in get_files_and_intervals(start, end)
+        file = await get_timefile(begin)
-    ]
+        intervals.append((file, start_cut, end_cut))
     if os.path.exists(outfile):
         raise OSError("file '%s' already exists" % outfile)
     for path, _s, _e in intervals:
@@ -184,4 +194,4 @@ def main_cmd(options):
     log = logging.getLogger("forge_main")
     outfile = os.path.abspath(os.path.join(options.cwd, options.outfile))
     log.debug("will forge an mp3 into %s" % (outfile))
-    create_mp3(options.starttime, options.endtime, outfile)
+    asyncio.run(create_mp3(options.starttime, options.endtime, outfile))

techrec/http_retriever.py

@@ -1,6 +1,8 @@
 # -*- encoding: utf-8 -*-
 import asyncio
 import os
+from typing import Optional
+from tempfile import mkdtemp
 
 import aiohttp  # type: ignore
 
@@ -9,52 +11,25 @@ from .config_manager import get_config
 CHUNK_SIZE = 2 ** 12
 
 
-class HTTPRetriever(object):
+async def download(remote: str, staging: Optional[str] = None) -> str:
     """
-    This class offers the `get` method to retrieve the file from the local staging path
+    This will download to AUDIO_STAGING the remote file and return the local
-    or, if missing, from the given remote.
+    path of the downloaded file
     """
-    _instance = None
+    _, filename = os.path.split(remote)
-
+    if staging:
-    def __new__(cls):
+        base = staging
-        if self._instance is None:
+    else:
-            self._instance = super().__new__(cls)
+        # if no staging is specified, and you want to clean the storage
-        return self._instance
+        # used by techrec: rm -rf /tmp/techrec*
-
+        base = mkdtemp(prefix="techrec-", dir="/tmp")
-    def __init__(self):
+    local = os.path.join(base, filename)
-        self.repo_path = get_config()["AUDIO_STAGING"]
+    async with aiohttp.ClientSession() as session:
-        self.repo = dict(
+        async with session.get(remote) as resp:
-            path=os.path.join(self.repo_path, path) for i in os.listdir(self.repo_path)
+            with open(local, "wb") as f:
-        )
+                while True:
-
+                    chunk = await resp.content.read(CHUNK_SIZE)
-    async def get(remote: str) -> str:
+                    if not chunk:
-        """
+                        break
-        This will look in the local staging path (ideally on a tmpfs or something
+                    f.write(chunk)
-        similar), and return the file from there if present. Otherwise, it will download
+    return local
-        it from the remote.
-        """
-        if remote in self.repo:
-            return self.repo[remote]
-        file = await self._download_from_remote(remote)
-        self.repo[] = file
-        return file
-
-    async def _download(remote: str) -> str:
-        """
-        This will download to AUDIO_STAGING the remote file and return the local path
-        of the downloaded file
-        """
-        _, filename = os.path.split(remote)
-        local = os.path.join(get_config()["AUDIO_STAGING"], filename)
-        async with aiohttp.ClientSession() as session:
-            async with session.get(remote) as resp:
-                with open(local) as f:
-                    while True:
-                        chunk = await resp.content.read(CHUNK_SIZE)
-                        if not chunk:
-                            break
-                        f.write(chunk)
-        return local
-
-
-RETRIEVER = HTTPRetriever()

techrec/server.py

@@ -256,7 +256,7 @@ def get_validator(expected_duration_s: float, error_threshold_s: float) -> Valid
     return validator
 
 
-def generate_mp3(db_id: int, **kwargs):
+async def generate_mp3(db_id: int, **kwargs):
     """creates and mark it as ready in the db"""
     if get_config()["FORGE_VERIFY"]:
         validator = get_validator(
@@ -269,7 +269,7 @@ def generate_mp3(db_id: int, **kwargs):
         retries = 1
 
     for i in range(retries):
-        result = create_mp3(validator=validator, **kwargs)
+        result = await create_mp3(validator=validator, **kwargs)
         logger.debug("Create mp3 for %d -> %s", db_id, result)
         if result:
             break