Adding docker machinery and makefile

setup.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python
 
 from distutils.core import setup
+import setuptools
 
 REQUIREMENTS = [
     "SQLAlchemy==0.8.3",
     "aiofiles==0.6.0",
-    "aiohttp==3.7.4",
     "click==7.1.2",
     "fastapi==0.62.0",
     "h11==0.11.0",

techrec/cli.py

@@ -4,7 +4,6 @@ import os.path
 import sys
 from argparse import Action, ArgumentParser
 from datetime import datetime
-import urllib.request
 
 from . import forge, maint, server
 from .config_manager import get_config
@@ -13,27 +12,18 @@ logging.basicConfig(stream=sys.stdout)
 logger = logging.getLogger("cli")
 
 CWD = os.getcwd()
-OK_CODES = [200, 301, 302]
-
-
-def is_writable(d):
-    return os.access(d, os.W_OK)
 
 
 def pre_check_permissions():
-    audio_input = get_config()["AUDIO_INPUT"]
+    def is_writable(d):
-    if audio_input.startswith("http://") or audio_input.startswith("https://"):
+        return os.access(d, os.W_OK)
-        with urllib.request.urlopen(audio_input) as req:
+
-            if req.code not in OK_CODES:
+    if is_writable(get_config()["AUDIO_INPUT"]):
-                yield f"Audio input {audio_input} not accessible"
+        yield "Audio input '%s' writable" % get_config()["AUDIO_INPUT"]
-                sys.exit(10)
+    if not os.access(get_config()["AUDIO_INPUT"], os.R_OK):
-    else:
+        yield "Audio input '%s' unreadable" % get_config()["AUDIO_INPUT"]
-        if is_writable(audio_input):
+        sys.exit(10)
-            yield "Audio input '%s' writable" % audio_input
+    if is_writable(os.getcwd()):
-        if not os.access(audio_input, os.R_OK):
-            yield "Audio input '%s' unreadable" % audio_input
-            sys.exit(10)
-    if is_writable(CWD):
         yield "Code writable"
     if not is_writable(get_config()["AUDIO_OUTPUT"]):
         yield "Audio output '%s' not writable" % get_config()["AUDIO_OUTPUT"]
@@ -82,8 +72,7 @@ def common_pre():
                 continue
             path = os.path.realpath(conf)
             if not os.path.exists(path):
-                logger.warn(
+                logger.warn("Configuration file '%s' does not exist; skipping" % path)
-                    "Configuration file '%s' does not exist; skipping" % path)
                 continue
             configs.append(path)
     if getattr(sys, "frozen", False):

techrec/forge.py

@@ -1,4 +1,3 @@
-import asyncio
 import logging
 import tempfile
 import os
@@ -8,7 +7,7 @@ from time import sleep
 from typing import Callable, Optional
 
 from .config_manager import get_config
-from .http_retriever import download
+from .http_retriever import RETRIEVER
 
 logger = logging.getLogger("forge")
 Validator = Callable[[datetime, datetime, str], bool]
@@ -23,10 +22,8 @@ async def get_timefile_exact(time) -> str:
     remote_path = os.path.join(
         remote_repo, time.strftime(get_config()["AUDIO_INPUT_FORMAT"])
     )
-    if remote_path.startswith("http://") or remote_path.startswith("https://"):
+    if remote.startswith("http"):
-        logger.info(f"downloading {remote_path}")
+        local = await RETRIEVER.get(remote_path)
-        print(f"DOWNLOADING -> {remote_path}")
-        local = await download(remote_path)
         return local
     return local_path
 
@@ -119,7 +116,7 @@ async def create_mp3(
     intervals = []
     for begin, start_cut, end_cut in get_files_and_intervals(start, end):
         file = await get_timefile(begin)
-        intervals.append((file, start_cut, end_cut))
+        interval.append(file, start_cut, end_cut)
     if os.path.exists(outfile):
         raise OSError("file '%s' already exists" % outfile)
     for path, _s, _e in intervals:
@@ -190,8 +187,8 @@ async def create_mp3(
     return True
 
 
-def main_cmd(options):
+async def main_cmd(options):
     log = logging.getLogger("forge_main")
     outfile = os.path.abspath(os.path.join(options.cwd, options.outfile))
     log.debug("will forge an mp3 into %s" % (outfile))
-    asyncio.run(create_mp3(options.starttime, options.endtime, outfile))
+    await create_mp3(options.starttime, options.endtime, outfile)

techrec/http_retriever.py

@@ -1,8 +1,6 @@
 # -*- encoding: utf-8 -*-
 import asyncio
 import os
-from typing import Optional
-from tempfile import mkdtemp
 
 import aiohttp  # type: ignore
 
@@ -11,25 +9,52 @@ from .config_manager import get_config
 CHUNK_SIZE = 2 ** 12
 
 
-async def download(remote: str, staging: Optional[str] = None) -> str:
+class HTTPRetriever(object):
     """
-    This will download to AUDIO_STAGING the remote file and return the local
+    This class offers the `get` method to retrieve the file from the local staging path
-    path of the downloaded file
+    or, if missing, from the given remote.
     """
-    _, filename = os.path.split(remote)
+    _instance = None
-    if staging:
+
-        base = staging
+    def __new__(cls):
-    else:
+        if self._instance is None:
-        # if no staging is specified, and you want to clean the storage
+            self._instance = super().__new__(cls)
-        # used by techrec: rm -rf /tmp/techrec*
+        return self._instance
-        base = mkdtemp(prefix="techrec-", dir="/tmp")
+
-    local = os.path.join(base, filename)
+    def __init__(self):
-    async with aiohttp.ClientSession() as session:
+        self.repo_path = get_config()["AUDIO_STAGING"]
-        async with session.get(remote) as resp:
+        self.repo = dict(
-            with open(local, "wb") as f:
+            path=os.path.join(self.repo_path, path) for i in os.listdir(self.repo_path)
-                while True:
+        )
-                    chunk = await resp.content.read(CHUNK_SIZE)
+
-                    if not chunk:
+    async def get(remote: str) -> str:
-                        break
+        """
-                    f.write(chunk)
+        This will look in the local staging path (ideally on a tmpfs or something
-    return local
+        similar), and return the file from there if present. Otherwise, it will download
+        it from the remote.
+        """
+        if remote in self.repo:
+            return self.repo[remote]
+        file = await self._download_from_remote(remote)
+        self.repo[] = file
+        return file
+
+    async def _download(remote: str) -> str:
+        """
+        This will download to AUDIO_STAGING the remote file and return the local path
+        of the downloaded file
+        """
+        _, filename = os.path.split(remote)
+        local = os.path.join(get_config()["AUDIO_STAGING"], filename)
+        async with aiohttp.ClientSession() as session:
+            async with session.get(remote) as resp:
+                with open(local) as f:
+                    while True:
+                        chunk = await resp.content.read(CHUNK_SIZE)
+                        if not chunk:
+                            break
+                        f.write(chunk)
+        return local
+
+
+RETRIEVER = HTTPRetriever()