Plug http retriever in current logic

techrec/forge.py

@@ -1,3 +1,4 @@
+import asyncio
 import logging
 import tempfile
 import os
@@ -7,19 +8,25 @@ from time import sleep
 from typing import Callable, Optional
 
 from .config_manager import get_config
+from .http_retriever import RETRIEVER
 
 logger = logging.getLogger("forge")
 Validator = Callable[[datetime, datetime, str], bool]
 
 
-def get_timefile_exact(time) -> str:
+async def get_timefile_exact(time) -> str:
     """
     time is of type `datetime`; it is not "rounded" to match the real file;
     that work is done in get_timefile(time)
     """
-    return os.path.join(
-        get_config()["AUDIO_INPUT"], time.strftime(get_config()["AUDIO_INPUT_FORMAT"])
+    remote_repo = get_config()["AUDIO_INPUT"]
+    remote_path = os.path.join(
+        remote_repo, time.strftime(get_config()["AUDIO_INPUT_FORMAT"])
     )
+    if remote.startswith("http"):
+        local = await RETRIEVER.get(remote_path)
+        return local
+    return local_path
 
 
 def round_timefile(exact: datetime) -> datetime:
@@ -29,8 +36,9 @@ def round_timefile(exact: datetime) -> datetime:
     return datetime(exact.year, exact.month, exact.day, exact.hour)
 
 
-def get_timefile(exact: datetime) -> str:
-    return get_timefile_exact(round_timefile(exact))
+async def get_timefile(exact: datetime) -> str:
+    file = await get_timefile_exact(round_timefile(exact))
+    return file
 
 
 def get_files_and_intervals(start, end, rounder=round_timefile):
@@ -93,7 +101,7 @@ def mp3_join(named_intervals):
     return cmdline
 
 
-def create_mp3(
+async def create_mp3(
     start: datetime,
     end: datetime,
     outfile: str,
@@ -106,10 +114,10 @@ def create_mp3(
         def validator(s, e, f):
             return True
 
-    intervals = [
-        (get_timefile(begin), start_cut, end_cut)
-        for begin, start_cut, end_cut in get_files_and_intervals(start, end)
-    ]
+    intervals = []
+    for begin, start_cut, end_cut in get_files_and_intervals(start, end):
+        file = await get_timefile(begin)
+        intervals.append((file, start_cut, end_cut))
     if os.path.exists(outfile):
         raise OSError("file '%s' already exists" % outfile)
     for path, _s, _e in intervals:
@@ -184,4 +192,4 @@ def main_cmd(options):
     log = logging.getLogger("forge_main")
     outfile = os.path.abspath(os.path.join(options.cwd, options.outfile))
     log.debug("will forge an mp3 into %s" % (outfile))
-    create_mp3(options.starttime, options.endtime, outfile)
+    asyncio.run(create_mp3(options.starttime, options.endtime, outfile))

techrec/server.py

@@ -256,7 +256,7 @@ def get_validator(expected_duration_s: float, error_threshold_s: float) -> Valid
     return validator
 
 
-def generate_mp3(db_id: int, **kwargs):
+async def generate_mp3(db_id: int, **kwargs):
     """creates and mark it as ready in the db"""
     if get_config()["FORGE_VERIFY"]:
         validator = get_validator(
@@ -269,7 +269,7 @@ def generate_mp3(db_id: int, **kwargs):
         retries = 1
 
     for i in range(retries):
-        result = create_mp3(validator=validator, **kwargs)
+        result = await create_mp3(validator=validator, **kwargs)
         logger.debug("Create mp3 for %d -> %s", db_id, result)
         if result:
             break