This commit is contained in:
Blallo 2021-08-25 12:03:39 -03:00
parent 7e99e31f43
commit 5124f2d3ca
No known key found for this signature in database
GPG key ID: 0CBE577C9B72DC3F
5 changed files with 85 additions and 39 deletions

View file

@ -11,7 +11,6 @@ from .config_manager import get_config
logging.basicConfig(stream=sys.stdout) logging.basicConfig(stream=sys.stdout)
logger = logging.getLogger("cli") logger = logging.getLogger("cli")
CWD = os.getcwd() CWD = os.getcwd()
@ -60,8 +59,10 @@ class DateTimeAction(Action):
raise ValueError("'%s' is not a valid datetime" % values) raise ValueError("'%s' is not a valid datetime" % values)
setattr(namespace, self.dest, parsed_val) setattr(namespace, self.dest, parsed_val)
code_dir = os.path.dirname(os.path.realpath(__file__)) code_dir = os.path.dirname(os.path.realpath(__file__))
def common_pre(): def common_pre():
prechecks = [pre_check_user, pre_check_permissions, pre_check_ffmpeg] prechecks = [pre_check_user, pre_check_permissions, pre_check_ffmpeg]
configs = ["default_config.py"] configs = ["default_config.py"]

View file

@ -7,8 +7,15 @@ import logging
import sys import sys
from datetime import datetime, timedelta from datetime import datetime, timedelta
from sqlalchemy import (Column, DateTime, Boolean, Integer, String, create_engine, from sqlalchemy import (
inspect) Column,
DateTime,
Boolean,
Integer,
String,
create_engine,
inspect,
)
from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker from sqlalchemy.orm import sessionmaker

View file

@ -35,7 +35,7 @@ TAG_LICENSE_URI = None
# defaults # defaults
STATIC_FILES = "static/" STATIC_FILES = "static/"
STATIC_PAGES = "pages/" STATIC_PAGES = "pages/"
if getattr(sys, 'frozen', False): # pyinstaller if getattr(sys, "frozen", False): # pyinstaller
STATIC_FILES = os.path.join(sys._MEIPASS, STATIC_FILES) STATIC_FILES = os.path.join(sys._MEIPASS, STATIC_FILES)
STATIC_PAGES = os.path.join(sys._MEIPASS, STATIC_PAGES) STATIC_PAGES = os.path.join(sys._MEIPASS, STATIC_PAGES)
else: else:

View file

@ -11,6 +11,7 @@ from .config_manager import get_config
logger = logging.getLogger("forge") logger = logging.getLogger("forge")
Validator = Callable[[datetime, datetime, str], bool] Validator = Callable[[datetime, datetime, str], bool]
def get_timefile_exact(time) -> str: def get_timefile_exact(time) -> str:
""" """
time is of type `datetime`; it is not "rounded" to match the real file; time is of type `datetime`; it is not "rounded" to match the real file;
@ -21,14 +22,14 @@ def get_timefile_exact(time) -> str:
) )
def round_timefile(exact:datetime) -> datetime: def round_timefile(exact: datetime) -> datetime:
""" """
This will round the datetime, so to match the file organization structure This will round the datetime, so to match the file organization structure
""" """
return datetime(exact.year, exact.month, exact.day, exact.hour) return datetime(exact.year, exact.month, exact.day, exact.hour)
def get_timefile(exact:datetime) -> str: def get_timefile(exact: datetime) -> str:
return get_timefile_exact(round_timefile(exact)) return get_timefile_exact(round_timefile(exact))
@ -88,9 +89,17 @@ def mp3_join(named_intervals):
cmdline += ["-t", str(len(files) * 3600 - (startskip + endskip))] cmdline += ["-t", str(len(files) * 3600 - (startskip + endskip))]
return cmdline return cmdline
def create_mp3(start: datetime, end: datetime, outfile: str, options={}, validator: Optional[Validator] = None, **kwargs):
def create_mp3(
start: datetime,
end: datetime,
outfile: str,
options={},
validator: Optional[Validator] = None,
**kwargs
):
if validator is None: if validator is None:
validator = lambda s,e,f: True validator = lambda s, e, f: True
intervals = [ intervals = [
(get_timefile(begin), start_cut, end_cut) (get_timefile(begin), start_cut, end_cut)
for begin, start_cut, end_cut in get_files_and_intervals(start, end) for begin, start_cut, end_cut in get_files_and_intervals(start, end)
@ -126,9 +135,17 @@ def create_mp3(start: datetime, end: datetime, outfile: str, options={}, validat
metadata_list.append("-metadata") metadata_list.append("-metadata")
metadata_list.append("%s=%s" % (tag, value)) metadata_list.append("%s=%s" % (tag, value))
prefix, suffix = os.path.basename(outfile).split('.', 1) prefix, suffix = os.path.basename(outfile).split(".", 1)
tmp_file = tempfile.NamedTemporaryFile(suffix='.%s' % suffix, prefix='forge-%s' % prefix, delete=False) tmp_file = tempfile.NamedTemporaryFile(
cmd = mp3_join(intervals) + metadata_list + ['-y'] + get_config()["FFMPEG_OPTIONS"] + [tmp_file.name] suffix=".%s" % suffix, prefix="forge-%s" % prefix, delete=False
)
cmd = (
mp3_join(intervals)
+ metadata_list
+ ["-y"]
+ get_config()["FFMPEG_OPTIONS"]
+ [tmp_file.name]
)
logger.info("Running %s", " ".join(cmd)) logger.info("Running %s", " ".join(cmd))
p = Popen(cmd) p = Popen(cmd)
if get_config()["FORGE_TIMEOUT"] == 0: if get_config()["FORGE_TIMEOUT"] == 0:

View file

@ -39,14 +39,16 @@ def rec_sanitize(rec):
d["endtime"] = date_write(d["endtime"]) d["endtime"] = date_write(d["endtime"])
return d return d
@app.on_event("startup") @app.on_event("startup")
async def startup_event(): async def startup_event():
global db global db
common_pre() common_pre()
if get_config()['DEBUG']: if get_config()["DEBUG"]:
logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)
db = RecDB(get_config()["DB_URI"]) db = RecDB(get_config()["DB_URI"])
@app.get("/date/date") @app.get("/date/date")
def date(): def date():
n = datetime.now() n = datetime.now()
@ -76,11 +78,13 @@ def help():
+ "/custom?strftime=FORMAT : get now().strftime(FORMAT)" + "/custom?strftime=FORMAT : get now().strftime(FORMAT)"
) )
class CreateInfo(BaseModel): class CreateInfo(BaseModel):
starttime: Optional[str] = None starttime: Optional[str] = None
endtime: Optional[str] = None endtime: Optional[str] = None
name: str = "" name: str = ""
@app.post("/api/create") @app.post("/api/create")
async def create(req: CreateInfo = None): async def create(req: CreateInfo = None):
ret = {} ret = {}
@ -100,9 +104,11 @@ async def create(req: CreateInfo = None):
"Nuova registrazione creata! (id:%d)" % ret.id, rec=rec_sanitize(rec) "Nuova registrazione creata! (id:%d)" % ret.id, rec=rec_sanitize(rec)
) )
class DeleteInfo(BaseModel): class DeleteInfo(BaseModel):
id: int id: int
@app.post("/api/delete") @app.post("/api/delete")
def delete(req: DeleteInfo): def delete(req: DeleteInfo):
if db.delete(req.id): if db.delete(req.id):
@ -113,15 +119,18 @@ def delete(req: DeleteInfo):
def timefield_factory(): def timefield_factory():
return int(time.time()) return int(time.time())
TimeField = Field(default_factory=timefield_factory) TimeField = Field(default_factory=timefield_factory)
class UpdateInfo(BaseModel): class UpdateInfo(BaseModel):
name: str = "" name: str = ""
starttime: int =Field(default_factory=timefield_factory) starttime: int = Field(default_factory=timefield_factory)
endtime: int =Field(default_factory=timefield_factory) endtime: int = Field(default_factory=timefield_factory)
filename: Optional[str] = None filename: Optional[str] = None
@app.post("/api/update/{recid}") @app.post("/api/update/{recid}")
async def update(recid: int, req: UpdateInfo): async def update(recid: int, req: UpdateInfo):
newrec = {} newrec = {}
@ -142,10 +151,12 @@ async def update(recid: int, req: UpdateInfo):
class GenerateInfo(BaseModel): class GenerateInfo(BaseModel):
id: int id: int
class GenerateResponse(BaseModel): class GenerateResponse(BaseModel):
status: str status: str
message: str message: str
@app.post("/api/generate/{recid}") @app.post("/api/generate/{recid}")
async def generate(recid: int, response: Response, background_tasks: BackgroundTasks): async def generate(recid: int, response: Response, background_tasks: BackgroundTasks):
# prendiamo la rec in causa # prendiamo la rec in causa
@ -162,10 +173,10 @@ async def generate(recid: int, response: Response, background_tasks: BackgroundT
> get_config()["FORGE_MAX_DURATION"] > get_config()["FORGE_MAX_DURATION"]
): ):
return JSONResponse( return JSONResponse(
status_code = 400, status_code=400,
status= "error", status="error",
message= "The requested recording is too long" message="The requested recording is too long"
+ " (%d seconds)" % (rec.endtime - rec.starttime).total_seconds(), + " (%d seconds)" % (rec.endtime - rec.starttime).total_seconds(),
) )
rec.filename = get_config()["AUDIO_OUTPUT_FORMAT"] % { rec.filename = get_config()["AUDIO_OUTPUT_FORMAT"] % {
"time": rec.starttime.strftime( "time": rec.starttime.strftime(
@ -204,17 +215,19 @@ async def generate(recid: int, response: Response, background_tasks: BackgroundT
rec=rec_sanitize(rec), rec=rec_sanitize(rec),
) )
def get_duration(fname) -> float: def get_duration(fname) -> float:
lineout = check_output( lineout = check_output(
[ [
"ffprobe", "ffprobe",
"-v", "-v",
"error", "error",
"-show_entries", "-show_entries",
"format=duration", "format=duration",
"-i", "-i",
fname, fname,
]).split(b'\n') ]
).split(b"\n")
duration = next(l for l in lineout if l.startswith(b"duration=")) duration = next(l for l in lineout if l.startswith(b"duration="))
value = duration.split(b"=")[1] value = duration.split(b"=")[1]
return float(value) return float(value)
@ -225,24 +238,31 @@ def get_validator(expected_duration_s: float, error_threshold_s: float) -> Valid
try: try:
duration = get_duration(fpath) duration = get_duration(fpath)
except Exception as exc: except Exception as exc:
logger.exception('Error determining duration of %s', fpath) logger.exception("Error determining duration of %s", fpath)
return False return False
logger.debug('expect %s to be %.1f±%.1fs, is %.1f', fpath, expected_duration_s, error_threshold_s, duration) logger.debug(
"expect %s to be %.1f±%.1fs, is %.1f",
fpath,
expected_duration_s,
error_threshold_s,
duration,
)
if duration > expected_duration_s + error_threshold_s: if duration > expected_duration_s + error_threshold_s:
return False return False
if duration < expected_duration_s - error_threshold_s: if duration < expected_duration_s - error_threshold_s:
return False return False
return True return True
return validator return validator
def generate_mp3(db_id: int, **kwargs): def generate_mp3(db_id: int, **kwargs):
'''creates and mark it as ready in the db''' """creates and mark it as ready in the db"""
if get_config()['FORGE_VERIFY']: if get_config()["FORGE_VERIFY"]:
validator = get_validator( validator = get_validator(
(kwargs['end'] - kwargs['start']).total_seconds(), (kwargs["end"] - kwargs["start"]).total_seconds(),
get_config()['FORGE_VERIFY_THRESHOLD'] get_config()["FORGE_VERIFY_THRESHOLD"],
) )
retries = 10 retries = 10
else: else:
validator = None validator = None
@ -250,11 +270,11 @@ def generate_mp3(db_id: int, **kwargs):
for i in range(retries): for i in range(retries):
result = create_mp3(validator=validator, **kwargs) result = create_mp3(validator=validator, **kwargs)
logger.debug('Create mp3 for %d -> %s', db_id, result) logger.debug("Create mp3 for %d -> %s", db_id, result)
if result: if result:
break break
elif i < retries - 1: elif i < retries - 1:
logger.debug("waiting %d", i+1) logger.debug("waiting %d", i + 1)
time.sleep(i + 1) # waiting time increases at each retry time.sleep(i + 1) # waiting time increases at each retry
else: else:
logger.warning("Could not create mp3 for %d: validation failed", db_id) logger.warning("Could not create mp3 for %d: validation failed", db_id)
@ -266,7 +286,6 @@ def generate_mp3(db_id: int, **kwargs):
return True return True
@app.get("/api/ready/{recid}") @app.get("/api/ready/{recid}")
def check_job(recid: int): def check_job(recid: int):
rec = db._search(_id=recid)[0] rec = db._search(_id=recid)[0]
@ -279,7 +298,6 @@ def check_job(recid: int):
return ret("WIP") return ret("WIP")
@app.get("/api/get/ongoing") @app.get("/api/get/ongoing")
def get_ongoing(): def get_ongoing():
return {rec.id: rec_sanitize(rec) for rec in db.get_ongoing()} return {rec.id: rec_sanitize(rec) for rec in db.get_ongoing()}
@ -341,9 +359,12 @@ def serve_pages(request: Request):
fpath = os.path.join(get_config()["STATIC_PAGES"], page) fpath = os.path.join(get_config()["STATIC_PAGES"], page)
return FileResponse(fpath) return FileResponse(fpath)
def main_cmd(options): def main_cmd(options):
import uvicorn import uvicorn
uvicorn.run(app, host=get_config()['HOST'], port=int(get_config()['PORT']))
uvicorn.run(app, host=get_config()["HOST"], port=int(get_config()["PORT"]))
if __name__ == "__main__": if __name__ == "__main__":
logger.warn("Usage of server.py is not supported anymore; use cli.py") logger.warn("Usage of server.py is not supported anymore; use cli.py")