techbloc/techrec
1
0
Fork 0
Code Issues 10 Pull requests Releases Activity

black reformatting

Browse source
This commit is contained in:
boyska 2019-11-15 22:35:45 +01:00
parent 5cd55d15bd
commit b956076fc5
10 changed files with 472 additions and 372 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

114
server/cli.py
View file

@ -3,8 +3,9 @@ import sys
from argparse import ArgumentParser, Action from argparse import ArgumentParser, Action
from datetime import datetime from datetime import datetime
import logging import logging
logging.basicConfig(stream=sys.stdout) logging.basicConfig(stream=sys.stdout)
logger = logging.getLogger('cli') logger = logging.getLogger("cli")
CWD = os.getcwd() CWD = os.getcwd()
@ -16,19 +17,18 @@ from . import server
def pre_check_permissions(): def pre_check_permissions():
def is_writable(d): def is_writable(d):
return os.access(d, os.W_OK) return os.access(d, os.W_OK)
if is_writable(get_config()['AUDIO_INPUT']): if is_writable(get_config()["AUDIO_INPUT"]):
yield "Audio input '%s' writable" % get_config()['AUDIO_INPUT'] yield "Audio input '%s' writable" % get_config()["AUDIO_INPUT"]
if not os.access(get_config()['AUDIO_INPUT'], os.R_OK): if not os.access(get_config()["AUDIO_INPUT"], os.R_OK):
yield "Audio input '%s' unreadable" % get_config()['AUDIO_INPUT'] yield "Audio input '%s' unreadable" % get_config()["AUDIO_INPUT"]
sys.exit(10) sys.exit(10)
if is_writable(os.getcwd()): if is_writable(os.getcwd()):
yield "Code writable" yield "Code writable"
if not is_writable(get_config()['AUDIO_OUTPUT']): if not is_writable(get_config()["AUDIO_OUTPUT"]):
yield "Audio output '%s' not writable" % get_config()['AUDIO_OUTPUT'] yield "Audio output '%s' not writable" % get_config()["AUDIO_OUTPUT"]
logger.critical("Aborting") logger.critical("Aborting")
sys.exit(10) sys.exit(10)
@ -37,13 +37,15 @@ def pre_check_user():
if os.geteuid() == 0: if os.geteuid() == 0:
yield "You're running as root; this is dangerous" yield "You're running as root; this is dangerous"
def pre_check_ffmpeg(): def pre_check_ffmpeg():
path = get_config()['FFMPEG_PATH'] path = get_config()["FFMPEG_PATH"]
if not path.startswith('/'): if not path.startswith("/"):
yield "FFMPEG_PATH is not absolute: %s" % path yield "FFMPEG_PATH is not absolute: %s" % path
from subprocess import check_output from subprocess import check_output
try: try:
check_output([path, '-version']) check_output([path, "-version"])
except OSError: except OSError:
yield "FFMPEG not found as " + path yield "FFMPEG not found as " + path
else: else:
@ -54,7 +56,7 @@ def pre_check_ffmpeg():
class DateTimeAction(Action): class DateTimeAction(Action):
def __call__(self, parser, namespace, values, option_string=None): def __call__(self, parser, namespace, values, option_string=None):
if len(values) == 15 or len(values) == 13: if len(values) == 15 or len(values) == 13:
parsed_val = datetime.strptime(values, '%Y%m%d-%H%M%S') parsed_val = datetime.strptime(values, "%Y%m%d-%H%M%S")
else: else:
raise ValueError("'%s' is not a valid datetime" % values) raise ValueError("'%s' is not a valid datetime" % values)
setattr(namespace, self.dest, parsed_val) setattr(namespace, self.dest, parsed_val)
@ -62,15 +64,14 @@ class DateTimeAction(Action):
def common_pre(): def common_pre():
prechecks = [pre_check_user, pre_check_permissions, pre_check_ffmpeg] prechecks = [pre_check_user, pre_check_permissions, pre_check_ffmpeg]
configs = ['default_config.py'] configs = ["default_config.py"]
if 'TECHREC_CONFIG' in os.environ: if "TECHREC_CONFIG" in os.environ:
for conf in os.environ['TECHREC_CONFIG'].split(':'): for conf in os.environ["TECHREC_CONFIG"].split(":"):
if not conf: if not conf:
continue continue
path = os.path.realpath(conf) path = os.path.realpath(conf)
if not os.path.exists(path): if not os.path.exists(path):
logger.warn("Configuration file '%s' does not exist; skipping" logger.warn("Configuration file '%s' does not exist; skipping" % path)
% path)
continue continue
configs.append(path) configs.append(path)
os.chdir(os.path.dirname(os.path.realpath(__file__))) os.chdir(os.path.dirname(os.path.realpath(__file__)))
@ -81,36 +82,65 @@ def common_pre():
for warn in check(): for warn in check():
logger.warn(warn) logger.warn(warn)
def main(): def main():
parser = ArgumentParser(description='creates mp3 from live recordings') parser = ArgumentParser(description="creates mp3 from live recordings")
parser.add_argument('--verbose', '-v', action='count', parser.add_argument(
default=0, "--verbose",
help='Increase verbosity; can be used multiple times') "-v",
parser.add_argument('--pretend', '-p', action='store_true', default=False, action="count",
help='Only pretend; no real action will be done') default=0,
sub = parser.add_subparsers(title='main subcommands', help="Increase verbosity; can be used multiple times",
description='valid subcommands') )
serve_p = sub.add_parser('serve', help="Start an HTTP server") parser.add_argument(
"--pretend",
"-p",
action="store_true",
default=False,
help="Only pretend; no real action will be done",
)
sub = parser.add_subparsers(
title="main subcommands", description="valid subcommands"
)
serve_p = sub.add_parser("serve", help="Start an HTTP server")
serve_p.set_defaults(func=server.main_cmd) serve_p.set_defaults(func=server.main_cmd)
forge_p = sub.add_parser('forge', help="Create an audio file") forge_p = sub.add_parser("forge", help="Create an audio file")
forge_p.add_argument('starttime', metavar='START', forge_p.add_argument(
help='Start time, espressed as 19450425_1200 (%%Y%%m%%d-%%H%%M%%S)', "starttime",
action=DateTimeAction) metavar="START",
forge_p.add_argument('endtime', metavar='END', help="Start time, espressed as 19450425_1200 (%%Y%%m%%d-%%H%%M%%S)",
help='End time, espressed as 19450425_1200 (%%Y%%m%%d-%%H%%M%%S)', action=DateTimeAction,
action=DateTimeAction) )
forge_p.add_argument('-o', metavar='OUTFILE', dest='outfile', forge_p.add_argument(
default='out.mp3', help='Path of the output mp3') "endtime",
metavar="END",
help="End time, espressed as 19450425_1200 (%%Y%%m%%d-%%H%%M%%S)",
action=DateTimeAction,
)
forge_p.add_argument(
"-o",
metavar="OUTFILE",
dest="outfile",
default="out.mp3",
help="Path of the output mp3",
)
forge_p.set_defaults(func=forge.main_cmd) forge_p.set_defaults(func=forge.main_cmd)
cleanold_p = sub.add_parser('cleanold', help="Remove old files from DB", cleanold_p = sub.add_parser(
description="Will remove oldfiles with no filename from DB") "cleanold",
cleanold_p.add_argument('-t', metavar='MINAGE', dest='minage', help="Remove old files from DB",
default='14', type=int, description="Will remove oldfiles with no filename from DB",
help='Minimum age (in days) for removal') )
cleanold_p.add_argument(
"-t",
metavar="MINAGE",
dest="minage",
default="14",
type=int,
help="Minimum age (in days) for removal",
)
cleanold_p.set_defaults(func=maint.cleanold_cmd) cleanold_p.set_defaults(func=maint.cleanold_cmd)
options = parser.parse_args() options = parser.parse_args()
options.cwd = CWD options.cwd = CWD
if options.verbose < 1: if options.verbose < 1:
@ -123,5 +153,7 @@ def main():
logging.info("giving verbose flag >2 times is useless") logging.info("giving verbose flag >2 times is useless")
common_pre() common_pre()
options.func(options) options.func(options)
if __name__ == "__main__": if __name__ == "__main__":
main() main()

35
server/config_manager.py
View file

@ -10,6 +10,8 @@ def get_config():
if get_config.instance is None: if get_config.instance is None:
get_config.instance = Config(os.getcwd()) get_config.instance = Config(os.getcwd())
return get_config.instance return get_config.instance
get_config.instance = None get_config.instance = None
@ -78,11 +80,12 @@ class Config(dict):
if not rv: if not rv:
if silent: if silent:
return False return False
raise RuntimeError('The environment variable %r is not set ' raise RuntimeError(
'and as such configuration could not be ' "The environment variable %r is not set "
'loaded. Set this variable and make it ' "and as such configuration could not be "
'point to a configuration file' % "loaded. Set this variable and make it "
variable_name) "point to a configuration file" % variable_name
)
return self.from_pyfile(rv, silent=silent) return self.from_pyfile(rv, silent=silent)
def from_pyfile(self, filename, silent=False): def from_pyfile(self, filename, silent=False):
@ -100,15 +103,15 @@ class Config(dict):
`silent` parameter. `silent` parameter.
""" """
filename = os.path.join(self.root_path, filename) filename = os.path.join(self.root_path, filename)
d = imp.new_module('config') d = imp.new_module("config")
d.__file__ = filename d.__file__ = filename
try: try:
with open(filename) as config_file: with open(filename) as config_file:
exec(compile(config_file.read(), filename, 'exec'), d.__dict__) exec(compile(config_file.read(), filename, "exec"), d.__dict__)
except IOError as e: except IOError as e:
if silent and e.errno in (errno.ENOENT, errno.EISDIR): if silent and e.errno in (errno.ENOENT, errno.EISDIR):
return False return False
e.strerror = 'Unable to load configuration file (%s)' % e.strerror e.strerror = "Unable to load configuration file (%s)" % e.strerror
raise raise
self.from_object(d) self.from_object(d)
return True return True
@ -143,7 +146,7 @@ class Config(dict):
self[key] = getattr(obj, key) self[key] = getattr(obj, key)
def __repr__(self): def __repr__(self):
return '<%s %s>' % (self.__class__.__name__, dict.__repr__(self)) return "<%s %s>" % (self.__class__.__name__, dict.__repr__(self))
def import_string(import_name, silent=False): def import_string(import_name, silent=False):
@ -159,27 +162,27 @@ def import_string(import_name, silent=False):
`None` is returned instead. `None` is returned instead.
:return: imported object :return: imported object
""" """
#XXX: py3 review needed # XXX: py3 review needed
assert isinstance(import_name, string_types) assert isinstance(import_name, string_types)
# force the import name to automatically convert to strings # force the import name to automatically convert to strings
import_name = str(import_name) import_name = str(import_name)
try: try:
if ':' in import_name: if ":" in import_name:
module, obj = import_name.split(':', 1) module, obj = import_name.split(":", 1)
elif '.' in import_name: elif "." in import_name:
module, obj = import_name.rsplit('.', 1) module, obj = import_name.rsplit(".", 1)
else: else:
return __import__(import_name) return __import__(import_name)
# __import__ is not able to handle unicode strings in the fromlist # __import__ is not able to handle unicode strings in the fromlist
# if the module is a package # if the module is a package
if sys.version_info[0] == 2 and isinstance(obj, unicode): if sys.version_info[0] == 2 and isinstance(obj, unicode):
obj = obj.encode('utf-8') obj = obj.encode("utf-8")
try: try:
return getattr(__import__(module, None, None, [obj]), obj) return getattr(__import__(module, None, None, [obj]), obj)
except (ImportError, AttributeError): except (ImportError, AttributeError):
# support importing modules not yet set up by the parent module # support importing modules not yet set up by the parent module
# (or package for that matter) # (or package for that matter)
modname = module + '.' + obj modname = module + "." + obj
__import__(modname) __import__(modname)
return sys.modules[modname] return sys.modules[modname]
except ImportError as e: except ImportError as e:

100
server/db.py
View file

@ -1,6 +1,6 @@
''' """
This module contains DB logic This module contains DB logic
''' """
from __future__ import print_function from __future__ import print_function
import logging import logging
from datetime import datetime, timedelta from datetime import datetime, timedelta
@ -20,33 +20,38 @@ Base = declarative_base()
class Rec(Base): class Rec(Base):
'''Entry on the DB''' """Entry on the DB"""
__tablename__ = 'rec'
__tablename__ = "rec"
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
name = Column(String, nullable=True) name = Column(String, nullable=True)
starttime = Column(DateTime, nullable=True) starttime = Column(DateTime, nullable=True)
endtime = Column(DateTime, nullable=True) endtime = Column(DateTime, nullable=True)
filename = Column(String, nullable=True) filename = Column(String, nullable=True)
def __init__(self, name="", starttime=None, endtime=None, def __init__(self, name="", starttime=None, endtime=None, filename=None):
filename=None):
self.name = name self.name = name
self.starttime = starttime self.starttime = starttime
self.endtime = endtime self.endtime = endtime
self.filename = filename self.filename = filename
def serialize(self): def serialize(self):
'''json-friendly encoding''' """json-friendly encoding"""
return {'id': self.id, return {
'name': self.name, "id": self.id,
'starttime': self.starttime, "name": self.name,
'endtime': self.endtime, "starttime": self.starttime,
'filename': self.filename "endtime": self.endtime,
} "filename": self.filename,
}
def __repr__(self): def __repr__(self):
contents = "id:'%s',name:'%s',Start: '%s',End: '%s'" % \ contents = "id:'%s',name:'%s',Start: '%s',End: '%s'" % (
(self.id, self.name, self.starttime, self.endtime) self.id,
self.name,
self.starttime,
self.endtime,
)
if self.filename is not None: if self.filename is not None:
contents += ",Filename: '%s'" % self.filename contents += ",Filename: '%s'" % self.filename
return "<Rec(%s)>" % contents return "<Rec(%s)>" % contents
@ -58,12 +63,11 @@ class RecDB:
self.conn = self.engine.connect() self.conn = self.engine.connect()
self.log = logging.getLogger(name=self.__class__.__name__) self.log = logging.getLogger(name=self.__class__.__name__)
logging.getLogger('sqlalchemy.engine').setLevel(logging.FATAL) logging.getLogger("sqlalchemy.engine").setLevel(logging.FATAL)
logging.getLogger('sqlalchemy.engine.base.Engine')\ logging.getLogger("sqlalchemy.engine.base.Engine").setLevel(logging.FATAL)
.setLevel(logging.FATAL) logging.getLogger("sqlalchemy.dialects").setLevel(logging.FATAL)
logging.getLogger('sqlalchemy.dialects').setLevel(logging.FATAL) logging.getLogger("sqlalchemy.pool").setLevel(logging.FATAL)
logging.getLogger('sqlalchemy.pool').setLevel(logging.FATAL) logging.getLogger("sqlalchemy.orm").setLevel(logging.FATAL)
logging.getLogger('sqlalchemy.orm').setLevel(logging.FATAL)
Base.metadata.create_all(self.engine) # create Database Base.metadata.create_all(self.engine) # create Database
@ -74,17 +78,17 @@ class RecDB:
def add(self, simplerecord): def add(self, simplerecord):
s = self.get_session() s = self.get_session()
s.add( simplerecord ) s.add(simplerecord)
s.commit() s.commit()
self.log.info("New Record: %s" % simplerecord) self.log.info("New Record: %s" % simplerecord)
return ( simplerecord ) return simplerecord
def update(self, id, rec): def update(self, id, rec):
# TODO: rlist = results list # TODO: rlist = results list
_rlist = self._search(_id=id) _rlist = self._search(_id=id)
if not len(_rlist) == 1: if not len(_rlist) == 1:
raise ValueError('Too many recs with id=%s' % id) raise ValueError("Too many recs with id=%s" % id)
self.log.debug("DB:: Update request %s:%s " % (id, rec)) self.log.debug("DB:: Update request %s:%s " % (id, rec))
self.log.debug("DB:: Update: data before %s" % _rlist[0]) self.log.debug("DB:: Update: data before %s" % _rlist[0])
@ -92,7 +96,7 @@ class RecDB:
# 2013-11-24 22:22:42 # 2013-11-24 22:22:42
_rlist[0].starttime = rec["starttime"] _rlist[0].starttime = rec["starttime"]
_rlist[0].endtime = rec["endtime"] _rlist[0].endtime = rec["endtime"]
if 'name' in rec: if "name" in rec:
_rlist[0].name = rec["name"] _rlist[0].name = rec["name"]
self.log.debug("DB:: Update: data AFTER %s" % _rlist[0]) self.log.debug("DB:: Update: data AFTER %s" % _rlist[0])
@ -150,34 +154,34 @@ class RecDB:
return query.all() return query.all()
def _query_ongoing(self, query=None): def _query_ongoing(self, query=None):
''' """
Not terminated AND recent. Not terminated AND recent.
The meaning is "a query that makes sense to stop" The meaning is "a query that makes sense to stop"
''' """
delta = timedelta(seconds=get_config()['FORGE_MAX_DURATION']) delta = timedelta(seconds=get_config()["FORGE_MAX_DURATION"])
return self._query_newer(delta, self._query_not_saved(query)) return self._query_newer(delta, self._query_not_saved(query))
def _query_not_saved(self, query=None): def _query_not_saved(self, query=None):
'''Still not saved''' """Still not saved"""
if query is None: if query is None:
query = self.get_session().query(Rec) query = self.get_session().query(Rec)
return query.filter(Rec.filename == None) return query.filter(Rec.filename == None)
def _query_saved(self, query=None): def _query_saved(self, query=None):
'''Still not saved''' """Still not saved"""
if query is None: if query is None:
query = self.get_session().query(Rec) query = self.get_session().query(Rec)
return query.filter(Rec.filename != None) return query.filter(Rec.filename != None)
def _query_newer(self, delta, query=None): def _query_newer(self, delta, query=None):
'''Get Rec older than delta seconds''' """Get Rec older than delta seconds"""
if query is None: if query is None:
query = self.get_session().query(Rec) query = self.get_session().query(Rec)
return query.filter(Rec.starttime > datetime.now() - delta) return query.filter(Rec.starttime > datetime.now() - delta)
def _query_older(self, delta, query=None): def _query_older(self, delta, query=None):
'''Get Rec older than delta seconds''' """Get Rec older than delta seconds"""
if query is None: if query is None:
query = self.get_session().query(Rec) query = self.get_session().query(Rec)
return query.filter(Rec.starttime < datetime.now() - delta) return query.filter(Rec.starttime < datetime.now() - delta)
@ -187,15 +191,14 @@ class RecDB:
page_size = int(page_size) page_size = int(page_size)
query = query.limit(page_size) query = query.limit(page_size)
if page: if page:
query = query.offset(page*page_size) query = query.offset(page * page_size)
return query return query
def _query_generic(self, query, _id=None, name=None, starttime=None, def _query_generic(self, query, _id=None, name=None, starttime=None, endtime=None):
endtime=None):
if _id is not None: if _id is not None:
query = query.filter_by(id=_id) query = query.filter_by(id=_id)
if name is not None: if name is not None:
query = query.filter(Rec.name.like("%"+name+"%")) query = query.filter(Rec.name.like("%" + name + "%"))
if starttime is not None: if starttime is not None:
_st = starttime _st = starttime
query = query.filter(Rec.starttime > _st) query = query.filter(Rec.starttime > _st)
@ -204,15 +207,22 @@ class RecDB:
query = query.filter(Rec.endtime < _et) query = query.filter(Rec.endtime < _et)
return query return query
def _search(self, _id=None, name=None, starttime=None, def _search(
endtime=None, page=0, page_size=PAGESIZE): self,
_id=None,
name=None,
starttime=None,
endtime=None,
page=0,
page_size=PAGESIZE,
):
self.log.debug( self.log.debug(
"DB: Search => id:%s name:%s starttime:%s endtime=%s" % "DB: Search => id:%s name:%s starttime:%s endtime=%s"
(_id, name, starttime, endtime)) % (_id, name, starttime, endtime)
)
query = self.get_session().query(Rec) query = self.get_session().query(Rec)
query = self._query_generic(query, _id, name, starttime, query = self._query_generic(query, _id, name, starttime, endtime)
endtime)
query = self._query_page(query, page, page_size) query = self._query_page(query, page, page_size)
self.log.debug("Searching: %s" % str(query)) self.log.debug("Searching: %s" % str(query))
ret = query.all() ret = query.all()
@ -226,6 +236,7 @@ class RecDB:
if __name__ == "__main__": if __name__ == "__main__":
def printall(queryres): def printall(queryres):
for record in queryres: for record in queryres:
print("Record: %s" % record) print("Record: %s" % record)
@ -245,12 +256,11 @@ if __name__ == "__main__":
print("Mimmo ") print("Mimmo ")
printall(db._search(name="Mimmo1")) printall(db._search(name="Mimmo1"))
print("Search") print("Search")
printall(db._search(name="Mimmo1", printall(db._search(name="Mimmo1", starttime=datetime(2014, 5, 24, 15, 16, 1)))
starttime=datetime(2014, 5, 24, 15, 16, 1) ))
a = db.get_by_id(5) a = db.get_by_id(5)
a.start() a.start()
db.delete(1) db.delete(1)
db.delete(2) db.delete(2)
db.delete(4) db.delete(4)
db.delete(1) db.delete(1)
printall( db._search() ) printall(db._search())

43
server/default_config.py
View file

@ -1,37 +1,40 @@
import logging import logging
HOST = 'localhost'
PORT = '8000' HOST = "localhost"
PORT = "8000"
# pastelog is just "paste", but customized to accept logging options # pastelog is just "paste", but customized to accept logging options
WSGI_SERVER = 'pastelog' WSGI_SERVER = "pastelog"
# these are pastelog-specific options for logging engine # these are pastelog-specific options for logging engine
TRANSLOGGER_OPTS = { TRANSLOGGER_OPTS = {
'logger_name': 'accesslog', "logger_name": "accesslog",
'set_logger_level': logging.WARNING, "set_logger_level": logging.WARNING,
'setup_console_handler': False } "setup_console_handler": False,
}
WSGI_SERVER_OPTIONS = {} WSGI_SERVER_OPTIONS = {}
DEBUG = True DEBUG = True
DB_URI = 'sqlite:///techrec.db' DB_URI = "sqlite:///techrec.db"
AUDIO_OUTPUT = 'output/' AUDIO_OUTPUT = "output/"
AUDIO_INPUT = 'rec/' AUDIO_INPUT = "rec/"
AUDIO_INPUT_FORMAT = '%Y-%m/%d/rec-%Y-%m-%d-%H-%M-%S.mp3' AUDIO_INPUT_FORMAT = "%Y-%m/%d/rec-%Y-%m-%d-%H-%M-%S.mp3"
AUDIO_OUTPUT_FORMAT = 'techrec-%(startdt)s-%(endtime)s-%(name)s.mp3' AUDIO_OUTPUT_FORMAT = "techrec-%(startdt)s-%(endtime)s-%(name)s.mp3"
FORGE_TIMEOUT = 20 FORGE_TIMEOUT = 20
FORGE_MAX_DURATION = 3600*5 FORGE_MAX_DURATION = 3600 * 5
FFMPEG_OUT_CODEC = ['-acodec', 'copy'] FFMPEG_OUT_CODEC = ["-acodec", "copy"]
FFMPEG_OPTIONS = ['-loglevel', 'warning', '-n'] FFMPEG_OPTIONS = ["-loglevel", "warning", "-n"]
FFMPEG_PATH = 'ffmpeg' FFMPEG_PATH = "ffmpeg"
# tag:value pairs # tag:value pairs
TAG_EXTRA = {} TAG_EXTRA = {}
# LICENSE URI is special because date need to be added # LICENSE URI is special because date need to be added
TAG_LICENSE_URI = None TAG_LICENSE_URI = None
STATIC_FILES='static/' STATIC_FILES = "static/"
STATIC_PAGES='pages/' STATIC_PAGES = "pages/"
try: try:
from pkg_resources import resource_filename, resource_isdir from pkg_resources import resource_filename, resource_isdir
if resource_isdir('techrec', 'pages'):
STATIC_PAGES = resource_filename('techrec', 'pages') if resource_isdir("techrec", "pages"):
STATIC_FILES = resource_filename('techrec', 'static') STATIC_PAGES = resource_filename("techrec", "pages")
STATIC_FILES = resource_filename("techrec", "static")
except ImportError: except ImportError:
logging.exception("Error loading resources from installed part") logging.exception("Error loading resources from installed part")

90
server/forge.py
View file

@ -8,20 +8,19 @@ from .config_manager import get_config
def get_timefile_exact(time): def get_timefile_exact(time):
''' """
time is of type `datetime`; it is not "rounded" to match the real file; time is of type `datetime`; it is not "rounded" to match the real file;
that work is done in get_timefile(time) that work is done in get_timefile(time)
''' """
return os.path.join( return os.path.join(
get_config()['AUDIO_INPUT'], get_config()["AUDIO_INPUT"], time.strftime(get_config()["AUDIO_INPUT_FORMAT"])
time.strftime(get_config()['AUDIO_INPUT_FORMAT'])
) )
def round_timefile(exact): def round_timefile(exact):
''' """
This will round the datetime, so to match the file organization structure This will round the datetime, so to match the file organization structure
''' """
return datetime(exact.year, exact.month, exact.day, exact.hour) return datetime(exact.year, exact.month, exact.day, exact.hour)
@ -30,11 +29,11 @@ def get_timefile(exact):
def get_files_and_intervals(start, end, rounder=round_timefile): def get_files_and_intervals(start, end, rounder=round_timefile):
''' """
both arguments are datetime objects both arguments are datetime objects
returns an iterator whose elements are (filename, start_cut, end_cut) returns an iterator whose elements are (filename, start_cut, end_cut)
Cuts are expressed in seconds Cuts are expressed in seconds
''' """
if end <= start: if end <= start:
raise ValueError("end < start!") raise ValueError("end < start!")
@ -50,7 +49,7 @@ def get_files_and_intervals(start, end, rounder=round_timefile):
def mp3_join(named_intervals): def mp3_join(named_intervals):
''' """
Note that these are NOT the intervals returned by get_files_and_intervals, Note that these are NOT the intervals returned by get_files_and_intervals,
as they do not supply a filename, but only a datetime. as they do not supply a filename, but only a datetime.
What we want in input is basically the same thing, but with get_timefile() What we want in input is basically the same thing, but with get_timefile()
@ -58,8 +57,8 @@ def mp3_join(named_intervals):
This function make the (quite usual) assumption that the only start_cut (if This function make the (quite usual) assumption that the only start_cut (if
any) is at the first file, and the last one is at the last file any) is at the first file, and the last one is at the last file
''' """
ffmpeg = get_config()['FFMPEG_PATH'] ffmpeg = get_config()["FFMPEG_PATH"]
startskip = None startskip = None
endskip = None endskip = None
files = [] files = []
@ -72,65 +71,64 @@ def mp3_join(named_intervals):
if end_cut: if end_cut:
assert endskip is None assert endskip is None
endskip = end_cut endskip = end_cut
assert '|' not in filename assert "|" not in filename
files.append(filename) files.append(filename)
cmdline = [ffmpeg, '-i', 'concat:%s' % '|'.join(files)] cmdline = [ffmpeg, "-i", "concat:%s" % "|".join(files)]
cmdline += get_config()['FFMPEG_OUT_CODEC'] cmdline += get_config()["FFMPEG_OUT_CODEC"]
if startskip is not None: if startskip is not None:
cmdline += ['-ss', str(startskip)] cmdline += ["-ss", str(startskip)]
else: else:
startskip = 0 startskip = 0
if endskip is not None: if endskip is not None:
cmdline += ['-t', str(len(files)*3600 - (startskip + endskip))] cmdline += ["-t", str(len(files) * 3600 - (startskip + endskip))]
return cmdline return cmdline
def create_mp3(start, end, outfile, options={}, **kwargs): def create_mp3(start, end, outfile, options={}, **kwargs):
intervals = [(get_timefile(begin), start_cut, end_cut) intervals = [
for begin, start_cut, end_cut (get_timefile(begin), start_cut, end_cut)
in get_files_and_intervals(start, end)] for begin, start_cut, end_cut in get_files_and_intervals(start, end)
]
if os.path.exists(outfile): if os.path.exists(outfile):
raise OSError("file '%s' already exists" % outfile) raise OSError("file '%s' already exists" % outfile)
for path, _s, _e in intervals: for path, _s, _e in intervals:
if not os.path.exists(path): if not os.path.exists(path):
raise OSError("file '%s' does not exist; recording system broken?" raise OSError("file '%s' does not exist; recording system broken?" % path)
% path)
# metadata date/time formatted according to # metadata date/time formatted according to
# https://wiki.xiph.org/VorbisComment#Date_and_time # https://wiki.xiph.org/VorbisComment#Date_and_time
metadata = {} metadata = {}
if outfile.endswith('.mp3'): if outfile.endswith(".mp3"):
metadata['TRDC'] = start.replace(microsecond=0).isoformat() metadata["TRDC"] = start.replace(microsecond=0).isoformat()
metadata['RECORDINGTIME'] = metadata['TRDC'] metadata["RECORDINGTIME"] = metadata["TRDC"]
metadata['ENCODINGTIME'] = datetime.now().replace( metadata["ENCODINGTIME"] = datetime.now().replace(microsecond=0).isoformat()
microsecond=0).isoformat()
else: else:
metadata['DATE'] = start.replace(microsecond=0).isoformat() metadata["DATE"] = start.replace(microsecond=0).isoformat()
metadata['ENCODER'] = 'https://github.com/boyska/techrec' metadata["ENCODER"] = "https://github.com/boyska/techrec"
if 'title' in options: if "title" in options:
metadata['TITLE'] = options['title'] metadata["TITLE"] = options["title"]
if options.get('license_uri', None) is not None: if options.get("license_uri", None) is not None:
metadata['RIGHTS-DATE'] = start.strftime('%Y-%m') metadata["RIGHTS-DATE"] = start.strftime("%Y-%m")
metadata['RIGHTS-URI'] = options['license_uri'] metadata["RIGHTS-URI"] = options["license_uri"]
if 'extra_tags' in options: if "extra_tags" in options:
metadata.update(options['extra_tags']) metadata.update(options["extra_tags"])
metadata_list = [] metadata_list = []
for tag, value in metadata.items(): for tag, value in metadata.items():
if '=' in tag: if "=" in tag:
logging.error('Received a tag with "=" inside, skipping') logging.error('Received a tag with "=" inside, skipping')
continue continue
metadata_list.append('-metadata') metadata_list.append("-metadata")
metadata_list.append('%s=%s' % (tag, value)) metadata_list.append("%s=%s" % (tag, value))
p = Popen(mp3_join(intervals) + metadata_list + p = Popen(
get_config()['FFMPEG_OPTIONS'] + [outfile]) mp3_join(intervals) + metadata_list + get_config()["FFMPEG_OPTIONS"] + [outfile]
if get_config()['FORGE_TIMEOUT'] == 0: )
if get_config()["FORGE_TIMEOUT"] == 0:
p.wait() p.wait()
else: else:
start = datetime.now() start = datetime.now()
while (datetime.now() - start).total_seconds() \ while (datetime.now() - start).total_seconds() < get_config()["FORGE_TIMEOUT"]:
< get_config()['FORGE_TIMEOUT']:
p.poll() p.poll()
if p.returncode is None: if p.returncode is None:
sleep(1) sleep(1)
@ -142,14 +140,14 @@ def create_mp3(start, end, outfile, options={}, **kwargs):
os.remove(outfile) os.remove(outfile)
except: except:
pass pass
raise Exception('timeout') # TODO: make a specific TimeoutError raise Exception("timeout") # TODO: make a specific TimeoutError
if p.returncode != 0: if p.returncode != 0:
raise OSError("return code was %d" % p.returncode) raise OSError("return code was %d" % p.returncode)
return True return True
def main_cmd(options): def main_cmd(options):
log = logging.getLogger('forge_main') log = logging.getLogger("forge_main")
outfile = os.path.abspath(os.path.join(options.cwd, options.outfile)) outfile = os.path.abspath(os.path.join(options.cwd, options.outfile))
log.debug('will forge an mp3 into %s' % (outfile)) log.debug("will forge an mp3 into %s" % (outfile))
create_mp3(options.starttime, options.endtime, outfile) create_mp3(options.starttime, options.endtime, outfile)

8
server/maint.py
View file

@ -7,11 +7,12 @@ from sqlalchemy import inspect
from .config_manager import get_config from .config_manager import get_config
from .db import RecDB from .db import RecDB
def cleanold_cmd(options): def cleanold_cmd(options):
log = logging.getLogger('cleanold') log = logging.getLogger("cleanold")
log.debug("starting cleanold[%d]" % options.minage) log.debug("starting cleanold[%d]" % options.minage)
db = RecDB(get_config()['DB_URI']) db = RecDB(get_config()["DB_URI"])
res = db.get_not_completed(options.minage*3600*24) res = db.get_not_completed(options.minage * 3600 * 24)
count = len(res) count = len(res)
if options.pretend: if options.pretend:
for rec in res: for rec in res:
@ -25,4 +26,5 @@ def cleanold_cmd(options):
logging.info("Cleanold complete: %d deleted" % count) logging.info("Cleanold complete: %d deleted" % count)
sys.exit(0) sys.exit(0)
# vim: set ai ts=4 sw=4 et: # vim: set ai ts=4 sw=4 et:

20
server/processqueue.py
View file

@ -12,18 +12,18 @@ class JobQueue(object):
job_id = self.last_job_id job_id = self.last_job_id
def clean_jobs(res): def clean_jobs(res):
'''this callback will remove the job from the queue''' """this callback will remove the job from the queue"""
del self.jobs[job_id] del self.jobs[job_id]
self.jobs[job_id] = self.pool.apply_async(function, args, kwargs,
clean_jobs) self.jobs[job_id] = self.pool.apply_async(function, args, kwargs, clean_jobs)
return job_id return job_id
def check_job(self, job_id): def check_job(self, job_id):
''' """
If the job is running, return the asyncResult. If the job is running, return the asyncResult.
If it has already completed, returns True. If it has already completed, returns True.
If no such job_id exists at all, returns False If no such job_id exists at all, returns False
''' """
if job_id <= 0: if job_id <= 0:
raise ValueError("non-valid job_id") raise ValueError("non-valid job_id")
if self.last_job_id < job_id: if self.last_job_id < job_id:
@ -38,13 +38,16 @@ class JobQueue(object):
self.pool = None self.pool = None
def simulate_long_job(recid=None, starttime=None, endtime=None, name='', filename=None): def simulate_long_job(recid=None, starttime=None, endtime=None, name="", filename=None):
from time import sleep from time import sleep
print("evviva " + name) print("evviva " + name)
sleep(2) sleep(2)
print("lavoro su " + name) print("lavoro su " + name)
sleep(2) sleep(2)
print("done su " + name) print("done su " + name)
_queue = None _queue = None
@ -54,12 +57,15 @@ def get_process_queue():
_queue = JobQueue() _queue = JobQueue()
return _queue return _queue
if __name__ == '__main__':
if __name__ == "__main__":
from datetime import datetime from datetime import datetime
n = datetime.now() n = datetime.now()
def sleep(n): def sleep(n):
import time import time
print("Inizio %d" % n) print("Inizio %d" % n)
time.sleep(n) time.sleep(n)
print("Finisco %d" % n) print("Finisco %d" % n)

342
server/server.py
View file

@ -8,8 +8,8 @@ import unicodedata
from bottle import Bottle, request, static_file, redirect, abort, response from bottle import Bottle, request, static_file, redirect, abort, response
import bottle import bottle
logger = logging.getLogger('server') logger = logging.getLogger("server")
botlog = logging.getLogger('bottle') botlog = logging.getLogger("bottle")
botlog.setLevel(logging.INFO) botlog.setLevel(logging.INFO)
botlog.addHandler(logging.StreamHandler(sys.stdout)) botlog.addHandler(logging.StreamHandler(sys.stdout))
bottle._stderr = lambda x: botlog.info(x.strip()) bottle._stderr = lambda x: botlog.info(x.strip())
@ -25,47 +25,49 @@ def date_read(s):
def date_write(dt): def date_write(dt):
return dt.strftime('%s') return dt.strftime("%s")
def rec_sanitize(rec): def rec_sanitize(rec):
d = rec.serialize() d = rec.serialize()
d['starttime'] = date_write(d['starttime']) d["starttime"] = date_write(d["starttime"])
d['endtime'] = date_write(d['endtime']) d["endtime"] = date_write(d["endtime"])
return d return d
class DateApp(Bottle): class DateApp(Bottle):
''' """
This application will expose some date-related functions; it is intended to This application will expose some date-related functions; it is intended to
be used when you need to know the server's time on the browser be used when you need to know the server's time on the browser
''' """
def __init__(self): def __init__(self):
Bottle.__init__(self) Bottle.__init__(self)
self.route('/help', callback=self.help) self.route("/help", callback=self.help)
self.route('/date', callback=self.date) self.route("/date", callback=self.date)
self.route('/custom', callback=self.custom) self.route("/custom", callback=self.custom)
def date(self): def date(self):
n = datetime.now() n = datetime.now()
return { return {
'unix': n.strftime('%s'), "unix": n.strftime("%s"),
'isoformat': n.isoformat(), "isoformat": n.isoformat(),
'ctime': n.ctime() "ctime": n.ctime(),
} }
def custom(self): def custom(self):
n = datetime.now() n = datetime.now()
if 'strftime' not in request.query: if "strftime" not in request.query:
abort(400, 'Need argument "strftime"') abort(400, 'Need argument "strftime"')
response.content_type = 'text/plain' response.content_type = "text/plain"
return n.strftime(request.query['strftime']) return n.strftime(request.query["strftime"])
def help(self): def help(self):
response.content_type = 'text/plain' response.content_type = "text/plain"
return \ return (
'/date : get JSON dict containing multiple formats of now()\n' + \ "/date : get JSON dict containing multiple formats of now()\n"
'/custom?strftime=FORMAT : get now().strftime(FORMAT)' + "/custom?strftime=FORMAT : get now().strftime(FORMAT)"
)
class RecAPI(Bottle): class RecAPI(Bottle):
@ -73,20 +75,20 @@ class RecAPI(Bottle):
Bottle.__init__(self) Bottle.__init__(self)
self._route() self._route()
self._app = app self._app = app
self.db = RecDB(get_config()['DB_URI']) self.db = RecDB(get_config()["DB_URI"])
def _route(self): def _route(self):
self.post('/create', callback=self.create) self.post("/create", callback=self.create)
self.post('/delete', callback=self.delete) self.post("/delete", callback=self.delete)
self.post('/update/<recid:int>', callback=self.update) self.post("/update/<recid:int>", callback=self.update)
self.post('/generate', callback=self.generate) self.post("/generate", callback=self.generate)
self.get('/help', callback=self.help) self.get("/help", callback=self.help)
self.get('/', callback=self.help) self.get("/", callback=self.help)
self.get('/get/search', callback=self.search) self.get("/get/search", callback=self.search)
self.get('/get/ongoing', callback=self.get_ongoing) self.get("/get/ongoing", callback=self.get_ongoing)
self.get('/get/archive', callback=self.get_archive) self.get("/get/archive", callback=self.get_archive)
self.get('/jobs', callback=self.running_jobs) self.get("/jobs", callback=self.running_jobs)
self.get('/jobs/<job_id:int>', callback=self.check_job) self.get("/jobs/<job_id:int>", callback=self.check_job)
def create(self): def create(self):
req = dict(request.POST.decode().allitems()) req = dict(request.POST.decode().allitems())
@ -94,22 +96,21 @@ class RecAPI(Bottle):
logger.debug("Create request %s " % req) logger.debug("Create request %s " % req)
now = datetime.now() now = datetime.now()
start = date_read(req['starttime']) if 'starttime' in req else now start = date_read(req["starttime"]) if "starttime" in req else now
name = req['name'] if 'name' in req else u"" name = req["name"] if "name" in req else u""
end = date_read(req['endtime']) if 'endtime' in req else now end = date_read(req["endtime"]) if "endtime" in req else now
rec = Rec(name=name, rec = Rec(name=name, starttime=start, endtime=end)
starttime=start,
endtime=end)
ret = self.db.add(rec) ret = self.db.add(rec)
return self.rec_msg("Nuova registrazione creata! (id:%d)" % ret.id, return self.rec_msg(
rec=rec_sanitize(rec)) "Nuova registrazione creata! (id:%d)" % ret.id, rec=rec_sanitize(rec)
)
def delete(self): def delete(self):
req = dict(request.POST.decode().allitems()) req = dict(request.POST.decode().allitems())
logging.info("Server: request delete %s " % (req)) logging.info("Server: request delete %s " % (req))
if 'id' not in req: if "id" not in req:
return self.rec_err("No valid ID") return self.rec_err("No valid ID")
if self.db.delete(req["id"]): if self.db.delete(req["id"]):
@ -122,16 +123,16 @@ class RecAPI(Bottle):
newrec = {} newrec = {}
now = datetime.now() now = datetime.now()
if 'starttime' not in req: if "starttime" not in req:
newrec['starttime'] = now newrec["starttime"] = now
else: else:
newrec['starttime'] = date_read(req['starttime']) newrec["starttime"] = date_read(req["starttime"])
if "endtime" not in req: if "endtime" not in req:
newrec['endtime'] = now newrec["endtime"] = now
else: else:
newrec['endtime'] = date_read(req['endtime']) newrec["endtime"] = date_read(req["endtime"])
if 'name' in req: if "name" in req:
newrec["name"] = req['name'] newrec["name"] = req["name"]
try: try:
logger.info("prima di update") logger.info("prima di update")
@ -139,88 +140,101 @@ class RecAPI(Bottle):
logger.info("dopo update") logger.info("dopo update")
except Exception as exc: except Exception as exc:
return self.rec_err("Errore Aggiornamento", exception=exc) return self.rec_err("Errore Aggiornamento", exception=exc)
return self.rec_msg("Aggiornamento completato!", return self.rec_msg("Aggiornamento completato!", rec=rec_sanitize(result_rec))
rec=rec_sanitize(result_rec))
def generate(self): def generate(self):
# prendiamo la rec in causa # prendiamo la rec in causa
recid = dict(request.POST.decode().allitems())['id'] recid = dict(request.POST.decode().allitems())["id"]
rec = self.db._search(_id=recid)[0] rec = self.db._search(_id=recid)[0]
if rec.filename is not None and os.path.exists(rec.filename): if rec.filename is not None and os.path.exists(rec.filename):
return {'status': 'ready', return {
'message': 'The file has already been generated at %s' % "status": "ready",
rec.filename, "message": "The file has already been generated at %s" % rec.filename,
'rec': rec "rec": rec,
} }
if get_config()['FORGE_MAX_DURATION'] > 0 and \ if (
(rec.endtime - rec.starttime).total_seconds() > \ get_config()["FORGE_MAX_DURATION"] > 0
get_config()['FORGE_MAX_DURATION']: and (rec.endtime - rec.starttime).total_seconds()
response.status = 400 > get_config()["FORGE_MAX_DURATION"]
return {'status': 'error', ):
'message': 'The requested recording is too long' + response.status = 400
' (%d seconds)' % return {
(rec.endtime - rec.starttime).total_seconds() "status": "error",
} "message": "The requested recording is too long"
rec.filename = get_config()['AUDIO_OUTPUT_FORMAT'] % { + " (%d seconds)" % (rec.endtime - rec.starttime).total_seconds(),
'time': rec.starttime.strftime('%y%m%d_%H%M'), # kept for retrocompatibility, should be dropped }
'endtime': rec.endtime.strftime('%H%M'), rec.filename = get_config()["AUDIO_OUTPUT_FORMAT"] % {
'startdt': rec.starttime.strftime('%y%m%d_%H%M'), "time": rec.starttime.strftime(
'enddt': rec.endtime.strftime('%y%m%d_%H%M'), "%y%m%d_%H%M"
'name': ''.join(filter(lambda c: c.isalpha(), ), # kept for retrocompatibility, should be dropped
unicodedata.normalize('NFKD', rec.name).encode('ascii', 'ignore').decode('ascii'))), "endtime": rec.endtime.strftime("%H%M"),
"startdt": rec.starttime.strftime("%y%m%d_%H%M"),
"enddt": rec.endtime.strftime("%y%m%d_%H%M"),
"name": "".join(
filter(
lambda c: c.isalpha(),
unicodedata.normalize("NFKD", rec.name)
.encode("ascii", "ignore")
.decode("ascii"),
)
),
} }
self.db.get_session(rec).commit() self.db.get_session(rec).commit()
job_id = self._app.pq.submit( job_id = self._app.pq.submit(
create_mp3, create_mp3,
start=rec.starttime, start=rec.starttime,
end=rec.endtime, end=rec.endtime,
outfile=os.path.join(get_config()['AUDIO_OUTPUT'], rec.filename), outfile=os.path.join(get_config()["AUDIO_OUTPUT"], rec.filename),
options={ options={
'title': rec.name, "title": rec.name,
'license_uri': get_config()['TAG_LICENSE_URI'], "license_uri": get_config()["TAG_LICENSE_URI"],
'extra_tags': get_config()['TAG_EXTRA'] "extra_tags": get_config()["TAG_EXTRA"],
} },
) )
logger.debug("SUBMITTED: %d" % job_id) logger.debug("SUBMITTED: %d" % job_id)
return self.rec_msg("Aggiornamento completato!", return self.rec_msg(
job_id=job_id, "Aggiornamento completato!",
result='/output/' + rec.filename, job_id=job_id,
rec=rec_sanitize(rec)) result="/output/" + rec.filename,
rec=rec_sanitize(rec),
)
def check_job(self, job_id): def check_job(self, job_id):
try: try:
job = self._app.pq.check_job(job_id) job = self._app.pq.check_job(job_id)
except ValueError: except ValueError:
abort(400, 'job_id not valid') abort(400, "job_id not valid")
def ret(status): def ret(status):
return {'job_status': status, 'job_id': job_id} return {"job_status": status, "job_id": job_id}
if job is True: if job is True:
return ret('DONE') return ret("DONE")
if job is False: if job is False:
abort(404, 'No such job has ever been spawned') abort(404, "No such job has ever been spawned")
else: else:
if job.ready(): if job.ready():
try: try:
res = job.get() res = job.get()
return res return res
except Exception as exc: except Exception as exc:
r = ret('FAILED') r = ret("FAILED")
r['exception'] = str(exc) r["exception"] = str(exc)
import traceback import traceback
tb = traceback.format_exc() tb = traceback.format_exc()
logger.warning(tb) logger.warning(tb)
if get_config()['DEBUG']: if get_config()["DEBUG"]:
r['exception'] = "%s: %s" % (str(exc), tb) r["exception"] = "%s: %s" % (str(exc), tb)
r['traceback'] = tb r["traceback"] = tb
return r return r
return ret('WIP') return ret("WIP")
def running_jobs(self): def running_jobs(self):
res = {} res = {}
res['last_job_id'] = self._app.pq.last_job_id res["last_job_id"] = self._app.pq.last_job_id
res['running'] = self._app.pq.jobs.keys() res["running"] = self._app.pq.jobs.keys()
return res return res
def search(self, args=None): def search(self, args=None):
@ -230,8 +244,8 @@ class RecAPI(Bottle):
values = self.db._search(**req) values = self.db._search(**req)
from pprint import pprint from pprint import pprint
logger.debug("Returned Values %s" %
pprint([r.serialize() for r in values])) logger.debug("Returned Values %s" % pprint([r.serialize() for r in values]))
ret = {} ret = {}
for rec in values: for rec in values:
@ -241,12 +255,10 @@ class RecAPI(Bottle):
return ret return ret
def get_ongoing(self): def get_ongoing(self):
return {rec.id: rec_sanitize(rec) return {rec.id: rec_sanitize(rec) for rec in self.db.get_ongoing()}
for rec in self.db.get_ongoing()}
def get_archive(self): def get_archive(self):
return {rec.id: rec_sanitize(rec) return {rec.id: rec_sanitize(rec) for rec in self.db.get_archive_recent()}
for rec in self.db.get_archive_recent()}
# @route('/help') # @route('/help')
def help(self): def help(self):
@ -279,104 +291,126 @@ class RecServer:
self._app.pq = get_process_queue() self._app.pq = get_process_queue()
self._route() self._route()
self.db = RecDB(get_config()['DB_URI']) self.db = RecDB(get_config()["DB_URI"])
def _route(self): def _route(self):
# Static part of the site # Static part of the site
self._app.route('/output/<filepath:path>', self._app.route(
callback=lambda filepath: "/output/<filepath:path>",
static_file(filepath, callback=lambda filepath: static_file(
root=get_config()['AUDIO_OUTPUT'], filepath, root=get_config()["AUDIO_OUTPUT"], download=True
download=True)) ),
)
self._app.route('/static/<filepath:path>', self._app.route(
callback=lambda filepath: static_file(filepath, "/static/<filepath:path>",
root=get_config()['STATIC_FILES'])) callback=lambda filepath: static_file(
self._app.route('/', callback=lambda: redirect('/new.html')) filepath, root=get_config()["STATIC_FILES"]
self._app.route('/new.html', ),
callback=partial(static_file, 'new.html', )
root=get_config()['STATIC_PAGES'])) self._app.route("/", callback=lambda: redirect("/new.html"))
self._app.route('/old.html', self._app.route(
callback=partial(static_file, 'old.html', "/new.html",
root=get_config()['STATIC_PAGES'])) callback=partial(
self._app.route('/archive.html', static_file, "new.html", root=get_config()["STATIC_PAGES"]
callback=partial(static_file, 'archive.html', ),
root=get_config()['STATIC_PAGES'])) )
self._app.route(
"/old.html",
callback=partial(
static_file, "old.html", root=get_config()["STATIC_PAGES"]
),
)
self._app.route(
"/archive.html",
callback=partial(
static_file, "archive.html", root=get_config()["STATIC_PAGES"]
),
)
class DebugAPI(Bottle): class DebugAPI(Bottle):
''' """
This application is useful for testing the webserver itself This application is useful for testing the webserver itself
''' """
def __init__(self): def __init__(self):
Bottle.__init__(self) Bottle.__init__(self)
self.route('/sleep/:milliseconds', callback=self.sleep) self.route("/sleep/:milliseconds", callback=self.sleep)
self.route('/cpusleep/:howmuch', callback=self.cpusleep) self.route("/cpusleep/:howmuch", callback=self.cpusleep)
self.route('/big/:exponent', callback=self.big) self.route("/big/:exponent", callback=self.big)
def sleep(self, milliseconds): def sleep(self, milliseconds):
import time import time
time.sleep(int(milliseconds)/1000.0)
return 'ok' time.sleep(int(milliseconds) / 1000.0)
return "ok"
def cpusleep(self, howmuch): def cpusleep(self, howmuch):
out = '' out = ""
for i in xrange(int(howmuch) * (10**3)): for i in xrange(int(howmuch) * (10 ** 3)):
if i % 11234 == 0: if i % 11234 == 0:
out += 'a' out += "a"
return out return out
def big(self, exponent): def big(self, exponent):
''' """
returns a 2**n -1 string returns a 2**n -1 string
''' """
for i in xrange(int(exponent)): for i in xrange(int(exponent)):
yield str(i) * (2 ** i) yield str(i) * (2 ** i)
def help(self): def help(self):
response.content_type = 'text/plain' response.content_type = "text/plain"
return ''' return """
/sleep/<int:milliseconds> : sleep, than say "ok" /sleep/<int:milliseconds> : sleep, than say "ok"
/cpusleep/<int:howmuch> : busysleep, than say "ok" /cpusleep/<int:howmuch> : busysleep, than say "ok"
/big/<int:exponent> : returns a 2**n -1 byte content /big/<int:exponent> : returns a 2**n -1 byte content
''' """
class PasteLoggingServer(bottle.PasteServer): class PasteLoggingServer(bottle.PasteServer):
def run(self, handler): # pragma: no cover def run(self, handler): # pragma: no cover
from paste import httpserver from paste import httpserver
from paste.translogger import TransLogger from paste.translogger import TransLogger
handler = TransLogger(handler, **self.options['translogger_opts'])
del self.options['translogger_opts'] handler = TransLogger(handler, **self.options["translogger_opts"])
httpserver.serve(handler, host=self.host, port=str(self.port), del self.options["translogger_opts"]
**self.options) httpserver.serve(handler, host=self.host, port=str(self.port), **self.options)
bottle.server_names['pastelog'] = PasteLoggingServer
bottle.server_names["pastelog"] = PasteLoggingServer
def main_cmd(*args): def main_cmd(*args):
"""meant to be called from argparse""" """meant to be called from argparse"""
c = RecServer() c = RecServer()
c._app.mount('/date', DateApp()) c._app.mount("/date", DateApp())
c._app.mount('/api', RecAPI(c._app)) c._app.mount("/api", RecAPI(c._app))
if get_config()['DEBUG']: if get_config()["DEBUG"]:
c._app.mount('/debug', DebugAPI()) c._app.mount("/debug", DebugAPI())
server = get_config()['WSGI_SERVER'] server = get_config()["WSGI_SERVER"]
if server == 'pastelog': if server == "pastelog":
from paste.translogger import TransLogger from paste.translogger import TransLogger
get_config()['WSGI_SERVER_OPTIONS']['translogger_opts'] = \
get_config()['TRANSLOGGER_OPTS']
c._app.run(server=server, get_config()["WSGI_SERVER_OPTIONS"]["translogger_opts"] = get_config()[
host=get_config()['HOST'], "TRANSLOGGER_OPTS"
port=get_config()['PORT'], ]
debug=get_config()['DEBUG'],
quiet=True, # this is to hide access.log style messages
**get_config()['WSGI_SERVER_OPTIONS']
)
if __name__ == '__main__': c._app.run(
server=server,
host=get_config()["HOST"],
port=get_config()["PORT"],
debug=get_config()["DEBUG"],
quiet=True, # this is to hide access.log style messages
**get_config()["WSGI_SERVER_OPTIONS"]
)
if __name__ == "__main__":
from cli import common_pre from cli import common_pre
common_pre() common_pre()
logger.warn("Usage of server.py is deprecated; use cli.py") logger.warn("Usage of server.py is deprecated; use cli.py")
main_cmd() main_cmd()

85
server/test_forge.py
View file

@ -2,18 +2,23 @@ from datetime import datetime, timedelta
from nose.tools import raises, eq_ from nose.tools import raises, eq_
from .forge import get_files_and_intervals, get_timefile_exact, round_timefile,\ from .forge import (
get_timefile, mp3_join get_files_and_intervals,
get_timefile_exact,
round_timefile,
get_timefile,
mp3_join,
)
from .config_manager import get_config from .config_manager import get_config
eight = datetime(2014, 5, 30, 20) eight = datetime(2014, 5, 30, 20)
nine = datetime(2014, 5, 30, 21) nine = datetime(2014, 5, 30, 21)
ten = datetime(2014, 5, 30, 22) ten = datetime(2014, 5, 30, 22)
get_config()['AUDIO_INPUT'] = '' get_config()["AUDIO_INPUT"] = ""
get_config()['AUDIO_INPUT_FORMAT'] = '%Y-%m/%d/%Y-%m-%d-%H-%M-%S.mp3' get_config()["AUDIO_INPUT_FORMAT"] = "%Y-%m/%d/%Y-%m-%d-%H-%M-%S.mp3"
get_config()['FFMPEG_PATH'] = 'ffmpeg' get_config()["FFMPEG_PATH"] = "ffmpeg"
get_config()['FFMPEG_OUT_CODEC'] = ['-acodec', 'copy'] get_config()["FFMPEG_OUT_CODEC"] = ["-acodec", "copy"]
def minutes(n): def minutes(n):
@ -23,18 +28,19 @@ def minutes(n):
def seconds(n): def seconds(n):
return timedelta(seconds=n) return timedelta(seconds=n)
# timefile # timefile
def test_timefile_exact(): def test_timefile_exact():
eq_(get_timefile_exact(eight), eq_(get_timefile_exact(eight), "2014-05/30/2014-05-30-20-00-00.mp3")
'2014-05/30/2014-05-30-20-00-00.mp3')
# Rounding # Rounding
def test_rounding_similarity(): def test_rounding_similarity():
eq_(round_timefile(eight), round_timefile(eight+minutes(20))) eq_(round_timefile(eight), round_timefile(eight + minutes(20)))
assert round_timefile(eight) != round_timefile(nine) assert round_timefile(eight) != round_timefile(nine)
@ -47,13 +53,12 @@ def test_rounding_value():
def test_timefile_alreadyround(): def test_timefile_alreadyround():
eq_(get_timefile(eight), eq_(get_timefile(eight), "2014-05/30/2014-05-30-20-00-00.mp3")
'2014-05/30/2014-05-30-20-00-00.mp3')
def test_timefile_toround(): def test_timefile_toround():
eq_(get_timefile(eight + minutes(20)), eq_(get_timefile(eight + minutes(20)), "2014-05/30/2014-05-30-20-00-00.mp3")
'2014-05/30/2014-05-30-20-00-00.mp3')
# Intervals # Intervals
@ -69,17 +74,17 @@ def test_intervals_before():
def test_intervals_full_1(): def test_intervals_full_1():
res = list(get_files_and_intervals(eight, nine-seconds(1))) res = list(get_files_and_intervals(eight, nine - seconds(1)))
eq_(len(res), 1) eq_(len(res), 1)
eq_(res[0][1], 0) eq_(res[0][1], 0)
eq_(res[0][2], 0) eq_(res[0][2], 0)
def test_intervals_partial_1(): def test_intervals_partial_1():
res = list(get_files_and_intervals(eight, nine-minutes(10))) res = list(get_files_and_intervals(eight, nine - minutes(10)))
eq_(len(res), 1) eq_(len(res), 1)
eq_(res[0][1], 0) eq_(res[0][1], 0)
eq_(res[0][2], 10*60 - 1) eq_(res[0][2], 10 * 60 - 1)
def test_intervals_exact_2(): def test_intervals_exact_2():
@ -101,8 +106,7 @@ def test_intervals_partial_2():
def test_intervals_full_2(): def test_intervals_full_2():
res = list(get_files_and_intervals(eight, res = list(get_files_and_intervals(eight, nine + minutes(59) + seconds(59)))
nine + minutes(59) + seconds(59)))
eq_(len(res), 2) eq_(len(res), 2)
eq_(res[0][1], 0) eq_(res[0][1], 0)
eq_(res[0][2], 0) eq_(res[0][2], 0)
@ -122,7 +126,7 @@ def test_intervals_exact_3():
def test_intervals_partial_3(): def test_intervals_partial_3():
res = list(get_files_and_intervals(eight, ten+minutes(50))) res = list(get_files_and_intervals(eight, ten + minutes(50)))
eq_(len(res), 3) eq_(len(res), 3)
eq_(res[0][1], 0) eq_(res[0][1], 0)
eq_(res[0][2], 0) eq_(res[0][2], 0)
@ -133,7 +137,7 @@ def test_intervals_partial_3():
def test_intervals_full_3(): def test_intervals_full_3():
res = list(get_files_and_intervals(eight, ten+minutes(59) + seconds(59))) res = list(get_files_and_intervals(eight, ten + minutes(59) + seconds(59)))
eq_(len(res), 3) eq_(len(res), 3)
eq_(res[0][1], 0) eq_(res[0][1], 0)
eq_(res[0][2], 0) eq_(res[0][2], 0)
@ -144,49 +148,54 @@ def test_intervals_full_3():
def test_intervals_middle_1(): def test_intervals_middle_1():
res = list(get_files_and_intervals(eight + minutes(20), res = list(get_files_and_intervals(eight + minutes(20), nine - minutes(20)))
nine - minutes(20)))
eq_(len(res), 1) eq_(len(res), 1)
eq_(res[0][1], 20*60) eq_(res[0][1], 20 * 60)
eq_(res[0][2], 20*60-1) eq_(res[0][2], 20 * 60 - 1)
def test_intervals_left_2(): def test_intervals_left_2():
res = list(get_files_and_intervals(eight+minutes(30), nine)) res = list(get_files_and_intervals(eight + minutes(30), nine))
eq_(len(res), 2) eq_(len(res), 2)
eq_(res[0][1], 30*60) eq_(res[0][1], 30 * 60)
eq_(res[0][2], 0) eq_(res[0][2], 0)
eq_(res[1][1], 0) eq_(res[1][1], 0)
eq_(res[1][2], 3599) eq_(res[1][2], 3599)
# MP3 Join # MP3 Join
def test_mp3_1(): def test_mp3_1():
eq_(' '.join(mp3_join((('a', 0, 0),))), eq_(" ".join(mp3_join((("a", 0, 0),))), "ffmpeg -i concat:a -acodec copy")
'ffmpeg -i concat:a -acodec copy')
def test_mp3_1_left(): def test_mp3_1_left():
eq_(' '.join(mp3_join((('a', 160, 0),))), eq_(" ".join(mp3_join((("a", 160, 0),))), "ffmpeg -i concat:a -acodec copy -ss 160")
'ffmpeg -i concat:a -acodec copy -ss 160')
def test_mp3_1_right(): def test_mp3_1_right():
eq_(' '.join(mp3_join((('a', 0, 1600),))), eq_(
'ffmpeg -i concat:a -acodec copy -t 2000') " ".join(mp3_join((("a", 0, 1600),))), "ffmpeg -i concat:a -acodec copy -t 2000"
)
def test_mp3_1_leftright(): def test_mp3_1_leftright():
eq_(' '.join(mp3_join((('a', 160, 1600),))), eq_(
'ffmpeg -i concat:a -acodec copy -ss 160 -t 1840') " ".join(mp3_join((("a", 160, 1600),))),
"ffmpeg -i concat:a -acodec copy -ss 160 -t 1840",
)
def test_mp3_2(): def test_mp3_2():
eq_(' '.join(mp3_join((('a', 0, 0), ('b', 0, 0)))), eq_(
'ffmpeg -i concat:a|b -acodec copy') " ".join(mp3_join((("a", 0, 0), ("b", 0, 0)))),
"ffmpeg -i concat:a|b -acodec copy",
)
def test_mp3_2_leftright(): def test_mp3_2_leftright():
eq_(' '.join(mp3_join((('a', 1000, 0), ('b', 0, 1600)))), eq_(
'ffmpeg -i concat:a|b -acodec copy -ss 1000 -t 4600') " ".join(mp3_join((("a", 1000, 0), ("b", 0, 1600)))),
"ffmpeg -i concat:a|b -acodec copy -ss 1000 -t 4600",
)

7
setup.py
View file

@ -4,7 +4,7 @@ from setuptools import setup
setup( setup(
name="techrec", name="techrec",
version="1.1.3", version="1.2.0",
description="A Python2 web application " description="A Python2 web application "
"that assist radio speakers in recording their shows", "that assist radio speakers in recording their shows",
long_description=open("README.md").read(), long_description=open("README.md").read(),
@ -14,7 +14,10 @@ setup(
packages=["techrec"], packages=["techrec"],
package_dir={"techrec": "server"}, package_dir={"techrec": "server"},
install_requires=["Paste~=3.2", "SQLAlchemy==0.8.3", "bottle~=0.12"], install_requires=["Paste~=3.2", "SQLAlchemy==0.8.3", "bottle~=0.12"],
classifiers=["Programming Language :: Python :: 2.7"], classifiers=[
"Programming Language :: Python :: 2.7",
"Programming Language :: Python :: 3.7",
],
entry_points={"console_scripts": ["techrec = techrec.cli:main"]}, entry_points={"console_scripts": ["techrec = techrec.cli:main"]},
zip_safe=False, zip_safe=False,
install_package_data=True, install_package_data=True,