FIX exception reporting causing another exception

larigira/audiogen_podcast.py

@@ -142,16 +142,21 @@ def get_audio_from_item(item):
 
 def get_urls(tree):
     items = tree.xpath("//item")
-    for it in items:
+    for i, it in enumerate(items):
-        # title = it.find("title").text
+        try:
             audio = get_audio_from_item(it)
+        except Exception:
+            logging.error("Could not parse item #%d, skipping", i)
+            continue
         if audio is None:
             continue
         if audio.date is None:
+            try:
                 audio.date = get_item_date(it)
+            except Exception:
+                logging.warn("Could not find date for item #%d", i)
         yield audio
 
-
 def parse_duration(arg):
     if arg.isdecimal():
         secs = int(arg)

larigira/config.py

@@ -19,7 +19,6 @@ def get_conf(prefix="LARIGIRA_"):
     conf["UMASK"] = None
     conf["CACHING_TIME"] = 10
     conf["DB_URI"] = os.path.join(conf_dir, "db.json")
-    conf["DB_ADDITIONAL_DIR"] = os.path.join(conf_dir, "db.d")
     conf["SCRIPTS_PATH"] = os.path.join(conf_dir, "scripts")
     conf["EXTRA_STATIC_PATH"] = os.path.join(conf_dir, "extra")
     conf["EXTRA_MENU_LINKS"] = []

larigira/db.py

@@ -1,109 +1,24 @@
-from logging import getLogger
 from tinydb import TinyDB
-from tinydb.storages import JSONStorage
-from tinydb.middlewares import Middleware
-from pathlib import Path
-from typing import Union, Tuple
-
-
-class ReadOnlyMiddleware(Middleware):
-    """
-    Make sure no write ever occurs
-    """
-
-    def __init__(self, storage_cls=TinyDB.DEFAULT_STORAGE):
-        super().__init__(storage_cls)
-
-
-    def write(self, data):
-        raise ReadOnlyException('You cannot write to a readonly db')
-
-
-class ReadOnlyException(ValueError):
-    pass
-
-def only_main(f):
-    '''assumes first argument is id, and must be "main"'''
-    def wrapper(self, *args, **kwargs):
-        _id = args[0]
-        db, db_id = EventModel.parse_id(_id)
-        if db != 'main':
-            raise ReadOnlyException('You called a write operation on a readonly db')
-        return f(self, db_id, *args[1:], **kwargs)
-    return wrapper
 
 
 class EventModel(object):
-    def __init__(self, uri, additional_db_dir=None):
+    def __init__(self, uri):
         self.uri = uri
-        self.additional_db_dir = Path(additional_db_dir) if additional_db_dir else None
+        self.db = None
-        self._dbs = {}
-        self.log = getLogger(self.__class__.__name__)
         self.reload()
 
     def reload(self):
-        for db in self._dbs.values():
+        if self.db is not None:
-            db.close()
+            self.db.close()
-        self._dbs['main'] = TinyDB(self.uri, indent=2)
+        self.db = TinyDB(self.uri, indent=2)
-        if self.additional_db_dir is not None:
+        self._actions = self.db.table("actions")
-            if self.additional_db_dir.is_dir():
+        self._alarms = self.db.table("alarms")
-                for db_file in self.additional_db_dir.glob('*.db.json'):
-                    name = db_file.name[:-8]
-                    if name == 'main':
-                        self.log.warning("%s db file name is not valid (any other name.db.json would have been ok!", str(db_file.name))
-                        continue
-                    if not name.isalpha():
-                        self.log.warning("%s db file name is not valid: it must be alphabetic only", str(db_file.name))
-                        continue
-                    try:
-                        self._dbs[name] = TinyDB(
-                                str(db_file),
-                                storage=ReadOnlyMiddleware(JSONStorage),
-                                default_table='actions'
-                        )
-                    except ReadOnlyException:
-                        # TinyDB adds the default_table if it is not present at read time.
-                        # This should not happen at all for a ReadOnlyMiddleware db, but at least we can notice it and
-                        # properly signal this to the user.
-                        self.log.error("Could not load db %s: 'actions' table is missing", db_file.name)
-                        continue
 
-        self.log.debug('Loaded %d databases: %s', len(self._dbs), ','.join(self._dbs.keys()))
+    def get_action_by_id(self, action_id):
-
+        return self._actions.get(eid=action_id)
-        self._actions = self._dbs['main'].table("actions")
-        self._alarms = self._dbs['main'].table("alarms")
-
-    @staticmethod
-    def canonicalize(eid_or_aid: Union[str, int]) -> str:
-        try:
-            int(eid_or_aid)
-        except ValueError:
-            return eid_or_aid
-        return 'main:%d' % int(eid_or_aid)
-
-    @staticmethod
-    def parse_id(eid_or_aid: Union[str, int]) -> Tuple[str, int]:
-        try:
-            int(eid_or_aid)
-        except ValueError:
-            pass
-        else:
-            return ('main', int(eid_or_aid))
-
-        dbname, num = eid_or_aid.split(':')
-        return (dbname, int(num))
-        
-
-    def get_action_by_id(self, action_id: Union[str, int]):
-        canonical = self.canonicalize(action_id)
-        db, db_action_id = self.__class__.parse_id(canonical)
-        out = self._dbs[db].table('actions').get(eid=db_action_id)
-        out.doc_id = canonical
-        return out
 
     def get_alarm_by_id(self, alarm_id):
-        db, alarm_id = self.__class__.parse_id(alarm_id)
+        return self._alarms.get(eid=alarm_id)
-        return self._dbs[db].table('alarms').get(eid=alarm_id)
 
     def get_actions_by_alarm(self, alarm):
         for action_id in alarm.get("actions", []):
@@ -112,21 +27,11 @@ class EventModel(object):
                 continue
             yield action
 
-    def get_all_alarms(self) -> list:
+    def get_all_alarms(self):
-        out = []
+        return self._alarms.all()
-        for db in self._dbs:
-            for alarm in self._dbs[db].table('alarms').all():
-                alarm.doc_id = '%s:%s' % (db, alarm.doc_id)
-                out.append(alarm)
-        return out
 
-    def get_all_actions(self) -> list:
+    def get_all_actions(self):
-        out = []
+        return self._actions.all()
-        for db in self._dbs:
-            for action in self._dbs[db].table('actions').all():
-                action.doc_id = '%s:%s' % (db, action.doc_id)
-                out.append(action)
-        return out
 
     def get_all_alarms_expanded(self):
         for alarm in self.get_all_alarms():
@@ -144,18 +49,14 @@ class EventModel(object):
     def add_alarm(self, alarm):
         return self.add_event(alarm, [])
 
-    @only_main
     def update_alarm(self, alarmid, new_fields={}):
         return self._alarms.update(new_fields, eids=[alarmid])
 
-    @only_main
     def update_action(self, actionid, new_fields={}):
         return self._actions.update(new_fields, eids=[actionid])
 
-    @only_main
     def delete_alarm(self, alarmid):
         return self._alarms.remove(eids=[alarmid])
 
-    @only_main
     def delete_action(self, actionid):
         return self._actions.remove(eids=[actionid])

larigira/dbadmin/__init__.py

@@ -110,7 +110,7 @@ def addtime():
     return render_template("add_time.html", kinds=kinds, info=info)
 
 
-@db.route("/edit/time/<alarmid>", methods=["GET", "POST"])
+@db.route("/edit/time/<int:alarmid>", methods=["GET", "POST"])
 def edit_time(alarmid):
     model = get_model()
     timespec = model.get_alarm_by_id(alarmid)
@@ -124,7 +124,7 @@ def edit_time(alarmid):
         model.update_alarm(alarmid, data)
         model.reload()
         return redirect(
-            url_for("db.events_calendar", highlight=model.canonicalize(alarmid))
+            url_for("db.events_calendar", highlight="%d" % alarmid)
         )
     return render_template(
         "add_time_kind.html",
@@ -197,7 +197,7 @@ def addaudio_kind(kind):
     )
 
 
-@db.route("/edit/audio/<actionid>", methods=["GET", "POST"])
+@db.route("/edit/audio/<int:actionid>", methods=["GET", "POST"])
 def edit_audio(actionid):
     model = get_model()
     audiospec = model.get_action_by_id(actionid)
@@ -223,7 +223,7 @@ def edit_audio(actionid):
 @db.route("/edit/event/<alarmid>")
 def edit_event(alarmid):
     model = current_app.larigira.controller.monitor.model
-    alarm = model.get_alarm_by_id(alarmid)
+    alarm = model.get_alarm_by_id(int(alarmid))
     if alarm is None:
         abort(404)
     allactions = model.get_all_actions()
@@ -245,16 +245,16 @@ def change_actions(alarmid):
         new_actions = []
     model = current_app.larigira.controller.monitor.model
     ret = model.update_alarm(
-        alarmid, new_fields={"actions": [a for a in new_actions]}
+        int(alarmid), new_fields={"actions": [int(a) for a in new_actions]}
     )
     return jsonify(dict(updated=alarmid, ret=ret))
 
 
-@db.route("/api/alarm/<alarmid>/delete", methods=["POST"])
+@db.route("/api/alarm/<int:alarmid>/delete", methods=["POST"])
 def delete_alarm(alarmid):
     model = current_app.larigira.controller.monitor.model
     try:
-        alarm = model.get_alarm_by_id(alarmid)
+        alarm = model.get_alarm_by_id(int(alarmid))
         print(alarm["nick"])
         model.delete_alarm(alarmid)
     except KeyError:

larigira/event.py

@@ -42,7 +42,7 @@ class Monitor(ParentedLet):
         self.running = {}
         self.conf = conf
         self.q = Queue()
-        self.model = EventModel(self.conf["DB_URI"], self.conf['DB_ADDITIONAL_DIR'])
+        self.model = EventModel(self.conf["DB_URI"])
         self.ticker = Timer(int(self.conf["EVENT_TICK_SECS"]) * 1000, self.q)
 
     def _alarm_missing_time(self, timespec):

larigira/event_manage.py

@@ -8,19 +8,19 @@ from .config import get_conf
 
 
 def main_list(args):
-    m = EventModel(args.file, args.additional_dir)
+    m = EventModel(args.file)
     for alarm, action in m.get_all_alarms_expanded():
         json.dump(dict(alarm=alarm, action=action), sys.stdout, indent=4)
         sys.stdout.write('\n')
 
 
 def main_getaction(args):
-    m = EventModel(args.file, args.additional_dir)
+    m = EventModel(args.file)
-    json.dump(m.get_action_by_id(args.actionid), sys.stdout, indent=4)
+    json.dump(m.get_action_by_id(int(args.actionid)), sys.stdout, indent=4)
 
 
 def main_add(args):
-    m = EventModel(args.file, args.additional_dir)
+    m = EventModel(args.file)
     m.add_event(
         dict(kind="frequency", interval=args.interval, start=1),
         [dict(kind="mpd", howmany=1)],
@@ -29,14 +29,11 @@ def main_add(args):
 
 def main():
     conf = get_conf()
-    p = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    p = argparse.ArgumentParser()
     p.set_defaults(func=None)
     p.add_argument(
         "-f", "--file", help="Filepath for DB", required=False, default=conf["DB_URI"]
     )
-    p.add_argument(
-        "-d", "--additional-dir", help="Filepath for extra DBs", required=False, default=conf["DB_ADDITIONAL_DIR"]
-    )
     sub = p.add_subparsers()
     sub_list = sub.add_parser("list")
     sub_list.set_defaults(func=main_list)

larigira/filters/basic.py

@@ -68,7 +68,7 @@ def percentwait(songs, context, conf, getdur=get_duration):
         # must be an error! mutagen support is not always perfect
         return (
             True,
-            ("mutagen could not calculate length of %s" % ",".songs["uris"]),
+            ("mutagen could not calculate length of %s" % ",".join(songs["uris"])),
         )
     wait = eventduration * (percentwait / 100.0)
     if remaining > wait:

larigira/fsutils.py

@@ -6,10 +6,6 @@ import posixpath
 import urllib.request
 from tempfile import mkstemp
 from urllib.parse import urlparse
-from pathlib import Path
-import hashlib
-
-import requests
 
 log = logging.getLogger(__name__)
 
@@ -50,14 +46,6 @@ def shortname(path):
     return name
 
 
-def http_expected_length(url):
-    resp = requests.head(url, allow_redirects=True)
-    resp.raise_for_status()
-    header_value = resp.headers.get('content-length')
-    expected_length = int(header_value)
-    return expected_length
-
-
 def download_http(url, destdir=None, copy=False, prefix="httpdl"):
     if url.split(":")[0] not in ("http", "https"):
         log.warning("Not a valid URL: %s", url)
@@ -68,43 +56,15 @@ def download_http(url, destdir=None, copy=False, prefix="httpdl"):
         return None
     if not copy:
         return url
-    if destdir is None:
-        destdir = os.getenv('TMPDIR', '/tmp/')
     fname = posixpath.basename(urlparse(url).path)
     # sanitize
     fname = "".join(
-        c for c in fname if c.isalnum() or c in list("_-")
+        c for c in fname if c.isalnum() or c in list("._-")
     ).rstrip()
-    url_hash = hashlib.sha1(url.encode('utf8')).hexdigest()
-
-    final_path = Path(destdir) / ('%s-%s-%s.%s' % (prefix, fname[:20], url_hash, ext))
-
-    # it might be already fully downloaded, let's check
-    if final_path.exists():
-
-        # this "touch" helps avoiding a race condition in which the
-        # UnusedCleaner could delete  this
-        final_path.touch()
-
-        actual_size = final_path.stat().st_size
-        try:
-            expected_size = http_expected_length(url)
-        except Exception as exc:
-            log.debug("Could not determine expected length for %s: %s", url, exc)
-        else:
-            if expected_size == actual_size:
-                log.debug("File %s already present and complete, download not needed", final_path)
-                return final_path.as_uri()
-            else:
-                log.debug("File %s is already present, but has the wrong length: %d but expected %d", final_path, actual_size, expected_size)
-    else:
-        log.debug("File %s does not exist", final_path)
     tmp = mkstemp(
-        suffix="." + ext, prefix="%s-%s-%s-" % (prefix, fname, url_hash), dir=destdir
+        suffix="." + ext, prefix="%s-%s-" % (prefix, fname), dir=destdir
     )
     os.close(tmp[0])
-    log.info("downloading %s -> %s -> %s", url, tmp[1], final_path)
+    log.info("downloading %s -> %s", url, tmp[1])
     fname, headers = urllib.request.urlretrieve(url, tmp[1])
-    Path(fname).rename(final_path)
+    return "file://%s" % os.path.realpath(tmp[1])
-    return final_path.as_uri()
-# "file://%s" % os.path.realpath(final_path)

larigira/static/calendar.js

@@ -17,7 +17,7 @@ jQuery(function ($) {
     var content = $('<div/>').append(
       $('<p/>').append($('<a class="btn btn-default"/>').text('Modifica orario evento').attr('href', 'edit/time/' + alarmid))
     )
-    if (actions.toString().indexOf(',') === -1) { // single one
+    if (Number.isInteger(actions)) { // else, it's a string representing a list
       content.append($('<p/>').append(
       $('<a class="btn btn-default"/>')
       .text('Modifica audio evento')

larigira/unused.py

@@ -7,8 +7,6 @@ This component will look for files to be removed. There are some assumptions:
 import logging
 import os
 from os.path import normpath
-from pathlib import Path
-import time
 
 import mpd
 
@@ -32,10 +30,6 @@ except ImportError:
 
 
 class UnusedCleaner:
-    # ONLY_DELETE_OLDER_THAN is expressed in seconds.
-    # It configures the maximum age a file can have before being removed.
-    # Set it to "None" if you want to disable this feature.
-    ONLY_DELETE_OLDER_THAN = 30
     def __init__(self, conf):
         self.conf = conf
         self.waiting_removal_files = set()
@@ -75,19 +69,7 @@ class UnusedCleaner:
             for song in mpdc.playlistid()
             if song["file"].startswith("/")
         }
-        now = time.time()
         for fpath in self.waiting_removal_files - files_in_playlist:
-
-            # audio files are sometimes reused, as in download_http. To avoid
-            # referencing a file that UnusedCleaner is going to remove, users
-            # are invited to touch the file, so that UnusedCleaner doesn't
-            # consider it for removal. While this doesn't conceptually solve
-            # the race condition, it should now be extremely rare.
-
-            if ONLY_DELETE_OLDER_THAN is not None:
-                mtime = Path(fpath).stat().st_mtime
-                if now - mtime < ONLY_DELETE_OLDER_THAN:
-                    continue
             # we can remove it!
             self.log.debug("removing unused: %s", fpath)
             self.waiting_removal_files.remove(fpath)