canonicalize ids here and there

refs #19

larigira/config.py

@@ -19,6 +19,7 @@ def get_conf(prefix="LARIGIRA_"):
     conf["UMASK"] = None
     conf["CACHING_TIME"] = 10
     conf["DB_URI"] = os.path.join(conf_dir, "db.json")
+    conf["DB_ADDITIONAL_DIR"] = os.path.join(conf_dir, "db.d")
     conf["SCRIPTS_PATH"] = os.path.join(conf_dir, "scripts")
     conf["EXTRA_STATIC_PATH"] = os.path.join(conf_dir, "extra")
     conf["EXTRA_MENU_LINKS"] = []

larigira/db.py

@@ -1,24 +1,109 @@
+from logging import getLogger
 from tinydb import TinyDB
+from tinydb.storages import JSONStorage
+from tinydb.middlewares import Middleware
+from pathlib import Path
+from typing import Union, Tuple
+
+
+class ReadOnlyMiddleware(Middleware):
+    """
+    Make sure no write ever occurs
+    """
+
+    def __init__(self, storage_cls=TinyDB.DEFAULT_STORAGE):
+        super().__init__(storage_cls)
+
+
+    def write(self, data):
+        raise ReadOnlyException('You cannot write to a readonly db')
+
+
+class ReadOnlyException(ValueError):
+    pass
+
+def only_main(f):
+    '''assumes first argument is id, and must be "main"'''
+    def wrapper(self, *args, **kwargs):
+        _id = args[0]
+        db, db_id = EventModel.parse_id(_id)
+        if db != 'main':
+            raise ReadOnlyException('You called a write operation on a readonly db')
+        return f(self, db_id, *args[1:], **kwargs)
+    return wrapper
 
 
 class EventModel(object):
-    def __init__(self, uri):
+    def __init__(self, uri, additional_db_dir=None):
         self.uri = uri
-        self.db = None
+        self.additional_db_dir = Path(additional_db_dir) if additional_db_dir else None
+        self._dbs = {}
+        self.log = getLogger(self.__class__.__name__)
         self.reload()
 
     def reload(self):
-        if self.db is not None:
+        for db in self._dbs.values():
-            self.db.close()
+            db.close()
-        self.db = TinyDB(self.uri, indent=2)
+        self._dbs['main'] = TinyDB(self.uri, indent=2)
-        self._actions = self.db.table("actions")
+        if self.additional_db_dir is not None:
-        self._alarms = self.db.table("alarms")
+            if self.additional_db_dir.is_dir():
+                for db_file in self.additional_db_dir.glob('*.db.json'):
+                    name = db_file.name[:-8]
+                    if name == 'main':
+                        self.log.warning("%s db file name is not valid (any other name.db.json would have been ok!", str(db_file.name))
+                        continue
+                    if not name.isalpha():
+                        self.log.warning("%s db file name is not valid: it must be alphabetic only", str(db_file.name))
+                        continue
+                    try:
+                        self._dbs[name] = TinyDB(
+                                str(db_file),
+                                storage=ReadOnlyMiddleware(JSONStorage),
+                                default_table='actions'
+                        )
+                    except ReadOnlyException:
+                        # TinyDB adds the default_table if it is not present at read time.
+                        # This should not happen at all for a ReadOnlyMiddleware db, but at least we can notice it and
+                        # properly signal this to the user.
+                        self.log.error("Could not load db %s: 'actions' table is missing", db_file.name)
+                        continue
 
-    def get_action_by_id(self, action_id):
+        self.log.debug('Loaded %d databases: %s', len(self._dbs), ','.join(self._dbs.keys()))
-        return self._actions.get(eid=action_id)
+
+        self._actions = self._dbs['main'].table("actions")
+        self._alarms = self._dbs['main'].table("alarms")
+
+    @staticmethod
+    def canonicalize(eid_or_aid: Union[str, int]) -> str:
+        try:
+            int(eid_or_aid)
+        except ValueError:
+            return eid_or_aid
+        return 'main:%d' % int(eid_or_aid)
+
+    @staticmethod
+    def parse_id(eid_or_aid: Union[str, int]) -> Tuple[str, int]:
+        try:
+            int(eid_or_aid)
+        except ValueError:
+            pass
+        else:
+            return ('main', int(eid_or_aid))
+
+        dbname, num = eid_or_aid.split(':')
+        return (dbname, int(num))
+        
+
+    def get_action_by_id(self, action_id: Union[str, int]):
+        canonical = self.canonicalize(action_id)
+        db, db_action_id = self.__class__.parse_id(canonical)
+        out = self._dbs[db].table('actions').get(eid=db_action_id)
+        out.doc_id = canonical
+        return out
 
     def get_alarm_by_id(self, alarm_id):
-        return self._alarms.get(eid=alarm_id)
+        db, alarm_id = self.__class__.parse_id(alarm_id)
+        return self._dbs[db].table('alarms').get(eid=alarm_id)
 
     def get_actions_by_alarm(self, alarm):
         for action_id in alarm.get("actions", []):
@@ -27,11 +112,21 @@ class EventModel(object):
                 continue
             yield action
 
-    def get_all_alarms(self):
+    def get_all_alarms(self) -> list:
-        return self._alarms.all()
+        out = []
+        for db in self._dbs:
+            for alarm in self._dbs[db].table('alarms').all():
+                alarm.doc_id = '%s:%s' % (db, alarm.doc_id)
+                out.append(alarm)
+        return out
 
-    def get_all_actions(self):
+    def get_all_actions(self) -> list:
-        return self._actions.all()
+        out = []
+        for db in self._dbs:
+            for action in self._dbs[db].table('actions').all():
+                action.doc_id = '%s:%s' % (db, action.doc_id)
+                out.append(action)
+        return out
 
     def get_all_alarms_expanded(self):
         for alarm in self.get_all_alarms():
@@ -49,14 +144,18 @@ class EventModel(object):
     def add_alarm(self, alarm):
         return self.add_event(alarm, [])
 
+    @only_main
     def update_alarm(self, alarmid, new_fields={}):
         return self._alarms.update(new_fields, eids=[alarmid])
 
+    @only_main
     def update_action(self, actionid, new_fields={}):
         return self._actions.update(new_fields, eids=[actionid])
 
+    @only_main
     def delete_alarm(self, alarmid):
         return self._alarms.remove(eids=[alarmid])
 
+    @only_main
     def delete_action(self, actionid):
         return self._actions.remove(eids=[actionid])

larigira/dbadmin/__init__.py

@@ -110,7 +110,7 @@ def addtime():
     return render_template("add_time.html", kinds=kinds, info=info)
 
 
-@db.route("/edit/time/<int:alarmid>", methods=["GET", "POST"])
+@db.route("/edit/time/<alarmid>", methods=["GET", "POST"])
 def edit_time(alarmid):
     model = get_model()
     timespec = model.get_alarm_by_id(alarmid)
@@ -124,7 +124,7 @@ def edit_time(alarmid):
         model.update_alarm(alarmid, data)
         model.reload()
         return redirect(
-            url_for("db.events_calendar", highlight="%d" % alarmid)
+            url_for("db.events_calendar", highlight=model.canonicalize(alarmid))
         )
     return render_template(
         "add_time_kind.html",
@@ -197,7 +197,7 @@ def addaudio_kind(kind):
     )
 
 
-@db.route("/edit/audio/<int:actionid>", methods=["GET", "POST"])
+@db.route("/edit/audio/<actionid>", methods=["GET", "POST"])
 def edit_audio(actionid):
     model = get_model()
     audiospec = model.get_action_by_id(actionid)
@@ -223,7 +223,7 @@ def edit_audio(actionid):
 @db.route("/edit/event/<alarmid>")
 def edit_event(alarmid):
     model = current_app.larigira.controller.monitor.model
-    alarm = model.get_alarm_by_id(int(alarmid))
+    alarm = model.get_alarm_by_id(alarmid)
     if alarm is None:
         abort(404)
     allactions = model.get_all_actions()
@@ -245,16 +245,16 @@ def change_actions(alarmid):
         new_actions = []
     model = current_app.larigira.controller.monitor.model
     ret = model.update_alarm(
-        int(alarmid), new_fields={"actions": [int(a) for a in new_actions]}
+        alarmid, new_fields={"actions": [a for a in new_actions]}
     )
     return jsonify(dict(updated=alarmid, ret=ret))
 
 
-@db.route("/api/alarm/<int:alarmid>/delete", methods=["POST"])
+@db.route("/api/alarm/<alarmid>/delete", methods=["POST"])
 def delete_alarm(alarmid):
     model = current_app.larigira.controller.monitor.model
     try:
-        alarm = model.get_alarm_by_id(int(alarmid))
+        alarm = model.get_alarm_by_id(alarmid)
         print(alarm["nick"])
         model.delete_alarm(alarmid)
     except KeyError:

larigira/event.py

@@ -42,7 +42,7 @@ class Monitor(ParentedLet):
         self.running = {}
         self.conf = conf
         self.q = Queue()
-        self.model = EventModel(self.conf["DB_URI"])
+        self.model = EventModel(self.conf["DB_URI"], self.conf['DB_ADDITIONAL_DIR'])
         self.ticker = Timer(int(self.conf["EVENT_TICK_SECS"]) * 1000, self.q)
 
     def _alarm_missing_time(self, timespec):

larigira/event_manage.py

@@ -8,19 +8,19 @@ from .config import get_conf
 
 
 def main_list(args):
-    m = EventModel(args.file)
+    m = EventModel(args.file, args.additional_dir)
     for alarm, action in m.get_all_alarms_expanded():
         json.dump(dict(alarm=alarm, action=action), sys.stdout, indent=4)
         sys.stdout.write('\n')
 
 
 def main_getaction(args):
-    m = EventModel(args.file)
+    m = EventModel(args.file, args.additional_dir)
-    json.dump(m.get_action_by_id(int(args.actionid)), sys.stdout, indent=4)
+    json.dump(m.get_action_by_id(args.actionid), sys.stdout, indent=4)
 
 
 def main_add(args):
-    m = EventModel(args.file)
+    m = EventModel(args.file, args.additional_dir)
     m.add_event(
         dict(kind="frequency", interval=args.interval, start=1),
         [dict(kind="mpd", howmany=1)],
@@ -29,11 +29,14 @@ def main_add(args):
 
 def main():
     conf = get_conf()
-    p = argparse.ArgumentParser()
+    p = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     p.set_defaults(func=None)
     p.add_argument(
         "-f", "--file", help="Filepath for DB", required=False, default=conf["DB_URI"]
     )
+    p.add_argument(
+        "-d", "--additional-dir", help="Filepath for extra DBs", required=False, default=conf["DB_ADDITIONAL_DIR"]
+    )
     sub = p.add_subparsers()
     sub_list = sub.add_parser("list")
     sub_list.set_defaults(func=main_list)

larigira/fsutils.py

@@ -6,6 +6,10 @@ import posixpath
 import urllib.request
 from tempfile import mkstemp
 from urllib.parse import urlparse
+from pathlib import Path
+import hashlib
+
+import requests
 
 log = logging.getLogger(__name__)
 
@@ -46,6 +50,14 @@ def shortname(path):
     return name
 
 
+def http_expected_length(url):
+    resp = requests.head(url, allow_redirects=True)
+    resp.raise_for_status()
+    header_value = resp.headers.get('content-length')
+    expected_length = int(header_value)
+    return expected_length
+
+
 def download_http(url, destdir=None, copy=False, prefix="httpdl"):
     if url.split(":")[0] not in ("http", "https"):
         log.warning("Not a valid URL: %s", url)
@@ -56,15 +68,43 @@ def download_http(url, destdir=None, copy=False, prefix="httpdl"):
         return None
     if not copy:
         return url
+    if destdir is None:
+        destdir = os.getenv('TMPDIR', '/tmp/')
     fname = posixpath.basename(urlparse(url).path)
     # sanitize
     fname = "".join(
-        c for c in fname if c.isalnum() or c in list("._-")
+        c for c in fname if c.isalnum() or c in list("_-")
     ).rstrip()
+    url_hash = hashlib.sha1(url.encode('utf8')).hexdigest()
+
+    final_path = Path(destdir) / ('%s-%s-%s.%s' % (prefix, fname[:20], url_hash, ext))
+
+    # it might be already fully downloaded, let's check
+    if final_path.exists():
+
+        # this "touch" helps avoiding a race condition in which the
+        # UnusedCleaner could delete  this
+        final_path.touch()
+
+        actual_size = final_path.stat().st_size
+        try:
+            expected_size = http_expected_length(url)
+        except Exception as exc:
+            log.debug("Could not determine expected length for %s: %s", url, exc)
+        else:
+            if expected_size == actual_size:
+                log.debug("File %s already present and complete, download not needed", final_path)
+                return final_path.as_uri()
+            else:
+                log.debug("File %s is already present, but has the wrong length: %d but expected %d", final_path, actual_size, expected_size)
+    else:
+        log.debug("File %s does not exist", final_path)
     tmp = mkstemp(
-        suffix="." + ext, prefix="%s-%s-" % (prefix, fname), dir=destdir
+        suffix="." + ext, prefix="%s-%s-%s-" % (prefix, fname, url_hash), dir=destdir
     )
     os.close(tmp[0])
-    log.info("downloading %s -> %s", url, tmp[1])
+    log.info("downloading %s -> %s -> %s", url, tmp[1], final_path)
     fname, headers = urllib.request.urlretrieve(url, tmp[1])
-    return "file://%s" % os.path.realpath(tmp[1])
+    Path(fname).rename(final_path)
+    return final_path.as_uri()
+# "file://%s" % os.path.realpath(final_path)

larigira/static/calendar.js

@@ -17,7 +17,7 @@ jQuery(function ($) {
     var content = $('<div/>').append(
       $('<p/>').append($('<a class="btn btn-default"/>').text('Modifica orario evento').attr('href', 'edit/time/' + alarmid))
     )
-    if (Number.isInteger(actions)) { // else, it's a string representing a list
+    if (actions.toString().indexOf(',') === -1) { // single one
       content.append($('<p/>').append(
       $('<a class="btn btn-default"/>')
       .text('Modifica audio evento')

larigira/unused.py

@@ -7,6 +7,8 @@ This component will look for files to be removed. There are some assumptions:
 import logging
 import os
 from os.path import normpath
+from pathlib import Path
+import time
 
 import mpd
 
@@ -30,6 +32,10 @@ except ImportError:
 
 
 class UnusedCleaner:
+    # ONLY_DELETE_OLDER_THAN is expressed in seconds.
+    # It configures the maximum age a file can have before being removed.
+    # Set it to "None" if you want to disable this feature.
+    ONLY_DELETE_OLDER_THAN = 30
     def __init__(self, conf):
         self.conf = conf
         self.waiting_removal_files = set()
@@ -69,7 +75,19 @@ class UnusedCleaner:
             for song in mpdc.playlistid()
             if song["file"].startswith("/")
         }
+        now = time.time()
         for fpath in self.waiting_removal_files - files_in_playlist:
+
+            # audio files are sometimes reused, as in download_http. To avoid
+            # referencing a file that UnusedCleaner is going to remove, users
+            # are invited to touch the file, so that UnusedCleaner doesn't
+            # consider it for removal. While this doesn't conceptually solve
+            # the race condition, it should now be extremely rare.
+
+            if ONLY_DELETE_OLDER_THAN is not None:
+                mtime = Path(fpath).stat().st_mtime
+                if now - mtime < ONLY_DELETE_OLDER_THAN:
+                    continue
             # we can remove it!
             self.log.debug("removing unused: %s", fpath)
             self.waiting_removal_files.remove(fpath)