FIX exception reporting causing another exception

editing should be easier now

ISSUE_TEMPLATE.md

@@ -0,0 +1,13 @@
+## Is it a bug or a feature request?
+
+## Describe what happens
+
+### what should happen, instead?
+
+### describe all the steps we can do to reproduce the bug
+
+### provide debug logs relevant to the bug
+
+## Which version of larigira are you using? From git?
+
+### Which OS do you use? which version?

doc/source/about.rst

@@ -40,6 +40,10 @@ action
   For example, ``{ 'kind': 'randomdir', 'paths': ['/my/dir', '/other/path'] }``
   will pick a random file from one of the two paths.
 
+  Its main atribute is ``kind``. The kind essentialy specifies the function that will be run among a
+  predefined set of :doc:`audiogenerators` . Every other attribute is an argument to the specified
+  audiogenerator.
+
 event
   An event is an alarm plus a list of actions. At given times, do those things
 

doc/source/api/larigira.filters.rst

@@ -0,0 +1,22 @@
+larigira\.filters package
+=========================
+
+Submodules
+----------
+
+larigira\.filters\.basic module
+-------------------------------
+
+.. automodule:: larigira.filters.basic
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Module contents
+---------------
+
+.. automodule:: larigira.filters
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/source/audiogenerators.rst

@@ -1,12 +1,15 @@
 Audiogenerators
 ===============
 
-mpdrandom
+mpd
 ---------
 
 picks ``howmany`` song randomly from your mpd library. It follows this
 strategy: it picks ``howmany`` artists from your MPD library, then picks a random song for each one
 
+if you specify a ``prefix``, then only files inside the ``prefix`` directory
+will be picked.
+
 randomdir
 ----------
 
@@ -33,6 +36,48 @@ mostrecent
 It is similar to randomdir, but instead of picking randomly, picks the most
 recent file (according to the ctime).
 
+podcast
+------------
+
+This is probably the most powerful generator that comes included with
+``larigira``. To use this generator, you would need to have a valid podcast
+URL. Beware, here the world *podcast* refer to its very specific meaning of
+an xml-based format which resembles a RSS feed but has more media-specific
+entities. See `this specification
+<https://github.com/simplepie/simplepie-ng/wiki/Spec:-iTunes-Podcast-RSS>`_ for
+more technical details.
+
+So, if you have a valid podcast URL, larigira can look at it, extract audios,
+ download and play the most recent one. Here are some typical usecases for this:
+ * You want to play replica based on what you host on your radio's website.
+ * You want to play some audio from some other radio (or other kind of podcast
+   source)
+
+The podcast form has many many options, but I promise you that 90% of the cases
+are easily solved using ONLY the first option: enter the URL of the podcast
+and... it works!
+
+So, what are all the other options for? Well, to cover some other use cases.
+
+For example, let's say that at night you want to play a *random* show (not the
+last one, which is the default) that happened on your radio. Then you can
+change the "sort by" to be "random". Easy, right?
+
+Another typical usecase is selecting an audio that has a duration which "fits"
+with the schedule of your radio: not too long and not too short. You can do
+that with the "min len" and "max len" fields. For example, setting a `min_len`
+of `30min` and `max_len` of `1h15m` you can avoid picking flash news (too
+short) and very long shows.
+
+You can do many other things with its options, but I left those to your
+immagination. Let's just clarify the workflow:
+
+ * the podcast URL is fetched and audio information is retrieved
+ * filter: audios are filtered by min/max length
+ * sort: audios are sorted according to `sort_by` and `reverse`
+ * select: the n-th episode is fetched, according to `start` field
+
+
 script
 --------
 

doc/source/conf.py

@@ -14,65 +14,66 @@
 # serve to show the default.
 
 from __future__ import print_function
-import sys
+
 import os
 import subprocess
+import sys
 
 from sphinx.util.console import red
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
+# sys.path.insert(0, os.path.abspath('.'))
 
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
+# needs_sphinx = '1.0'
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.coverage',
-    'sphinx.ext.viewcode',
+    "sphinx.ext.autodoc",
+    "sphinx.ext.coverage",
+    "sphinx.ext.viewcode",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix of source filenames.
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
-#source_encoding = 'utf-8-sig'
+# source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = 'larigira'
-copyright = '2015-2017, boyska'
+project = "larigira"
+copyright = "2015-2017, boyska"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '1.3'
+version = "1.3"
 # The full version, including alpha/beta/rc tags.
-release = '1.3.1'
+release = "1.3.3"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
-#language = None
+# language = None
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-#today = ''
+# today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -80,167 +81,161 @@ exclude_patterns = []
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.
-#default_role = None
+# default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
+# add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
-#add_module_names = True
+# add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
-#show_authors = False
+# show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+# modindex_common_prefix = []
 
 # If true, keep warnings as "system message" paragraphs in the built documents.
-#keep_warnings = False
+# keep_warnings = False
 
 
 # -- Options for HTML output ----------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'default'
+html_theme = "default"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-#html_theme_options = {}
+# html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+# html_theme_path = []
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
-#html_title = None
+# html_title = None
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-#html_logo = None
+# html_logo = None
 
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-#html_favicon = None
+# html_favicon = None
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.
-#html_extra_path = []
+# html_extra_path = []
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+# html_last_updated_fmt = '%b %d, %Y'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
-#html_use_smartypants = True
+# html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+# html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-#html_additional_pages = {}
+# html_additional_pages = {}
 
 # If false, no module index is generated.
-#html_domain_indices = True
+# html_domain_indices = True
 
 # If false, no index is generated.
-#html_use_index = True
+# html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
+# html_show_sourcelink = True
 
 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
+# html_show_sphinx = True
 
 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
+# html_show_copyright = True
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
-#html_use_opensearch = ''
+# html_use_opensearch = ''
 
 # This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
+# html_file_suffix = None
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'larigiradoc'
+htmlhelp_basename = "larigiradoc"
 
 
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
-
-# The font size ('10pt', '11pt' or '12pt').
-#'pointsize': '10pt',
-
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-  ('index', 'larigira.tex', 'larigira Documentation',
-   'boyska', 'manual'),
+    ("index", "larigira.tex", "larigira Documentation", "boyska", "manual")
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-#latex_logo = None
+# latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
-#latex_use_parts = False
+# latex_use_parts = False
 
 # If true, show page references after internal links.
-#latex_show_pagerefs = False
+# latex_show_pagerefs = False
 
 # If true, show URL addresses after external links.
-#latex_show_urls = False
+# latex_show_urls = False
 
 # Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []
 
 # If false, no module index is generated.
-#latex_domain_indices = True
+# latex_domain_indices = True
 
 
 # -- Options for manual page output ---------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    ('index', 'larigira', 'larigira Documentation',
-     ['boyska'], 1)
-]
+man_pages = [("index", "larigira", "larigira Documentation", ["boyska"], 1)]
 
 # If true, show URL addresses after external links.
-#man_show_urls = False
+# man_show_urls = False
 
 
 # -- Options for Texinfo output -------------------------------------------
@@ -249,52 +244,58 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'larigira', 'larigira Documentation',
-   'boyska', 'larigira', 'One line description of project.',
-   'Miscellaneous'),
+    (
+        "index",
+        "larigira",
+        "larigira Documentation",
+        "boyska",
+        "larigira",
+        "One line description of project.",
+        "Miscellaneous",
+    )
 ]
 
 # Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
+# texinfo_appendices = []
 
 # If false, no module index is generated.
-#texinfo_domain_indices = True
+# texinfo_domain_indices = True
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
+# texinfo_show_urls = 'footnote'
 
 # If true, do not generate a @detailmenu in the "Top" node's menu.
-#texinfo_no_detailmenu = False
+# texinfo_no_detailmenu = False
 
 
 def run_apidoc(_):
     cur_dir = os.path.abspath(os.path.dirname(__file__))
-    proj_dir = os.path.abspath(os.path.join(cur_dir, '..', '..'))
-    modules = ['larigira']
-    exclude_files = [os.path.abspath(os.path.join(proj_dir, excl))
-                     for excl in ('larigira/rpc.py', 'larigira/dbadmin/')]
-    output_path = os.path.join(cur_dir, 'api')
-    cmd_path = 'sphinx-apidoc'
-    if hasattr(sys, 'real_prefix'):  # Are we in a virtualenv?
+    proj_dir = os.path.abspath(os.path.join(cur_dir, "..", ".."))
+    modules = ["larigira"]
+    exclude_files = [
+        os.path.abspath(os.path.join(proj_dir, excl))
+        for excl in ("larigira/rpc.py", "larigira/dbadmin/")
+    ]
+    output_path = os.path.join(cur_dir, "api")
+    cmd_path = "sphinx-apidoc"
+    if hasattr(sys, "real_prefix"):  # Are we in a virtualenv?
         # assemble the path manually
-        cmd_path = os.path.abspath(os.path.join(sys.prefix,
-                                                'bin',
-                                                'sphinx-apidoc'))
+        cmd_path = os.path.abspath(
+            os.path.join(sys.prefix, "bin", "sphinx-apidoc")
+        )
     if not os.path.exists(cmd_path):
         print(red("No apidoc available!"), file=sys.stderr)
         return
     for module in modules:
         try:
-            subprocess.check_call([cmd_path,
-                                   '--force',
-                                   '-o', output_path,
-                                   module
-                                   ] + exclude_files,
-                                  cwd=proj_dir
+            subprocess.check_call(
+                [cmd_path, "--force", "-o", output_path, module]
+                + exclude_files,
+                cwd=proj_dir,
             )
         except subprocess.CalledProcessError:
             print(red("APIdoc failed for module {}".format(module)))
 
 
 def setup(app):
-    app.connect('builder-inited', run_apidoc)
+    app.connect("builder-inited", run_apidoc)

doc/source/index.rst

@@ -12,11 +12,13 @@ Contents:
    :maxdepth: 2
 
    about
+   quickstart
    install
    timegenerators
    audiogenerators
    eventfilters
    audiogenerators-write
+   troubleshooting
    debug
    api/modules
 

doc/source/install.rst

@@ -6,7 +6,7 @@ using ``pip install larigira``. Or you can ``git clone
 https://git.lattuga.net/boyska/larigira.git`` and run ``python setup.py install``.
 As always, the usage of a virtualenv is recommended.
 
-The only supported python version is 3.4.
+Python greater or equal than 3.4 is supported.
 
 Configuration
 ---------------
@@ -22,7 +22,7 @@ inside ``~/.mpdconf``, add the following line::
     bind_to_address "~/.mpd/socket"
 
 For larigira, you need to set the ``MPD_HOST`` environment variable to
-``$HOME/.mpd/socket``. If you don't do this, you'll find many::
+``$HOME/.mpd/socket``. If you don't do this, you'll find many lines like these in the logs::
 
     15:37:10|ERROR[Player:93] Cannot insert song file:///tmp/larigira.1002/audiogen-randomdir-8eoklcee.mp3
     Traceback (most recent call last):
@@ -68,12 +68,16 @@ MPD_PORT
     If you are not using a socket, but a TCP address (which is *not* suggested), this is how you can specify the
     port.
 DEBUG
-    you can set it to ``true`` or ``false``. Defaults to ``false``.
+    you can set it to ``true`` or ``false``. Defaults to ``false``. Will enable extremely verbose output.
 TMPDIR
     The base for larigira tmpdir. Please note that larigira will create its own directory inside this
     temporary directory. This defaults to the system-wide ``$TMPDIR``, or to ``/tmp/`` if not ``TMPDIR`` is
     not set. Choose it wisely, keeping in mind that in this directory a lot of cache files will be stored, and
     could therefore require hundreds of MB.
+UMASK
+    Umask affects created files permissions. This is important if you want to pass files to a MPD instance
+    that is running with a different users. There is no default umask, so that you can apply umask via your
+    standard system tools.
 
 Events
 ^^^^^^^^^
@@ -81,13 +85,25 @@ Events
 CONTINOUS_AUDIOSPEC
     when the playlist is too short, larigira picks something new. How? this is controlled by this variable.
     This variable should be set to the JSON representation of an audiospec describing how to generate new
-    audios. The default is ``{"kind": "mpd", "howmany": 1}``. You could, for example, change it to
-    ``{ "kind": "randomdir", "paths": ["/var/music"], "howmany": 10}``
+    audios. The default is ``{"kind": "mpd", "howmany": 1}``, which picks a random song from MPD library. You could, for example, change it to:
+
+    - ``{ "kind": "randomdir", "paths": ["/var/music"], "howmany": 5}``
+      to pick files from a specified directory, ignoring MPD library completely. Here, ``howmany`` is set to 5 for performance sake
+    - ``{"kind": "mpd", "howmany": 10, "prefix": "background"}``
+      if you want to use the MPD library, but only use one of its subdirectories.
+      Since using ``mpd`` with a ``prefix`` can be slow, so this is setting ``howmany`` to 10 to
+      gain some performance
+
+    Yes, there's a typo in the name, but I'll keep it like this for compatibility
 EVENT_FILTERS
     See :doc:`eventfilters`
 LOG_CONFIG
     Path to an INI-formatted file to configure logging. See `python logging documentation
     <https://docs.python.org/2/library/logging.config.html#logging-config-dictschema>`_
+MPD_ENFORCE_ALWAYS_PLAYING
+    If this is set to 1, larigira will make sure that MPD is always playing. This means that you can't stop
+    mpd, not even manually running ``mpd stop``. That's probably useful for radios in which mpd is meant to be
+    run unattended. Default: 0
 
 Internals
 ^^^^^^^^^^^^^^
@@ -123,6 +139,10 @@ MPD_WAIT_START_RETRYSECS
 Web interface
 ^^^^^^^^^^^^^^^^
 
+HTTP_ADDRESS
+   The address that the HTTP interface will listen to; defaults to `0.0.0.0`
+HTTP_PORT
+   The port that the HTTP interface will listen to; defaults to `5000`
 FILE_PATH_SUGGESTION
     A list of paths. Those paths will be scanned for suggestions in audiogenerator forms.
 UI_CALENDAR_FREQUENCY_THRESHOLD
@@ -145,3 +165,9 @@ LARIGIRA_UI_CALENDAR_DATE_FMT
     The format to show in ``/db/calendar`` page. The format is specified `here <http://babel.pocoo.org/en/latest/dates.html>`_. Default is ``medium``.
 
     As an example, ``eee dd LLL`` will show ``Sun 10 Mar`` for english, and ``dom 10 mar`` for italian.
+
+Debug and development
+^^^^^^^^^^^^^^^^^^^^^^^
+
+REMOVE_UNUSED_FILES
+    By default, larigira removes the file it generates, as soon as they are no longer in MPD playlist. There is no good reason to change this behavior, unless you need to debug what's going on. For example, if you want to inspect files. Set this to `false` keep everything.

doc/source/quickstart.rst

@@ -0,0 +1,152 @@
+Quick start: install larigira on Debian buster
+==============================================
+
+This guides have this assumptions or conventions:
+ * you have a Debian buster installation
+   - actually 99% of this should work in any distro with recent-enough python3 and systemd
+   - if you don't like systemd, you are free to use any other service manager; larigira integrates nicely with systemd, but has no requirement at all on it.
+ * you have a non-root main user, which we'll call ``radio``
+ * all commands are meant to be run as root. Use ``sudo -i`` if you don't have root password
+
+Install
+-----------
+
+
+Let's start!::
+
+   apt-get install python3 python3-dev build-essential virtualenv mpd mpc libxml2-dev libxslt1-dev zlib1g-dev
+   virtualenv -p /usr/bin/python3 /opt/larigira/
+   /opt/larigira/bin/pip3 install --no-binary :all: larigira
+   touch /etc/default/larigira
+   mkdir -p /home/radio/.mpd/ /etc/larigira/ /var/log/larigira/
+   chown radio. /home/radio/.mpd/
+   chown radio:adm /var/log/larigira/
+   touch /etc/systemd/system/larigira.service
+
+Edit ``/etc/systemd/system/larigira.service`` and put this content::
+
+   [Unit]
+   Description=Radio Automation
+   After=mpd.service
+
+   [Service]
+   Type=notify
+   NotifyAccess=all
+   EnvironmentFile=/etc/default/larigira
+   User=radio
+   ExecStart=/opt/larigira/bin/larigira
+   Restart=always
+
+   [Install]
+   WantedBy=multi-user.target
+
+Now let's edit ``/etc/mpd.conf``::
+
+   music_directory		"/home/radio/Music/"
+   playlist_directory		"/home/radio/.mpd/playlists"
+   db_file			"/home/radio/.mpd/tag_cache"
+   log_file			"syslog"
+   pid_file			"/home/radio/.mpd/pid"
+   state_file			"/home/radio/.mpd/state"
+   sticker_file                   "/home/radio/.mpd/sticker.sql"
+   user				"radio"
+   bind_to_address		"/home/radio/.mpd/socket"
+   bind_to_address			"127.0.0.1"
+   port				"6600"
+   log_level			"default"
+   replaygain			"track"
+   replaygain_limit		"yes"
+   volume_normalization		"yes"
+   max_connections "30"
+
+Now let's edit larigira settings, editing the file ``/etc/default/larigira``::
+   
+   MPD_HOST=/home/radio/.mpd/socket
+   LARIGIRA_DEBUG=false
+   LARIGIRA_LOG_CONFIG=/etc/larigira/logging.ini
+
+   LARIGIRA_EVENT_FILTERS='["percentwait"]'
+   LARIGIRA_EF_MAXWAIT_PERC=400
+   LARIGIRA_MPD_ENFORCE_ALWAYS_PLAYING=1
+   LARIGIRA_SECRET_KEY="changeme with a random, secret string of any length"
+
+Let's include logging configuration, editing ``/etc/larigira/logging.ini``::
+
+   [loggers]
+   keys=root
+
+   [formatters]
+   keys=brief,ext,debug
+
+   [handlers]
+   keys=syslog,own,owndebug,ownerr
+
+   [logger_root]
+   handlers=syslog,own,owndebug,ownerr
+   level=DEBUG
+
+   [handler_syslog]
+   class=handlers.SysLogHandler
+   level=INFO
+   args=('/dev/log', handlers.SysLogHandler.LOG_USER)
+   formatter=brief
+
+   [handler_own]
+   class=handlers.WatchedFileHandler
+   level=INFO
+   args=('/var/log/larigira/larigira.log',)
+   formatter=ext
+   [handler_owndebug]
+   class=handlers.WatchedFileHandler
+   level=DEBUG
+   args=('/var/log/larigira/larigira.debug',)
+   formatter=debug
+   [handler_ownerr]
+   class=handlers.WatchedFileHandler
+   level=ERROR
+   args=('/var/log/larigira/larigira.err',)
+   formatter=ext
+
+   [formatter_ext]
+   format=%(asctime)s|%(levelname)s[%(name)s] %(message)s
+
+   [formatter_debug]
+   format=%(asctime)s|%(levelname)s[%(name)s:%(lineno)d] %(message)s
+
+   [formatter_brief]
+   format=%(levelname)s:%(message)s
+
+For hygiene's sake, let's configure rotation for this log, editing ``/etc/logrotate.d/larigira``::
+
+   /var/log/larigira/*.err
+   /var/log/larigira/*.log {
+           daily
+           missingok
+           rotate 14
+           compress
+           notifempty
+           copytruncate
+           create 600
+   }
+
+   /var/log/larigira/*.debug {
+           daily
+           rotate 2
+           missingok
+           compress
+           notifempty
+           copytruncate
+           create 600
+   }
+
+
+Restart everything::
+
+   systemctl daemon-reload
+   systemctl restart mpd
+   systemctl restart larigira
+   systemctl enable larigira
+   systemctl enable mpd
+
+
+Everything should work now!

doc/source/troubleshooting.rst

@@ -0,0 +1,16 @@
+Common problems
+====================
+
+
+I got AccessDenied in the logs
+------------------------------
+
+You are not using the UNIX socket to access MPD. See the configuration variable MPD_HOST. This is required by
+MPD.
+
+I got ``mpd.base.CommandError: [50@0] {} No such song``
+-------------------------------------------------------
+
+There is permissions issues going on. Probably you are running larigira and mpd with two different users, and
+you haven't set up a common group for them.
+

larigira/audioform_podcast.py

@@ -0,0 +1,63 @@
+from flask_wtf import Form
+from wtforms import (BooleanField, IntegerField, SelectField, StringField,
+                     SubmitField, validators)
+from wtforms.fields.html5 import URLField
+
+
+class AudioForm(Form):
+    nick = StringField(
+        "Audio nick",
+        validators=[validators.required()],
+        description="A simple name to recognize this audio",
+    )
+    url = URLField(
+        "URL",
+        validators=[validators.required()],
+        description="URL of the podcast; it must be valid xml",
+    )
+
+    # TODO: group by filters/sort/select
+    min_len = StringField(
+        "Accetta solo audio lunghi almeno:",
+        description="Leaving this empty will disable this filter",
+    )
+    max_len = StringField(
+        "Accetta solo audio lunghi al massimo:",
+        description="Leaving this empty will disable this filter",
+    )
+    sort_by = SelectField(
+        "Sort episodes",
+        choices=[
+            ("none", "Don't sort"),
+            ("random", "Random"),
+            ("duration", "Duration"),
+            ("date", "date"),
+        ],
+    )
+    start = IntegerField(
+        "Play from episode number",
+        description="Episodes count from 0; 0 is a sane default",
+    )
+    reverse = BooleanField("Reverse sort (descending)")
+    submit = SubmitField("Submit")
+
+    def populate_from_audiospec(self, audiospec):
+        for key in ("nick", "url", "sort_by", "reverse", "min_len", "max_len"):
+            if key in audiospec:
+                getattr(self, key).data = audiospec[key]
+        self.start.data = int(audiospec.get("start", 0))
+
+
+def audio_receive(form):
+    d = {"kind": "podcast"}
+    for key in (
+        "nick",
+        "url",
+        "sort_by",
+        "reverse",
+        "min_len",
+        "max_len",
+        "start",
+    ):
+        d[key] = getattr(form, key).data
+    return d

larigira/audiogen.py

@@ -3,7 +3,7 @@ import sys
 import argparse
 from .entrypoints_utils import get_one_entrypoint
 import json
-from logging import getLogger
+from logging import getLogger, basicConfig
 
 log = getLogger("audiogen")
 
@@ -15,6 +15,7 @@ def get_audiogenerator(kind):
 
 def get_parser():
     parser = argparse.ArgumentParser(description="Generate audio and output paths")
+    parser.add_argument("--log-level", choices=['DEBUG', 'INFO', 'WARNING', 'DEBUG'], default='WARNING')
     parser.add_argument(
         "audiospec",
         metavar="AUDIOSPEC",
@@ -49,6 +50,7 @@ def audiogenerate(spec):
 def main():
     """Main function for the "larigira-audiogen" executable"""
     args = get_parser().parse_args()
+    basicConfig(level=args.log_level)
     spec = read_spec(args.audiospec[0])
     errors = tuple(check_spec(spec))
     if errors:

larigira/audiogen_http.py

@@ -1,31 +1,4 @@
-import os
-import logging
-import posixpath
-from tempfile import mkstemp
-import urllib.request
-from urllib.parse import urlparse
-
-log = logging.getLogger(__name__)
-
-
-def put(url, destdir=None, copy=False):
-    if url.split(":")[0] not in ("http", "https"):
-        log.warning("Not a valid URL: %s", url)
-        return None
-    ext = url.split(".")[-1]
-    if ext.lower() not in ("mp3", "ogg", "oga", "wma", "m4a"):
-        log.warning('Invalid format (%s) for "%s"', ext, url)
-        return None
-    if not copy:
-        return url
-    fname = posixpath.basename(urlparse(url).path)
-    # sanitize
-    fname = "".join(c for c in fname if c.isalnum() or c in list("._-")).rstrip()
-    tmp = mkstemp(suffix="." + ext, prefix="http-%s-" % fname, dir=destdir)
-    os.close(tmp[0])
-    log.info("downloading %s -> %s", url, tmp[1])
-    fname, headers = urllib.request.urlretrieve(url, tmp[1])
-    return "file://%s" % os.path.realpath(tmp[1])
+from larigira.fsutils import download_http
 
 
 def generate(spec):
@@ -35,10 +8,10 @@ def generate(spec):
     Recognized argument is  "paths" (list of static paths)
     """
     if "urls" not in spec:
-        raise ValueError("Malformed audiospec: missing 'paths'")
+        raise ValueError("Malformed audiospec: missing 'urls'")
 
     for url in spec["urls"]:
-        ret = put(url, copy=True)
+        ret = download_http(url, copy=True, prefix="http")
         if ret is None:
             continue
         yield ret

larigira/audiogen_mpdrandom.py

@@ -1,25 +1,43 @@
 import logging
-
-log = logging.getLogger("mpdrandom")
 import random
 
 from mpd import MPDClient
 
 from .config import get_conf
 
+log = logging.getLogger(__name__)
+
 
 def generate_by_artist(spec):
-    """choose HOWMANY random artists, and for each one choose a random song"""
+    """Choose HOWMANY random artists, and for each one choose a random song."""
     spec.setdefault("howmany", 1)
+    prefix = spec.get("prefix", "").rstrip("/")
     log.info("generating")
     conf = get_conf()
     c = MPDClient(use_unicode=True)
     c.connect(conf["MPD_HOST"], conf["MPD_PORT"])
 
+    if prefix:
+        # TODO: listallinfo is discouraged.
+        #       how else could we achieve the same result?
+        artists = list(
+            {r["artist"] for r in c.listallinfo(prefix) if "artist" in r}
+        )
+    else:
         artists = c.list("artist")
-    log.debug("got %d artists", len(artists))
     if not artists:
         raise ValueError("no artists in your mpd database")
     for _ in range(spec["howmany"]):
-        artist = random.choice(artists)
-        yield random.choice(c.find("artist", artist))["file"]
+        artist = random.choice(artists)  # pick one artist
+        if type(artist) is not str:
+            # different mpd library versions have different behavior
+            artist = artist['artist']
+        # pick one song from that artist
+        artist_songs = (res["file"] for res in c.find("artist", artist))
+        if prefix:
+            artist_songs = [
+                fname
+                for fname in artist_songs
+                if fname.startswith(prefix + "/")
+            ]
+        yield random.choice(list(artist_songs))

larigira/audiogen_podcast.py

@@ -0,0 +1,216 @@
+import datetime
+import logging
+import os
+import random
+import sys
+from subprocess import CalledProcessError, check_output
+
+import dateutil.parser
+import requests
+from lxml import html
+from pytimeparse.timeparse import timeparse
+
+from larigira.fsutils import download_http
+
+
+def delta_humanreadable(tdelta):
+    if tdelta is None:
+        return ""
+    days = tdelta.days
+    hours = (tdelta - datetime.timedelta(days=days)).seconds // 3600
+    if days:
+        return "{}d{}h".format(days, hours)
+    return "{}h".format(hours)
+
+
+def get_duration(url):
+    try:
+        lineout = check_output(
+            [
+                "ffprobe",
+                "-v",
+                "error",
+                "-show_entries",
+                "format=duration",
+                "-i",
+                url,
+            ]
+        ).split(b"\n")
+    except CalledProcessError as exc:
+        raise ValueError("error probing `%s`" % url) from exc
+    duration = next(l for l in lineout if l.startswith(b"duration="))
+    value = duration.split(b"=")[1]
+    return int(float(value))
+
+
+class Audio(object):
+    def __init__(self, url, duration=None, date=None):
+        self.url = url
+        self._duration = duration
+        self.date = date
+        self.end_date = datetime.datetime(
+            9999, 12, 31, tzinfo=datetime.timezone.utc
+        )
+
+    def __str__(self):
+        return self.url
+
+    def __repr__(self):
+        return "<Audio {} ({} {})>".format(
+            self.url, self._duration, delta_humanreadable(self.age)
+        )
+
+    @property
+    def duration(self):
+        """lazy-calculation"""
+        if self._duration is None:
+            try:
+                self._duration = get_duration(self.url.encode("utf-8"))
+            except:
+                logging.exception(
+                    "Error while computing duration of %s; set it to 0",
+                    self.url,
+                )
+                self._duration = 0
+        return self._duration
+
+    @property
+    def urls(self):
+        return [self.url]
+
+    @property
+    def age(self):
+        if self.date is None:
+            return None
+        now = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc)
+
+        return now - self.date
+
+    @property
+    def valid(self):
+        return self.end_date >= datetime.datetime.utcnow().replace(
+            tzinfo=datetime.timezone.utc
+        )
+
+
+def get_tree(feed_url):
+    if feed_url.startswith("http:") or feed_url.startswith("https:"):
+        tree = html.fromstring(requests.get(feed_url).content)
+    else:
+        if not os.path.exists(feed_url):
+            raise ValueError("file not found: {}".format(feed_url))
+        tree = html.parse(open(feed_url))
+    return tree
+
+
+def get_item_date(el):
+    el_date = el.find("pubdate")
+    if el_date is None:
+        return None
+    for time_format in ("%Y-%m-%dT%H:%M:%S%z", "%a, %d %b %Y %H:%M:%S %z"):
+        try:
+            return datetime.datetime.strptime(el_date.text, time_format)
+        except:
+            continue
+    return dateutil.parser.parse(el_date.text)
+
+
+def get_audio_from_item(item):
+    encl = item.find("enclosure")
+    if encl is not None:
+        url = encl.get("url")
+    else:
+        return None
+    audio_args = {}
+    if item.find("duration") is not None:
+        duration_parts = item.findtext("duration").split(":")
+        total_seconds = 0
+        for i, num in enumerate(reversed(duration_parts)):
+            total_seconds += int(float(num)) * (60 ** i)
+        if total_seconds:
+            audio_args["duration"] = total_seconds
+    else:
+        contents = item.xpath("group/content")
+        if not contents:
+            contents = item.xpath("content")
+        for child in contents:
+            if child.get("url") == url and child.get("duration") is not None:
+                audio_args["duration"] = int(float(child.get("duration")))
+                break
+    return Audio(url, **audio_args)
+
+
+def get_urls(tree):
+    items = tree.xpath("//item")
+    for i, it in enumerate(items):
+        try:
+            audio = get_audio_from_item(it)
+        except Exception:
+            logging.error("Could not parse item #%d, skipping", i)
+            continue
+        if audio is None:
+            continue
+        if audio.date is None:
+            try:
+                audio.date = get_item_date(it)
+            except Exception:
+                logging.warn("Could not find date for item #%d", i)
+        yield audio
+
+def parse_duration(arg):
+    if arg.isdecimal():
+        secs = int(arg)
+    else:
+        secs = timeparse(arg)
+        if secs is None:
+            raise ValueError("%r is not a valid duration" % arg)
+    return secs
+
+
+def generate(spec):
+    if "url" not in spec:
+        raise ValueError("Malformed audiospec: missing 'url'")
+    audios = list(get_urls(get_tree(spec["url"])))
+    if spec.get("min_len", False):
+        audios = [
+            a for a in audios if a.duration >= parse_duration(spec["min_len"])
+        ]
+    if spec.get("max_len", False):
+        audios = [
+            a for a in audios if a.duration <= parse_duration(spec["max_len"])
+        ]
+
+    # sort
+    sort_by = spec.get("sort_by", "none")
+    if sort_by == "random":
+        random.shuffle(audios)
+    elif sort_by == "date":
+        audios.sort(key=lambda x: x.age)
+    elif sort_by == "duration":
+        audios.sort(key=lambda x: x.duration)
+
+    if spec.get("reverse", False):
+        audios.reverse()
+
+    # slice
+    audios = audios[int(spec.get("start", 0)) :]
+    audios = audios[: int(spec.get("howmany", 1))]
+
+    # copy local
+    local_audios = [
+        download_http(a.url, copy=spec.get("copy", True), prefix="podcast")
+        for a in audios
+    ]
+    return local_audios
+
+
+# TODO: testing
+# TODO: lxml should maybe be optional?
+# TODO: ui
+
+
+if __name__ == "__main__":
+    # less than proper testing
+    logging.basicConfig(level=logging.DEBUG)
+    for u in get_urls(get_tree(sys.argv[1])):
+        print(" -", repr(u))

larigira/config.py

@@ -16,6 +16,7 @@ def get_conf(prefix="LARIGIRA_"):
     conf["CONTINOUS_AUDIOSPEC"] = dict(kind="mpd", howmany=1)
     conf["MPD_HOST"] = os.getenv("MPD_HOST", "localhost")
     conf["MPD_PORT"] = int(os.getenv("MPD_PORT", "6600"))
+    conf["UMASK"] = None
     conf["CACHING_TIME"] = 10
     conf["DB_URI"] = os.path.join(conf_dir, "db.json")
     conf["SCRIPTS_PATH"] = os.path.join(conf_dir, "scripts")
@@ -26,15 +27,23 @@ def get_conf(prefix="LARIGIRA_"):
     conf["SECRET_KEY"] = "Please replace me!"
     conf["MPD_WAIT_START"] = True
     conf["MPD_WAIT_START_RETRYSECS"] = 5
+    conf["MPD_ENFORCE_ALWAYS_PLAYING"] = False
     conf["CHECK_SECS"] = 20  # period for checking playlist length
     conf["EVENT_TICK_SECS"] = 30  # period for scheduling events
     conf["DEBUG"] = False
+    conf[
+        "REMOVE_UNUSED_FILES"
+    ] = True  # please keep it to True unless you are into deep debugging!
     conf["LOG_CONFIG"] = False
     conf["TMPDIR"] = os.getenv("TMPDIR", "/tmp/")
     conf["FILE_PATH_SUGGESTION"] = ()  # tuple of paths
+    # UI_CALENDAR_FREQUENCY_THRESHOLD"] has been removed
+    # use UI_CALENDAR_OCCURRENCIES_THRESHOLD instead
     conf["UI_CALENDAR_FREQUENCY_THRESHOLD"] = 4 * 60 * 60  # 4 hours
+    conf["UI_CALENDAR_OCCURRENCIES_THRESHOLD"] = 40
     conf["UI_CALENDAR_DATE_FMT"] = "medium"
     conf["EVENT_FILTERS"] = []
+    conf["HTTP_ADDRESS"] = "0.0.0.0"
     conf["HTTP_PORT"] = 5000
     conf["HOME_URL"] = "/db/calendar"
     conf.update(from_envvars(prefix=prefix))
@@ -66,7 +75,9 @@ def from_envvars(prefix=None, envvars=None, as_json=True):
 
     if not envvars:
         envvars = {
-            k: k[len(prefix) :] for k in os.environ.keys() if k.startswith(prefix)
+            k: k[len(prefix) :]
+            for k in os.environ.keys()
+            if k.startswith(prefix)
         }
 
     for env_name, name in envvars.items():

larigira/dbadmin/__init__.py

@@ -4,30 +4,22 @@ This module contains a flask blueprint for db administration stuff
 Templates are self-contained in this directory
 """
 from __future__ import print_function
-import os
-from datetime import datetime, timedelta, time
-from collections import defaultdict
+
 import mimetypes
+import os
+from collections import defaultdict
+from datetime import datetime, time, timedelta
 
-from flask import (
-    current_app,
-    Blueprint,
-    Response,
-    render_template,
-    jsonify,
-    abort,
-    request,
-    redirect,
-    url_for,
-    flash,
-)
+from flask import (Blueprint, Response, abort, current_app, flash, jsonify,
+                   redirect, render_template, request, url_for)
 
-from larigira.entrypoints_utils import get_avail_entrypoints
-from larigira.audiogen import get_audiogenerator
-from larigira.timegen_every import FrequencyAlarm
-from larigira.timegen import get_timegenerator, timegenerate
 from larigira import forms
+from larigira.audiogen import get_audiogenerator
 from larigira.config import get_conf
+from larigira.entrypoints_utils import get_avail_entrypoints
+from larigira.timegen import get_timegenerator, timegenerate
+from larigira.timegen_every import FrequencyAlarm
+
 from .suggestions import get_suggestions
 
 db = Blueprint(
@@ -39,10 +31,13 @@ db = Blueprint(
 
 
 def request_wants_json():
-    best = request.accept_mimetypes.best_match(["application/json", "text/html"])
+    best = request.accept_mimetypes.best_match(
+        ["application/json", "text/html"]
+    )
     return (
         best == "application/json"
-        and request.accept_mimetypes[best] > request.accept_mimetypes["text/html"]
+        and request.accept_mimetypes[best]
+        > request.accept_mimetypes["text/html"]
     )
 
 
@@ -67,26 +62,31 @@ def events_list():
 def events_calendar():
     model = current_app.larigira.controller.monitor.model
     today = datetime.now().date()
-    maxdays = 30
+    max_days = 30
+    max_occurrences = get_conf()["UI_CALENDAR_OCCURRENCIES_THRESHOLD"]
     # {date: {datetime: [(alarm1,actions1), (alarm2,actions2)]}}
     days = defaultdict(lambda: defaultdict(list))
-    freq_threshold = get_conf()["UI_CALENDAR_FREQUENCY_THRESHOLD"]
+    show_all = request.args.get("all", "0") == "1"
     for alarm in model.get_all_alarms():
-        if (
-            freq_threshold
-            and alarm["kind"] == "frequency"
-            and FrequencyAlarm(alarm).interval < freq_threshold
-        ):
-            continue
         actions = tuple(model.get_actions_by_alarm(alarm))
         if not actions:
             continue
-        t = datetime.fromtimestamp(int(today.strftime("%s")))
-        for t in timegenerate(alarm, now=t, howmany=maxdays):
-            if t is None or t > datetime.combine(
-                today + timedelta(days=maxdays), time()
+        today_dt = datetime.fromtimestamp(int(today.strftime("%s")))
+        max_dt = datetime.combine(today_dt + timedelta(days=max_days), time())
+        occurrences = []
+        for t in timegenerate(
+            alarm, now=today_dt, howmany=max_occurrences + 1
         ):
+            if t is None:
                 break
+            if t > max_dt:
+                break
+            occurrences.append(t)
+        if not occurrences:
+            continue
+        if not show_all and len(occurrences) > max_occurrences:
+            continue
+        for t in occurrences:
             days[t.date()][t].append((alarm, actions))
 
     # { weeknum: [day1, day2, day3] }
@@ -94,7 +94,9 @@ def events_calendar():
     for d in sorted(days.keys()):
         weeks[d.isocalendar()[:2]].append(d)
 
-    return render_template("calendar.html", days=days, weeks=weeks)
+    return render_template(
+        "calendar.html", days=days, weeks=weeks, show_all=show_all
+    )
 
 
 @db.route("/add/time")
@@ -121,9 +123,15 @@ def edit_time(alarmid):
         data = receiver(form)
         model.update_alarm(alarmid, data)
         model.reload()
-        return redirect(url_for("db.events_calendar", highlight="%d" % alarmid))
+        return redirect(
+            url_for("db.events_calendar", highlight="%d" % alarmid)
+        )
     return render_template(
-        "add_time_kind.html", form=form, kind=kind, mode="edit", alarmid=alarmid
+        "add_time_kind.html",
+        form=form,
+        kind=kind,
+        mode="edit",
+        alarmid=alarmid,
     )
 
 
@@ -150,7 +158,9 @@ def addtime_kind(kind):
             resp.status_code = 400
             return resp
 
-    return render_template("add_time_kind.html", form=form, kind=kind, mode="add")
+    return render_template(
+        "add_time_kind.html", form=form, kind=kind, mode="add"
+    )
 
 
 @db.route("/add/audio")
@@ -180,7 +190,10 @@ def addaudio_kind(kind):
             return resp
 
     return render_template(
-        "add_audio_kind.html", form=form, kind=kind, suggestions=get_suggestions()
+        "add_audio_kind.html",
+        form=form,
+        kind=kind,
+        suggestions=get_suggestions(),
     )
 
 
@@ -214,6 +227,7 @@ def edit_event(alarmid):
     if alarm is None:
         abort(404)
     allactions = model.get_all_actions()
+    allactions.sort(key=lambda a: a.eid, reverse=True)
     actions = tuple(model.get_actions_by_alarm(alarm))
     return render_template(
         "edit_event.html",

larigira/dbadmin/templates/calendar.html

@@ -19,6 +19,19 @@ li.alarm .alarm-actions { display: none; }
 
 {% block content %}
 <div class="container-fluid">
+	<div class="row">
+	{% if not show_all %}
+	<a href="{{url_for('db.events_calendar')}}?all=1"
+		class="btn btn-sm  btn-default state-collapsed pull-right">
+		<span class="glyphicon glyphicon-resize-full"></span>
+		Mostra tutti gli eventi</a>
+	{% else %}
+	<a href="{{url_for('db.events_calendar')}}?all=0"
+		class="btn btn-sm btn-default state-collapsed pull-right">
+		<span class="glyphicon glyphicon-resize-small"></span>
+		Nascondi gli eventi troppo frequenti</a>
+	{% endif %}
+	</div>
 	{% for week, weekdays in weeks|dictsort %}
 	<div class="week row" id="week-{{week[0]}}-{{week[1]}}">
 	{% for day in weeks[week] %}
@@ -45,5 +58,6 @@ li.alarm .alarm-actions { display: none; }
 	<hr/>
 		{%endfor %}
 	</div><!-- container -->
+
 {% endblock content %}
 {# vim: set ts=2 sw=2 noet: #}

larigira/dbadmin/templates/edit_event.html

@@ -7,6 +7,10 @@
 	padding: 5px;
 	border: 2px dashed #999;
 }
+#available-actions {
+	max-height: 50vw;
+	overflow-y: scroll;
+}
 </style>
 {%endblock styles%}
 {%block scripts %}
@@ -49,7 +53,8 @@ $(function() {
 	You are currently editing: <code>{{alarm|tojson}}</code>
 	{% endif %}
 	<h2>Change actions</h2>
-	<div>
+	<div class="row">
+		<div id="available-actions" class="col-md-8">
 			Available actions:
 			<ul>
 				{% for a in all_actions %}
@@ -60,7 +65,9 @@ $(function() {
 				{% endfor %}
 			</ul>
 		</div>
-	<div>
+		<div class="col-md-4">
+			<div class="row">
+				<div class="col-md-12">
 					Actions currently added:
 					<ul id="selected">
 						{% for a in actions %}
@@ -70,11 +77,13 @@ $(function() {
 								class="ui-state-default">{% if a.nick %} {{a.nick}} {% else %} {{a.eid}} {%endif%}</li>
 						{% endfor %}
 					</ul>
-	</div>
-	<div>
+				</div><!-- .col-md-12 -->
+				<div class="col-md-12">
 					<button>Save</button>
-	</div>
-</div>
+				</div><!-- .col-md-12 -->
+			</div><!-- .row -->
+		</div><!-- .col-md-4 -->
+	</div><!-- .row -->
 {% endblock content %}
 {# vim: set ts=2 sw=2 noet: #}
 

larigira/event.py

@@ -1,17 +1,19 @@
 from __future__ import print_function
-from gevent import monkey
 
-monkey.patch_all(subprocess=True)
 import logging
 from datetime import datetime, timedelta
 
 import gevent
+from gevent import monkey
 from gevent.queue import Queue
 
-from .eventutils import ParentedLet, Timer
-from .timegen import timegenerate
 from .audiogen import audiogenerate
 from .db import EventModel
+from .eventutils import ParentedLet, Timer
+from .timegen import timegenerate
+
+monkey.patch_all(subprocess=True)
+
 
 logging.getLogger("mpd").setLevel(logging.WARNING)
 
@@ -51,6 +53,7 @@ class Monitor(ParentedLet):
             logging.exception(
                 "Could not generate " "an alarm from timespec %s", timespec
             )
+            return None
         if when is None:
             # expired
             return None
@@ -75,9 +78,11 @@ class Monitor(ParentedLet):
             # but it is "tricky"; any small delay would cause the event to be
             # missed
             if delta is None:
-                self.log.debug(
-                    "Skipping event %s: will never ring", alarm.get("nick", alarm.eid)
-                )
+                # this is way too much logging! we need more levels!
+                # self.log.debug(
+                #    "Skipping event %s: will never ring", alarm.get("nick", alarm.eid)
+                # )
+                pass
             elif delta <= 2 * self.conf["EVENT_TICK_SECS"]:
                 self.log.debug(
                     "Scheduling event %s (%ds) => %s",
@@ -87,7 +92,7 @@ class Monitor(ParentedLet):
                 )
                 self.schedule(alarm, actions, delta)
             else:
-                self.log.debug(
+                self.log.debugv(
                     "Skipping event %s too far (%ds)",
                     alarm.get("nick", alarm.eid),
                     delta,
@@ -104,7 +109,9 @@ class Monitor(ParentedLet):
         if delta is None:
             delta = self._alarm_missing_time(timespec)
 
-        audiogen = gevent.spawn_later(delta, self.process_action, timespec, audiospecs)
+        audiogen = gevent.spawn_later(
+            delta, self.process_action, timespec, audiospecs
+        )
         audiogen.parent_greenlet = self
         audiogen.doc = 'Will wait {} seconds, then generate audio "{}"'.format(
             delta, ",".join(aspec.get("nick", "") for aspec in audiospecs)

larigira/filters/basic.py

@@ -64,6 +64,12 @@ def percentwait(songs, context, conf, getdur=get_duration):
             continue
         eventduration += songduration
 
+    if eventduration == 0:
+        # must be an error! mutagen support is not always perfect
+        return (
+            True,
+            ("mutagen could not calculate length of %s" % ",".join(songs["uris"])),
+        )
     wait = eventduration * (percentwait / 100.0)
     if remaining > wait:
         return False, "remaining %d max allowed %d" % (remaining, wait)

larigira/fsutils.py

@@ -1,6 +1,13 @@
-import os
 import fnmatch
+import logging
 import mimetypes
+import os
+import posixpath
+import urllib.request
+from tempfile import mkstemp
+from urllib.parse import urlparse
+
+log = logging.getLogger(__name__)
 
 
 def scan_dir(dirname, extension=None):
@@ -37,3 +44,27 @@ def shortname(path):
     name = name.rsplit(".", 1)[0]  # no extension
     name = "".join(c for c in name if c.isalnum())  # no strange chars
     return name
+
+
+def download_http(url, destdir=None, copy=False, prefix="httpdl"):
+    if url.split(":")[0] not in ("http", "https"):
+        log.warning("Not a valid URL: %s", url)
+        return None
+    ext = url.split(".")[-1]
+    if ext.lower() not in ("mp3", "ogg", "oga", "wma", "m4a"):
+        log.warning('Invalid format (%s) for "%s"', ext, url)
+        return None
+    if not copy:
+        return url
+    fname = posixpath.basename(urlparse(url).path)
+    # sanitize
+    fname = "".join(
+        c for c in fname if c.isalnum() or c in list("._-")
+    ).rstrip()
+    tmp = mkstemp(
+        suffix="." + ext, prefix="%s-%s-" % (prefix, fname), dir=destdir
+    )
+    os.close(tmp[0])
+    log.info("downloading %s -> %s", url, tmp[1])
+    fname, headers = urllib.request.urlretrieve(url, tmp[1])
+    return "file://%s" % os.path.realpath(tmp[1])

larigira/main.py

@@ -2,25 +2,26 @@
 This module is for the main application logic
 """
 from __future__ import print_function
-from gevent import monkey
 
-monkey.patch_all(subprocess=True)
-
-import sys
-import os
-import tempfile
-import signal
-from time import sleep
+import json
 import logging
 import logging.config
+import os
+import signal
 import subprocess
+import sys
+import tempfile
+from time import sleep
 
 import gevent
+from gevent import monkey
 from gevent.pywsgi import WSGIServer
 
-from .mpc import Controller, get_mpd_client
-from .config import get_conf
-from .rpc import create_app
+from larigira.config import get_conf
+from larigira.mpc import Controller, get_mpd_client
+from larigira.rpc import create_app
+
+monkey.patch_all(subprocess=True)
 
 
 def on_main_crash(*args, **kwargs):
@@ -30,12 +31,14 @@ def on_main_crash(*args, **kwargs):
 
 class Larigira(object):
     def __init__(self):
+
         self.log = logging.getLogger("larigira")
         self.conf = get_conf()
         self.controller = Controller(self.conf)
         self.controller.link_exception(on_main_crash)
         self.http_server = WSGIServer(
-            ("", int(self.conf["HTTP_PORT"])), create_app(self.controller.q, self)
+            (self.conf["HTTP_ADDRESS"], int(self.conf["HTTP_PORT"])),
+            create_app(self.controller.q, self),
         )
 
     def start(self):
@@ -57,28 +60,57 @@ def sd_notify(ready=False, status=None):
 
 
 def main():
-    tempfile.tempdir = os.environ["TMPDIR"] = os.path.join(
-        os.getenv("TMPDIR", "/tmp"), "larigira.%d" % os.getuid()
-    )
-    if not os.path.isdir(os.environ["TMPDIR"]):
-        os.makedirs(os.environ["TMPDIR"])
+    logging.addLevelName(9, "DEBUGV")
     if get_conf()["LOG_CONFIG"]:
         logging.config.fileConfig(
             get_conf()["LOG_CONFIG"], disable_existing_loggers=True
         )
     else:
-        log_format = "%(asctime)s|%(levelname)s[%(name)s:%(lineno)d] %(message)s"
+        log_format = (
+            "%(asctime)s|%(levelname)s[%(name)s:%(lineno)d] %(message)s"
+        )
         logging.basicConfig(
-            level=logging.DEBUG if get_conf()["DEBUG"] else logging.INFO,
+            level="DEBUGV" if get_conf()["DEBUG"] else logging.INFO,
             format=log_format,
             datefmt="%H:%M:%S",
         )
+
+    def debugv(self, message, *args, **kws):
+        if self.isEnabledFor(9):
+            self._log(9, message, args, **kws)
+
+    logging.Logger.debugv = debugv
+
+    logging.debug(
+        "Starting larigira with this conf:\n%s",
+        json.dumps(get_conf(), indent=2),
+    )
+
+    if get_conf()["UMASK"]:
+        umask = int(get_conf()["UMASK"], base=8)
+        logging.debug(
+            "Setting umask %s (decimal: %d)", get_conf()["UMASK"], umask
+        )
+        os.umask(umask)
+
+    tempfile.tempdir = os.environ["TMPDIR"] = os.path.join(
+        os.getenv("TMPDIR", "/tmp"), "larigira.%d" % os.getuid()
+    )
+    if not os.path.isdir(os.environ["TMPDIR"]):
+        os.makedirs(os.environ["TMPDIR"])
+
     if get_conf()["MPD_WAIT_START"]:
+
         while True:
             try:
                 get_mpd_client(get_conf())
-            except Exception:
-                logging.debug("Could not connect to MPD, waiting")
+            except Exception as exc:
+                print("exc", exc, file=sys.stderr)
+                logging.debug(
+                    "Could not connect to MPD at (%s,%s), waiting",
+                    get_conf()["MPD_HOST"],
+                    get_conf()["MPD_PORT"],
+                )
                 sd_notify(status="Waiting MPD connection")
                 sleep(int(get_conf()["MPD_WAIT_START_RETRYSECS"]))
             else:

larigira/mpc.py

@@ -1,17 +1,19 @@
 from __future__ import print_function
+
 import logging
 import signal
+
+import mpd
 from pkg_resources import iter_entry_points
 
 import gevent
 from gevent.queue import Queue
-import mpd
 
+from .audiogen import audiogenerate
+from .entrypoints_utils import get_avail_entrypoints
 from .event import Monitor
 from .eventutils import ParentedLet, Timer
-from .audiogen import audiogenerate
 from .unused import UnusedCleaner
-from .entrypoints_utils import get_avail_entrypoints
 
 
 def get_mpd_client(conf):
@@ -51,7 +53,9 @@ class MPDWatcher(ParentedLet):
                 FileNotFoundError,
             ) as exc:
                 self.log.warning(
-                    "Connection to MPD failed (%s: %s)", exc.__class__.__name__, exc
+                    "Connection to MPD failed (%s: %s)",
+                    exc.__class__.__name__,
+                    exc,
                 )
                 self.client = None
                 first_after_connection = True
@@ -83,9 +87,15 @@ class Player:
         mpd_client = mpd.MPDClient(use_unicode=True)
         try:
             mpd_client.connect(self.conf["MPD_HOST"], self.conf["MPD_PORT"])
-        except (mpd.ConnectionError, ConnectionRefusedError, FileNotFoundError) as exc:
+        except (
+            mpd.ConnectionError,
+            ConnectionRefusedError,
+            FileNotFoundError,
+        ) as exc:
             self.log.warning(
-                "Connection to MPD failed (%s: %s)", exc.__class__.__name__, exc
+                "Connection to MPD failed (%s: %s)",
+                exc.__class__.__name__,
+                exc,
             )
             raise gevent.GreenletExit()
         return mpd_client
@@ -124,8 +134,8 @@ class Player:
             uris = greenlet.value
             for uri in uris:
                 assert type(uri) is str, type(uri)
-                self.tmpcleaner.watch(uri.strip())
                 mpd_client.add(uri.strip())
+                self.tmpcleaner.watch(uri.strip())
 
         picker.link_value(add)
         picker.start()
@@ -146,7 +156,9 @@ class Player:
                 try:
                     ret = ef(songs=songs, context=ctx, conf=self.conf)
                 except ImportError as exc:
-                    self.log.warn("Filter %s skipped: %s" % (entrypoint.name, exc))
+                    self.log.warn(
+                        "Filter %s skipped: %s" % (entrypoint.name, exc)
+                    )
                     continue
                 if ret is None:  # bad behavior!
                     continue
@@ -157,6 +169,11 @@ class Player:
                 reason = "Filtered by %s (%s)" % (entrypoint.name, reason)
                 if ret is False:
                     return ret, reason
+                else:
+                    if reason:
+                        self.log.debug(
+                            "filter %s says ok: %s", entrypoint.name, reason
+                        )
         return True, "Passed through %s" % ",".join(availfilters)
 
     def enqueue(self, songs):
@@ -197,6 +214,11 @@ class Player:
                 self.log.exception("Cannot insert song %s", uri)
             self.tmpcleaner.watch(uri.strip())
 
+    def play(self):
+        """make sure that MPD is playing"""
+        mpd_client = self._get_mpd()
+        mpd_client.play()
+
 
 class Controller(gevent.Greenlet):
     def __init__(self, conf):
@@ -224,7 +246,6 @@ class Controller(gevent.Greenlet):
         gevent.Greenlet.spawn(self.player.check_playlist)
         while True:
             value = self.q.get()
-            self.log.debug("<- %s", str(value))
             # emitter = value['emitter']
             kind = value["kind"]
             args = value["args"]
@@ -232,6 +253,8 @@ class Controller(gevent.Greenlet):
                 kind == "mpc" and args[0] in ("player", "playlist", "connect")
             ):
                 gevent.Greenlet.spawn(self.player.check_playlist)
+                if self.conf["MPD_ENFORCE_ALWAYS_PLAYING"]:
+                    gevent.Greenlet.spawn(self.player.play)
                 try:
                     self.player.tmpcleaner.check_playlist()
                 except:
@@ -249,7 +272,9 @@ class Controller(gevent.Greenlet):
                     self.log.exception(
                         "Error while adding to queue; " "bad audiogen output?"
                     )
-            elif (kind == "signal" and args[0] == signal.SIGALRM) or kind == "refresh":
+            elif (
+                kind == "signal" and args[0] == signal.SIGALRM
+            ) or kind == "refresh":
                 # it's a tick!
                 self.log.debug("Reload")
                 self.monitor.q.put(dict(kind="forcetick"))

larigira/rpc.py

@@ -1,24 +1,17 @@
-import logging
 import gc
+import logging
 from copy import deepcopy
 
-from greenlet import greenlet
-from flask import (
-    current_app,
-    Blueprint,
-    Flask,
-    jsonify,
-    render_template,
-    request,
-    abort,
-    redirect,
-)
-from flask_bootstrap import Bootstrap
+from flask import (Blueprint, Flask, abort, current_app, jsonify, redirect,
+                   render_template, request)
 from flask.ext.babel import Babel
-from werkzeug.contrib.cache import SimpleCache
+from flask_bootstrap import Bootstrap
+
+from cachelib import SimpleCache
+from greenlet import greenlet
 
-from .dbadmin import db
 from .config import get_conf
+from .dbadmin import db
 
 rpc = Blueprint("rpc", __name__, url_prefix=get_conf()["ROUTE_PREFIX"] + "/api")
 viewui = Blueprint(

larigira/static/calendar.js

@@ -1,4 +1,5 @@
 /* global jQuery */
+
 jQuery(function ($) {
   $('.button').button({
     icons: {

larigira/tests/test_web.py

@@ -6,7 +6,7 @@ monkey.patch_all(subprocess=True)
 import pytest
 
 from larigira.rpc import create_app
-from larigira.larigira import Larigira
+from larigira.main import Larigira
 
 
 @pytest.fixture

larigira/timeform_base.py

@@ -1,18 +1,12 @@
 import logging
-
 from datetime import datetime
+
 from pytimeparse.timeparse import timeparse
 
 from flask_wtf import Form
-from wtforms import (
-    StringField,
-    validators,
-    SubmitField,
-    SelectMultipleField,
-    ValidationError,
-)
-
 from larigira.formutils import EasyDateTimeField
+from wtforms import (SelectMultipleField, StringField, SubmitField,
+                     ValidationError, validators)
 
 log = logging.getLogger(__name__)
 
@@ -90,9 +84,9 @@ class FrequencyAlarmForm(Form):
     def populate_from_timespec(self, timespec):
         if "nick" in timespec:
             self.nick.data = timespec["nick"]
-        if "start" in timespec:
+        if timespec.get("start"):
             self.start.data = datetime.fromtimestamp(timespec["start"])
-        if "end" in timespec:
+        if timespec.get("end"):
             self.end.data = datetime.fromtimestamp(timespec["end"])
         if "weekdays" in timespec:
             self.weekdays.data = timespec["weekdays"]
@@ -123,6 +117,6 @@ def frequencyalarm_receive(form):
         obj["start"] = int(form.start.data.strftime("%s"))
     else:
         obj["start"] = 0
-    if form.end.data:
-        obj["end"] = int(form.end.data.strftime("%s"))
+
+    obj["end"] = int(form.end.data.strftime("%s")) if form.end.data else None
     return obj

larigira/unused.py

@@ -4,9 +4,10 @@ This component will look for files to be removed. There are some assumptions:
       own specific TMPDIR
     * MPD URIs are parsed, and only file:/// is supported
 """
+import logging
 import os
 from os.path import normpath
-import logging
+
 import mpd
 
 
@@ -64,11 +65,13 @@ class UnusedCleaner:
         """check playlist + internal watchlist to see what can be removed"""
         mpdc = self._get_mpd()
         files_in_playlist = {
-            song["file"] for song in mpdc.playlistid() if song["file"].startswith("/")
+            song["file"]
+            for song in mpdc.playlistid()
+            if song["file"].startswith("/")
         }
         for fpath in self.waiting_removal_files - files_in_playlist:
             # we can remove it!
             self.log.debug("removing unused: %s", fpath)
             self.waiting_removal_files.remove(fpath)
-            if os.path.exists(fpath):
+            if os.path.exists(fpath) and self.conf["REMOVE_UNUSED_FILES"]:
                 os.unlink(fpath)

setup.py

@@ -1,5 +1,5 @@
-import sys
 import os
+import sys
 
 from setuptools import setup
 from setuptools.command.test import test as TestCommand
@@ -11,7 +11,7 @@ def read(fname):
 
 
 class PyTest(TestCommand):
-    user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")]
+    user_options = [("pytest-args=", "a", "Arguments to pass to py.test")]
 
     def initialize_options(self):
         TestCommand.initialize_options(self)
@@ -25,89 +25,104 @@ class PyTest(TestCommand):
     def run_tests(self):
         # import here, cause outside the eggs aren't loaded
         import pytest
+
         errno = pytest.main(self.pytest_args)
         sys.exit(errno)
 
-setup(name='larigira',
-      version='1.3.1',
-      description='A radio automation based on MPD',
-      long_description=read('README.rst'),
-      long_description_content_type='text/x-rst',
-      author='boyska',
-      author_email='piuttosto@logorroici.org',
-      license='AGPL',
-      packages=['larigira', 'larigira.dbadmin', 'larigira.filters'],
+
+setup(
+    name="larigira",
+    version="1.3.3",
+    description="A radio automation based on MPD",
+    long_description=read("README.rst"),
+    long_description_content_type="text/x-rst",
+    author="boyska",
+    author_email="piuttosto@logorroici.org",
+    license="AGPL",
+    packages=["larigira", "larigira.dbadmin", "larigira.filters"],
     install_requires=[
-          'Babel==2.6.0',
-          'Flask-Babel==0.12.2',
-          'pyxdg',
-          'gevent',
-          'flask-bootstrap',
-          'python-mpd2',
-          'wtforms',
-          'Flask-WTF',
-          'flask==0.11',
-          'pytimeparse',
-          'croniter==0.3.29',
-          'tinydb'
+        "Babel==2.6.0",
+        "Flask-Babel==1.0.0",
+        "pyxdg==0.26",
+        "gevent==1.4.0",
+        "flask-bootstrap",
+        "python-mpd2",
+        "wtforms==2.2.1",
+        "Flask-WTF==0.14.2",
+        "flask==0.11",
+        "pytimeparse==1.1.8",
+        "croniter==0.3.29",
+        "werkzeug==0.14.1",
+        "cachelib==0.1",
+        "tinydb==3.12.2",
+        "lxml==4.5.1",
+        "requests==2.23.0",
     ],
-      tests_require=['pytest-timeout==1.0', 'py>=1.4.29', 'pytest==3.0', ],
-      python_requires='>=3.5',
-      extras_require={
-          'percentwait': ['mutagen'],
-      },
-      cmdclass={'test': PyTest},
+    tests_require=["pytest-timeout==1.0", "py>=1.4.29", "pytest==3.0"],
+    python_requires=">=3.5",
+    extras_require={"percentwait": ["mutagen"]},
+    cmdclass={"test": PyTest},
     zip_safe=False,
     include_package_data=True,
     entry_points={
-          'console_scripts': ['larigira=larigira.larigira:main',
-                              'larigira-timegen=larigira.timegen:main',
-                              'larigira-audiogen=larigira.audiogen:main',
-                              'larigira-dbmanage=larigira.event_manage:main'],
-          'larigira.audiogenerators': [
-              'mpd = larigira.audiogen_mpdrandom:generate_by_artist',
-              'static = larigira.audiogen_static:generate',
-              'http = larigira.audiogen_http:generate',
-              'randomdir = larigira.audiogen_randomdir:generate',
-              'mostrecent = larigira.audiogen_mostrecent:generate',
-              'script = larigira.audiogen_script:generate',
+        "console_scripts": [
+            "larigira=larigira.main:main",
+            "larigira-timegen=larigira.timegen:main",
+            "larigira-audiogen=larigira.audiogen:main",
+            "larigira-dbmanage=larigira.event_manage:main",
         ],
-          'larigira.timegenerators': [
-              'frequency = larigira.timegen_every:FrequencyAlarm',
-              'single = larigira.timegen_every:SingleAlarm',
-              'cron = larigira.timegen_cron:CronAlarm',
+        "larigira.audiogenerators": [
+            "mpd = larigira.audiogen_mpdrandom:generate_by_artist",
+            "static = larigira.audiogen_static:generate",
+            "http = larigira.audiogen_http:generate",
+            "podcast = larigira.audiogen_podcast:generate",
+            "randomdir = larigira.audiogen_randomdir:generate",
+            "mostrecent = larigira.audiogen_mostrecent:generate",
+            "script = larigira.audiogen_script:generate",
         ],
-          'larigira.timeform_create': [
-              'single = larigira.timeform_base:SingleAlarmForm',
-              'frequency = larigira.timeform_base:FrequencyAlarmForm',
-              'cron = larigira.timeform_cron:CronAlarmForm',
+        "larigira.timegenerators": [
+            "frequency = larigira.timegen_every:FrequencyAlarm",
+            "single = larigira.timegen_every:SingleAlarm",
+            "cron = larigira.timegen_cron:CronAlarm",
         ],
-          'larigira.timeform_receive': [
-              'single = larigira.timeform_base:singlealarm_receive',
-              'frequency = larigira.timeform_base:frequencyalarm_receive',
-              'cron = larigira.timeform_cron:cronalarm_receive',
+        "larigira.timeform_create": [
+            "single = larigira.timeform_base:SingleAlarmForm",
+            "frequency = larigira.timeform_base:FrequencyAlarmForm",
+            "cron = larigira.timeform_cron:CronAlarmForm",
         ],
-          'larigira.audioform_create': [
-              'static = larigira.audioform_static:StaticAudioForm',
-              'http = larigira.audioform_http:AudioForm',
-              'script = larigira.audioform_script:ScriptAudioForm',
-              'randomdir = larigira.audioform_randomdir:Form',
-              'mostrecent = larigira.audioform_mostrecent:AudioForm',
+        "larigira.timeform_receive": [
+            "single = larigira.timeform_base:singlealarm_receive",
+            "frequency = larigira.timeform_base:frequencyalarm_receive",
+            "cron = larigira.timeform_cron:cronalarm_receive",
         ],
-          'larigira.audioform_receive': [
-              'static = larigira.audioform_static:staticaudio_receive',
-              'http = larigira.audioform_http:audio_receive',
-              'script = larigira.audioform_script:scriptaudio_receive',
-              'randomdir = larigira.audioform_randomdir:receive',
-              'mostrecent = larigira.audioform_mostrecent:audio_receive',
+        "larigira.audioform_create": [
+            "static = larigira.audioform_static:StaticAudioForm",
+            "http = larigira.audioform_http:AudioForm",
+            "podcast = larigira.audioform_podcast:AudioForm",
+            "script = larigira.audioform_script:ScriptAudioForm",
+            "randomdir = larigira.audioform_randomdir:Form",
+            "mostrecent = larigira.audioform_mostrecent:AudioForm",
         ],
-          'larigira.eventfilter': [
-              'maxwait = larigira.filters:maxwait',
-              'percentwait = larigira.filters:percentwait',
+        "larigira.audioform_receive": [
+            "static = larigira.audioform_static:staticaudio_receive",
+            "http = larigira.audioform_http:audio_receive",
+            "podcast = larigira.audioform_podcast:audio_receive",
+            "script = larigira.audioform_script:scriptaudio_receive",
+            "randomdir = larigira.audioform_randomdir:receive",
+            "mostrecent = larigira.audioform_mostrecent:audio_receive",
+        ],
+        "larigira.eventfilter": [
+            "maxwait = larigira.filters:maxwait",
+            "percentwait = larigira.filters:percentwait",
         ],
     },
     classifiers=[
         "License :: OSI Approved :: GNU Affero General Public License v3",
-          "Programming Language :: Python :: 3",
-      ]
-      )
+        "Development Status :: 5 - Production/Stable",
+        "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Topic :: Multimedia :: Sound/Audio",
+    ],
+)