python3 support

eventman_server.py

@@ -1,9 +1,9 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """EventMan(ager)
 
 Your friendly manager of attendees at an event.
 
-Copyright 2015-2016 Davide Alberani <da@erlug.linux.it>
+Copyright 2015-2017 Davide Alberani <da@erlug.linux.it>
                     RaspiBO <info@raspibo.org>
 
 Licensed under the Apache License, Version 2.0 (the "License");
@@ -38,7 +38,8 @@ import tornado.websocket
 from tornado import gen, escape, process
 
 import utils
-import backend
+import monco
+import collections
 
 ENCODING = 'utf-8'
 PROCESS_TIMEOUT = 60
@@ -102,8 +103,8 @@ class BaseHandler(tornado.web.RequestHandler):
     _users_cache = {}
 
     # A property to access the first value of each argument.
-    arguments = property(lambda self: dict([(k, v[0])
-        for k, v in self.request.arguments.iteritems()]))
+    arguments = property(lambda self: dict([(k, v[0].decode('utf-8'))
+        for k, v in self.request.arguments.items()]))
 
     # A property to access both the UUID and the clean arguments.
     @property
@@ -150,23 +151,26 @@ class BaseHandler(tornado.web.RequestHandler):
         """Convert some textual values to boolean."""
         if isinstance(obj, (list, tuple)):
             obj = obj[0]
-        if isinstance(obj, (str, unicode)):
+        if isinstance(obj, str):
             obj = obj.lower()
         return self._bool_convert.get(obj, obj)
 
     def arguments_tobool(self):
         """Return a dictionary of arguments, converted to booleans where possible."""
-        return dict([(k, self.tobool(v)) for k, v in self.arguments.iteritems()])
+        return dict([(k, self.tobool(v)) for k, v in self.arguments.items()])
 
     def initialize(self, **kwargs):
         """Add every passed (key, value) as attributes of the instance."""
-        for key, value in kwargs.iteritems():
+        for key, value in kwargs.items():
             setattr(self, key, value)
 
     @property
     def current_user(self):
         """Retrieve current user name from the secure cookie."""
-        return self.get_secure_cookie("user")
+        current_user = self.get_secure_cookie("user")
+        if isinstance(current_user, bytes):
+            current_user = current_user.decode('utf-8')
+        return current_user
 
     @property
     def current_user_info(self):
@@ -174,7 +178,7 @@ class BaseHandler(tornado.web.RequestHandler):
         current_user = self.current_user
         if current_user in self._users_cache:
             return self._users_cache[current_user]
-        permissions = set([k for (k, v) in self.permissions.iteritems() if v is True])
+        permissions = set([k for (k, v) in self.permissions.items() if v is True])
         user_info = {'permissions': permissions}
         if current_user:
             user_info['username'] = current_user
@@ -204,7 +208,7 @@ class BaseHandler(tornado.web.RequestHandler):
         collection_permission = self.permissions.get(permission)
         if isinstance(collection_permission, bool):
             return collection_permission
-        if callable(collection_permission):
+        if isinstance(collection_permission, collections.Callable):
             return collection_permission(permission)
         return False
 
@@ -305,7 +309,7 @@ class CollectionHandler(BaseHandler):
         :rtype: str"""
         t = str(time.time()).replace('.', '_')
         seq = str(self.get_next_seq(seq))
-        rand = ''.join([random.choice(self._id_chars) for x in xrange(random_alpha)])
+        rand = ''.join([random.choice(self._id_chars) for x in range(random_alpha)])
         return '-'.join((t, seq, rand))
 
     def _filter_results(self, results, params):
@@ -320,11 +324,11 @@ class CollectionHandler(BaseHandler):
         :rtype: list"""
         if not params:
             return results
-        params = backend.convert(params)
+        params = monco.convert(params)
         filtered = []
         for result in results:
             add = True
-            for key, value in params.iteritems():
+            for key, value in params.items():
                 if key not in result or result[key] != value:
                     add = False
                     break
@@ -338,8 +342,8 @@ class CollectionHandler(BaseHandler):
         :param data: dictionary to clean
         :type data: dict"""
         if isinstance(data, dict):
-            for key in data.keys():
-                if isinstance(key, (str, unicode)) and key.startswith('$'):
+            for key in list(data.keys()):
+                if isinstance(key, str) and key.startswith('$'):
                     del data[key]
         return data
 
@@ -349,7 +353,7 @@ class CollectionHandler(BaseHandler):
         :param data: dictionary to convert
         :type data: dict"""
         ret = {}
-        for key, value in data.iteritems():
+        for key, value in data.items():
             if isinstance(value, (list, tuple, dict)):
                 continue
             try:
@@ -357,7 +361,7 @@ class CollectionHandler(BaseHandler):
                 key = re_env_key.sub('', key)
                 if not key:
                     continue
-                ret[key] = unicode(value).encode(ENCODING)
+                ret[key] = str(value).encode(ENCODING)
             except:
                 continue
         return ret
@@ -382,7 +386,7 @@ class CollectionHandler(BaseHandler):
             if acl and not self.has_permission(permission):
                 return self.build_error(status=401, message='insufficient permissions: %s' % permission)
             handler = getattr(self, 'handle_get_%s' % resource, None)
-            if handler and callable(handler):
+            if handler and isinstance(handler, collections.Callable):
                 output = handler(id_, resource_id, **kwargs) or {}
                 output = self.apply_filter(output, 'get_%s' % resource)
                 self.write(output)
@@ -432,7 +436,7 @@ class CollectionHandler(BaseHandler):
                 return self.build_error(status=401, message='insufficient permissions: %s' % permission)
             # Handle access to sub-resources.
             handler = getattr(self, 'handle_%s_%s' % (method, resource), None)
-            if handler and callable(handler):
+            if handler and isinstance(handler, collections.Callable):
                 data = self.apply_filter(data, 'input_%s_%s' % (method, resource))
                 output = handler(id_, resource_id, data, **kwargs)
                 output = self.apply_filter(output, 'get_%s' % resource)
@@ -478,7 +482,7 @@ class CollectionHandler(BaseHandler):
             if not self.has_permission(permission):
                 return self.build_error(status=401, message='insufficient permissions: %s' % permission)
             method = getattr(self, 'handle_delete_%s' % resource, None)
-            if method and callable(method):
+            if method and isinstance(method, collections.Callable):
                 output = method(id_, resource_id, **kwargs)
                 env['RESOURCE'] = resource
                 if resource_id:
@@ -584,7 +588,7 @@ class CollectionHandler(BaseHandler):
             ws = yield tornado.websocket.websocket_connect(self.build_ws_url(path))
             ws.write_message(message)
             ws.close()
-        except Exception, e:
+        except Exception as e:
             self.logger.error('Error yielding WebSocket message: %s', e)
 
 
@@ -641,7 +645,7 @@ class EventsHandler(CollectionHandler):
         if group_id is None:
             return {'persons': persons}
         this_persons = [p for p in (this_event.get('tickets') or []) if not p.get('cancelled')]
-        this_emails = filter(None, [p.get('email') for p in this_persons])
+        this_emails = [_f for _f in [p.get('email') for p in this_persons] if _f]
         all_query = {'group_id': group_id}
         events = self.db.query('events', all_query)
         for event in events:
@@ -655,7 +659,7 @@ class EventsHandler(CollectionHandler):
         or which set of keys specified in a dictionary match their respective values."""
         for ticket in tickets:
             if isinstance(ticket_id_or_query, dict):
-                if all(ticket.get(k) == v for k, v in ticket_id_or_query.iteritems()):
+                if all(ticket.get(k) == v for k, v in ticket_id_or_query.items()):
                     return ticket
             else:
                 if str(ticket.get('_id')) == ticket_id_or_query:
@@ -765,7 +769,7 @@ class EventsHandler(CollectionHandler):
         # Update an existing entry for a ticket registered at this event.
         self._clean_dict(data)
         uuid, arguments = self.uuid_arguments
-        query = dict([('tickets.%s' % k, v) for k, v in arguments.iteritems()])
+        query = dict([('tickets.%s' % k, v) for k, v in arguments.items()])
         query['_id'] = id_
         if ticket_id is not None:
             query['tickets._id'] = ticket_id
@@ -970,7 +974,7 @@ class EbCSVImportPersonsHandler(BaseHandler):
         #[x.get('email') for x in (event_details[0].get('tickets') or []) if x.get('email')])
         for ticket in (event_details[0].get('tickets') or []):
             all_emails.add('%s_%s_%s' % (ticket.get('name'), ticket.get('surname'), ticket.get('email')))
-        for fieldname, contents in self.request.files.iteritems():
+        for fieldname, contents in self.request.files.items():
             for content in contents:
                 filename = content['filename']
                 parseStats, persons = utils.csvParse(content['body'], remap=self.csvRemap)
@@ -1038,7 +1042,7 @@ class WebSocketEventUpdatesHandler(tornado.websocket.WebSocketHandler):
         logging.debug('WebSocketEventUpdatesHandler.on_message url:%s' % url)
         count = 0
         _to_delete = set()
-        for uuid, client in _ws_clients.get(url, {}).iteritems():
+        for uuid, client in _ws_clients.get(url, {}).items():
             try:
                 client.write_message(message)
             except:
@@ -1132,7 +1136,7 @@ def run():
         ssl_options = dict(certfile=options.ssl_cert, keyfile=options.ssl_key)
 
     # database backend connector
-    db_connector = backend.EventManDB(url=options.mongo_url, dbName=options.db_name)
+    db_connector = monco.Monco(url=options.mongo_url, dbName=options.db_name)
     init_params = dict(db=db_connector, data_dir=options.data_dir, listen_port=options.port,
             authentication=options.authentication, logger=logger, ssl_options=ssl_options)
 

backend.py → monco.py

@@ -1,8 +1,8 @@
-"""EventMan(ager) database backend
+"""Monco: a MongoDB database backend
 
-Classes and functions used to manage events and attendees database.
+Classes and functions used to issue queries to a MongoDB database.
 
-Copyright 2015-2016 Davide Alberani <da@erlug.linux.it>
+Copyright 2016-2017 Davide Alberani <da@erlug.linux.it>
                     RaspiBO <info@raspibo.org>
 
 Licensed under the Apache License, Version 2.0 (the "License");
@@ -23,6 +23,7 @@ from bson.objectid import ObjectId
 re_objectid = re.compile(r'[0-9a-f]{24}')
 
 _force_conversion = {
+    '_id': ObjectId,
     'seq_hex': str,
     'tickets.seq_hex': str
 }
@@ -40,7 +41,8 @@ def convert_obj(obj):
     if isinstance(obj, bool):
         return obj
     try:
-        return ObjectId(obj)
+        if re_objectid.match(obj):
+            return ObjectId(obj)
     except:
         pass
     return obj
@@ -56,9 +58,12 @@ def convert(seq):
     """
     if isinstance(seq, dict):
         d = {}
-        for key, item in seq.iteritems():
+        for key, item in seq.items():
             if key in _force_conversion:
-                d[key] = _force_conversion[key](item)
+                try:
+                    d[key] = _force_conversion[key](item)
+                except:
+                    d[key] = item
             else:
                 d[key] = convert(item)
         return d
@@ -67,23 +72,35 @@ def convert(seq):
     return convert_obj(seq)
 
 
-class EventManDB(object):
+class MoncoError(Exception):
+    """Base class for Monco exceptions."""
+    pass
+
+
+class MoncoConnectionError(MoncoError):
+    """Monco exceptions raise when a connection problem occurs."""
+    pass
+
+
+class Monco(object):
     """MongoDB connector."""
     db = None
     connection = None
 
     # map operations on lists of items.
     _operations = {
-            'update': '$set',
-            'append': '$push',
-            'appendUnique': '$addToSet',
-            'delete': '$pull',
-            'increment': '$inc'
+        'update': '$set',
+        'append': '$push',
+        'appendUnique': '$addToSet',
+        'delete': '$pull',
+        'increment': '$inc'
     }
 
-    def __init__(self, url=None, dbName='eventman'):
+    def __init__(self, dbName, url=None):
         """Initialize the instance, connecting to the database.
 
+        :param dbName: name of the database
+        :type dbName: str (or None to use the dbName passed at initialization)
         :param url: URL of the database
         :type url: str (or None to connect to localhost)
         """
@@ -91,9 +108,11 @@ class EventManDB(object):
         self._dbName = dbName
         self.connect(url)
 
-    def connect(self, url=None, dbName=None):
+    def connect(self, dbName=None, url=None):
         """Connect to the database.
 
+        :param dbName: name of the database
+        :type dbName: str (or None to use the dbName passed at initialization)
         :param url: URL of the database
         :type url: str (or None to connect to localhost)
 
@@ -106,10 +125,26 @@ class EventManDB(object):
             self._url = url
         if dbName:
             self._dbName = dbName
+        if not self._dbName:
+            raise MoncoConnectionError('no database name specified')
         self.connection = pymongo.MongoClient(self._url)
         self.db = self.connection[self._dbName]
         return self.db
 
+    def getOne(self, collection, query=None):
+        """Get a single document with the specified `query`.
+
+        :param collection: search the document in this collection
+        :type collection: str
+        :param query: query to filter the documents
+        :type query: dict or None
+
+        :returns: the first document matching the query
+        :rtype: dict
+        """
+        results = self.query(collection, convert(query))
+        return results and results[0] or {}
+
     def get(self, collection, _id):
         """Get a single document with the specified `_id`.
 
@@ -121,8 +156,7 @@ class EventManDB(object):
         :returns: the document with the given `_id`
         :rtype: dict
         """
-        results = self.query(collection, convert({'_id': _id}))
-        return results and results[0] or {}
+        return self.getOne(collection, {'_id': _id})
 
     def query(self, collection, query=None, condition='or'):
         """Get multiple documents matching a query.
@@ -130,7 +164,7 @@ class EventManDB(object):
         :param collection: search for documents in this collection
         :type collection: str
         :param query: search for documents with those attributes
-        :type query: dict or None
+        :type query: dict, list or None
 
         :returns: list of matching documents
         :rtype: list
@@ -222,7 +256,7 @@ class EventManDB(object):
         operator = self._operations.get(operation)
         if updateList:
             newData = {}
-            for key, value in data.iteritems():
+            for key, value in data.items():
                 newData['%s.$.%s' % (updateList, key)] = value
             data = newData
         res = db[collection].find_and_modify(query=_id_or_query,
@@ -230,6 +264,30 @@ class EventManDB(object):
         lastErrorObject = res.get('lastErrorObject') or {}
         return lastErrorObject.get('updatedExisting', False), res.get('value') or {}
 
+    def updateMany(self, collection, query, data):
+        """Update multiple existing documents.
+
+        query can be an ID or a dict representing a query.
+
+        :param collection: update documents in this collection
+        :type collection: str
+        :param query: a query or a list of attributes in the data that must match
+        :type query: str or :class:`~bson.objectid.ObjectId` or iterable
+        :param data: the updated information to store
+        :type data: dict
+
+        :returns: a dict with the success state and number of updated items
+        :rtype: dict
+        """
+        db = self.connect()
+        data = convert(data or {})
+        query = convert(query)
+        if not isinstance(query, dict):
+            query = {'_id': query}
+        if '_id' in data:
+            del data['_id']
+        return db[collection].update(query, {'$set': data}, multi=True)
+
     def delete(self, collection, _id_or_query=None, force=False):
         """Remove one or more documents from a collection.
 
@@ -240,8 +298,8 @@ class EventManDB(object):
         :param force: force the deletion of all documents, when `_id_or_query` is empty
         :type force: bool
 
-        :returns: how many documents were removed
-        :rtype: int
+        :returns: dictionary with the number or removed documents
+        :rtype: dict
         """
         if not _id_or_query and not force:
             return
@@ -250,4 +308,3 @@ class EventManDB(object):
             _id_or_query = {'_id': _id_or_query}
         _id_or_query = convert(_id_or_query)
         return db[collection].remove(_id_or_query)
-

utils.py

@@ -22,7 +22,7 @@ import string
 import random
 import hashlib
 import datetime
-import StringIO
+import io
 from bson.objectid import ObjectId
 
 
@@ -39,7 +39,9 @@ def csvParse(csvStr, remap=None, merge=None):
     :returns: tuple with a dict of total and valid lines and the data
     :rtype: tuple
     """
-    fd = StringIO.StringIO(csvStr)
+    if isinstance(csvStr, bytes):
+        csvStr = csvStr.decode('utf-8')
+    fd = io.StringIO(csvStr)
     reader = csv.reader(fd)
     remap = remap or {}
     merge = merge or {}
@@ -47,7 +49,7 @@ def csvParse(csvStr, remap=None, merge=None):
     reply = dict(total=0, valid=0)
     results = []
     try:
-        headers = reader.next()
+        headers = next(reader)
         fields = len(headers)
     except (StopIteration, csv.Error):
         return reply, {}
@@ -63,8 +65,7 @@ def csvParse(csvStr, remap=None, merge=None):
                 reply['total'] += 1
                 if len(row) != fields:
                     continue
-                row = [unicode(cell, 'utf-8', 'replace') for cell in row]
-                values = dict(map(None, headers, row))
+                values = dict(zip(headers, row))
                 values.update(merge)
                 results.append(values)
                 reply['valid'] += 1
@@ -88,19 +89,25 @@ def hash_password(password, salt=None):
     :rtype: str"""
     if salt is None:
         salt_pool = string.ascii_letters + string.digits
-        salt = ''.join(random.choice(salt_pool) for x in xrange(32))
-    hash_ = hashlib.sha512('%s%s' % (salt, password))
+        salt = ''.join(random.choice(salt_pool) for x in range(32))
+    pwd = '%s%s' % (salt, password)
+    hash_ = hashlib.sha512(pwd.encode('utf-8'))
     return '$%s$%s' % (salt, hash_.hexdigest())
 
 
 class ImprovedEncoder(json.JSONEncoder):
     """Enhance the default JSON encoder to serialize datetime and ObjectId instances."""
     def default(self, o):
-        if isinstance(o, (datetime.datetime, datetime.date,
+        if isinstance(o, bytes):
+            try:
+                return o.decode('utf-8')
+            except:
+                pass
+        elif isinstance(o, (datetime.datetime, datetime.date,
                 datetime.time, datetime.timedelta, ObjectId)):
             try:
                 return str(o)
-            except Exception, e:
+            except Exception as e:
                 pass
         elif isinstance(o, set):
             return list(o)