From d0776487a214673e9280856df28d7ca3b20e5132 Mon Sep 17 00:00:00 2001 From: Davide Alberani Date: Mon, 27 Mar 2017 21:57:40 +0200 Subject: [PATCH] python3 support --- eventman_server.py | 60 +++++++++++++------------ backend.py => monco.py | 99 +++++++++++++++++++++++++++++++++--------- utils.py | 25 +++++++---- 3 files changed, 126 insertions(+), 58 deletions(-) rename backend.py => monco.py (74%) diff --git a/eventman_server.py b/eventman_server.py index 51f640b..98d7325 100755 --- a/eventman_server.py +++ b/eventman_server.py @@ -1,9 +1,9 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """EventMan(ager) Your friendly manager of attendees at an event. -Copyright 2015-2016 Davide Alberani +Copyright 2015-2017 Davide Alberani RaspiBO Licensed under the Apache License, Version 2.0 (the "License"); @@ -38,7 +38,8 @@ import tornado.websocket from tornado import gen, escape, process import utils -import backend +import monco +import collections ENCODING = 'utf-8' PROCESS_TIMEOUT = 60 @@ -102,8 +103,8 @@ class BaseHandler(tornado.web.RequestHandler): _users_cache = {} # A property to access the first value of each argument. - arguments = property(lambda self: dict([(k, v[0]) - for k, v in self.request.arguments.iteritems()])) + arguments = property(lambda self: dict([(k, v[0].decode('utf-8')) + for k, v in self.request.arguments.items()])) # A property to access both the UUID and the clean arguments. @property @@ -150,23 +151,26 @@ class BaseHandler(tornado.web.RequestHandler): """Convert some textual values to boolean.""" if isinstance(obj, (list, tuple)): obj = obj[0] - if isinstance(obj, (str, unicode)): + if isinstance(obj, str): obj = obj.lower() return self._bool_convert.get(obj, obj) def arguments_tobool(self): """Return a dictionary of arguments, converted to booleans where possible.""" - return dict([(k, self.tobool(v)) for k, v in self.arguments.iteritems()]) + return dict([(k, self.tobool(v)) for k, v in self.arguments.items()]) def initialize(self, **kwargs): """Add every passed (key, value) as attributes of the instance.""" - for key, value in kwargs.iteritems(): + for key, value in kwargs.items(): setattr(self, key, value) @property def current_user(self): """Retrieve current user name from the secure cookie.""" - return self.get_secure_cookie("user") + current_user = self.get_secure_cookie("user") + if isinstance(current_user, bytes): + current_user = current_user.decode('utf-8') + return current_user @property def current_user_info(self): @@ -174,7 +178,7 @@ class BaseHandler(tornado.web.RequestHandler): current_user = self.current_user if current_user in self._users_cache: return self._users_cache[current_user] - permissions = set([k for (k, v) in self.permissions.iteritems() if v is True]) + permissions = set([k for (k, v) in self.permissions.items() if v is True]) user_info = {'permissions': permissions} if current_user: user_info['username'] = current_user @@ -204,7 +208,7 @@ class BaseHandler(tornado.web.RequestHandler): collection_permission = self.permissions.get(permission) if isinstance(collection_permission, bool): return collection_permission - if callable(collection_permission): + if isinstance(collection_permission, collections.Callable): return collection_permission(permission) return False @@ -305,7 +309,7 @@ class CollectionHandler(BaseHandler): :rtype: str""" t = str(time.time()).replace('.', '_') seq = str(self.get_next_seq(seq)) - rand = ''.join([random.choice(self._id_chars) for x in xrange(random_alpha)]) + rand = ''.join([random.choice(self._id_chars) for x in range(random_alpha)]) return '-'.join((t, seq, rand)) def _filter_results(self, results, params): @@ -320,11 +324,11 @@ class CollectionHandler(BaseHandler): :rtype: list""" if not params: return results - params = backend.convert(params) + params = monco.convert(params) filtered = [] for result in results: add = True - for key, value in params.iteritems(): + for key, value in params.items(): if key not in result or result[key] != value: add = False break @@ -338,8 +342,8 @@ class CollectionHandler(BaseHandler): :param data: dictionary to clean :type data: dict""" if isinstance(data, dict): - for key in data.keys(): - if isinstance(key, (str, unicode)) and key.startswith('$'): + for key in list(data.keys()): + if isinstance(key, str) and key.startswith('$'): del data[key] return data @@ -349,7 +353,7 @@ class CollectionHandler(BaseHandler): :param data: dictionary to convert :type data: dict""" ret = {} - for key, value in data.iteritems(): + for key, value in data.items(): if isinstance(value, (list, tuple, dict)): continue try: @@ -357,7 +361,7 @@ class CollectionHandler(BaseHandler): key = re_env_key.sub('', key) if not key: continue - ret[key] = unicode(value).encode(ENCODING) + ret[key] = str(value).encode(ENCODING) except: continue return ret @@ -382,7 +386,7 @@ class CollectionHandler(BaseHandler): if acl and not self.has_permission(permission): return self.build_error(status=401, message='insufficient permissions: %s' % permission) handler = getattr(self, 'handle_get_%s' % resource, None) - if handler and callable(handler): + if handler and isinstance(handler, collections.Callable): output = handler(id_, resource_id, **kwargs) or {} output = self.apply_filter(output, 'get_%s' % resource) self.write(output) @@ -432,7 +436,7 @@ class CollectionHandler(BaseHandler): return self.build_error(status=401, message='insufficient permissions: %s' % permission) # Handle access to sub-resources. handler = getattr(self, 'handle_%s_%s' % (method, resource), None) - if handler and callable(handler): + if handler and isinstance(handler, collections.Callable): data = self.apply_filter(data, 'input_%s_%s' % (method, resource)) output = handler(id_, resource_id, data, **kwargs) output = self.apply_filter(output, 'get_%s' % resource) @@ -478,7 +482,7 @@ class CollectionHandler(BaseHandler): if not self.has_permission(permission): return self.build_error(status=401, message='insufficient permissions: %s' % permission) method = getattr(self, 'handle_delete_%s' % resource, None) - if method and callable(method): + if method and isinstance(method, collections.Callable): output = method(id_, resource_id, **kwargs) env['RESOURCE'] = resource if resource_id: @@ -584,7 +588,7 @@ class CollectionHandler(BaseHandler): ws = yield tornado.websocket.websocket_connect(self.build_ws_url(path)) ws.write_message(message) ws.close() - except Exception, e: + except Exception as e: self.logger.error('Error yielding WebSocket message: %s', e) @@ -641,7 +645,7 @@ class EventsHandler(CollectionHandler): if group_id is None: return {'persons': persons} this_persons = [p for p in (this_event.get('tickets') or []) if not p.get('cancelled')] - this_emails = filter(None, [p.get('email') for p in this_persons]) + this_emails = [_f for _f in [p.get('email') for p in this_persons] if _f] all_query = {'group_id': group_id} events = self.db.query('events', all_query) for event in events: @@ -655,7 +659,7 @@ class EventsHandler(CollectionHandler): or which set of keys specified in a dictionary match their respective values.""" for ticket in tickets: if isinstance(ticket_id_or_query, dict): - if all(ticket.get(k) == v for k, v in ticket_id_or_query.iteritems()): + if all(ticket.get(k) == v for k, v in ticket_id_or_query.items()): return ticket else: if str(ticket.get('_id')) == ticket_id_or_query: @@ -765,7 +769,7 @@ class EventsHandler(CollectionHandler): # Update an existing entry for a ticket registered at this event. self._clean_dict(data) uuid, arguments = self.uuid_arguments - query = dict([('tickets.%s' % k, v) for k, v in arguments.iteritems()]) + query = dict([('tickets.%s' % k, v) for k, v in arguments.items()]) query['_id'] = id_ if ticket_id is not None: query['tickets._id'] = ticket_id @@ -970,7 +974,7 @@ class EbCSVImportPersonsHandler(BaseHandler): #[x.get('email') for x in (event_details[0].get('tickets') or []) if x.get('email')]) for ticket in (event_details[0].get('tickets') or []): all_emails.add('%s_%s_%s' % (ticket.get('name'), ticket.get('surname'), ticket.get('email'))) - for fieldname, contents in self.request.files.iteritems(): + for fieldname, contents in self.request.files.items(): for content in contents: filename = content['filename'] parseStats, persons = utils.csvParse(content['body'], remap=self.csvRemap) @@ -1038,7 +1042,7 @@ class WebSocketEventUpdatesHandler(tornado.websocket.WebSocketHandler): logging.debug('WebSocketEventUpdatesHandler.on_message url:%s' % url) count = 0 _to_delete = set() - for uuid, client in _ws_clients.get(url, {}).iteritems(): + for uuid, client in _ws_clients.get(url, {}).items(): try: client.write_message(message) except: @@ -1132,7 +1136,7 @@ def run(): ssl_options = dict(certfile=options.ssl_cert, keyfile=options.ssl_key) # database backend connector - db_connector = backend.EventManDB(url=options.mongo_url, dbName=options.db_name) + db_connector = monco.Monco(url=options.mongo_url, dbName=options.db_name) init_params = dict(db=db_connector, data_dir=options.data_dir, listen_port=options.port, authentication=options.authentication, logger=logger, ssl_options=ssl_options) diff --git a/backend.py b/monco.py similarity index 74% rename from backend.py rename to monco.py index 5124b17..81ab1eb 100644 --- a/backend.py +++ b/monco.py @@ -1,8 +1,8 @@ -"""EventMan(ager) database backend +"""Monco: a MongoDB database backend -Classes and functions used to manage events and attendees database. +Classes and functions used to issue queries to a MongoDB database. -Copyright 2015-2016 Davide Alberani +Copyright 2016-2017 Davide Alberani RaspiBO Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,6 +23,7 @@ from bson.objectid import ObjectId re_objectid = re.compile(r'[0-9a-f]{24}') _force_conversion = { + '_id': ObjectId, 'seq_hex': str, 'tickets.seq_hex': str } @@ -40,7 +41,8 @@ def convert_obj(obj): if isinstance(obj, bool): return obj try: - return ObjectId(obj) + if re_objectid.match(obj): + return ObjectId(obj) except: pass return obj @@ -56,9 +58,12 @@ def convert(seq): """ if isinstance(seq, dict): d = {} - for key, item in seq.iteritems(): + for key, item in seq.items(): if key in _force_conversion: - d[key] = _force_conversion[key](item) + try: + d[key] = _force_conversion[key](item) + except: + d[key] = item else: d[key] = convert(item) return d @@ -67,23 +72,35 @@ def convert(seq): return convert_obj(seq) -class EventManDB(object): +class MoncoError(Exception): + """Base class for Monco exceptions.""" + pass + + +class MoncoConnectionError(MoncoError): + """Monco exceptions raise when a connection problem occurs.""" + pass + + +class Monco(object): """MongoDB connector.""" db = None connection = None # map operations on lists of items. _operations = { - 'update': '$set', - 'append': '$push', - 'appendUnique': '$addToSet', - 'delete': '$pull', - 'increment': '$inc' + 'update': '$set', + 'append': '$push', + 'appendUnique': '$addToSet', + 'delete': '$pull', + 'increment': '$inc' } - def __init__(self, url=None, dbName='eventman'): + def __init__(self, dbName, url=None): """Initialize the instance, connecting to the database. + :param dbName: name of the database + :type dbName: str (or None to use the dbName passed at initialization) :param url: URL of the database :type url: str (or None to connect to localhost) """ @@ -91,9 +108,11 @@ class EventManDB(object): self._dbName = dbName self.connect(url) - def connect(self, url=None, dbName=None): + def connect(self, dbName=None, url=None): """Connect to the database. + :param dbName: name of the database + :type dbName: str (or None to use the dbName passed at initialization) :param url: URL of the database :type url: str (or None to connect to localhost) @@ -106,10 +125,26 @@ class EventManDB(object): self._url = url if dbName: self._dbName = dbName + if not self._dbName: + raise MoncoConnectionError('no database name specified') self.connection = pymongo.MongoClient(self._url) self.db = self.connection[self._dbName] return self.db + def getOne(self, collection, query=None): + """Get a single document with the specified `query`. + + :param collection: search the document in this collection + :type collection: str + :param query: query to filter the documents + :type query: dict or None + + :returns: the first document matching the query + :rtype: dict + """ + results = self.query(collection, convert(query)) + return results and results[0] or {} + def get(self, collection, _id): """Get a single document with the specified `_id`. @@ -121,8 +156,7 @@ class EventManDB(object): :returns: the document with the given `_id` :rtype: dict """ - results = self.query(collection, convert({'_id': _id})) - return results and results[0] or {} + return self.getOne(collection, {'_id': _id}) def query(self, collection, query=None, condition='or'): """Get multiple documents matching a query. @@ -130,7 +164,7 @@ class EventManDB(object): :param collection: search for documents in this collection :type collection: str :param query: search for documents with those attributes - :type query: dict or None + :type query: dict, list or None :returns: list of matching documents :rtype: list @@ -222,7 +256,7 @@ class EventManDB(object): operator = self._operations.get(operation) if updateList: newData = {} - for key, value in data.iteritems(): + for key, value in data.items(): newData['%s.$.%s' % (updateList, key)] = value data = newData res = db[collection].find_and_modify(query=_id_or_query, @@ -230,6 +264,30 @@ class EventManDB(object): lastErrorObject = res.get('lastErrorObject') or {} return lastErrorObject.get('updatedExisting', False), res.get('value') or {} + def updateMany(self, collection, query, data): + """Update multiple existing documents. + + query can be an ID or a dict representing a query. + + :param collection: update documents in this collection + :type collection: str + :param query: a query or a list of attributes in the data that must match + :type query: str or :class:`~bson.objectid.ObjectId` or iterable + :param data: the updated information to store + :type data: dict + + :returns: a dict with the success state and number of updated items + :rtype: dict + """ + db = self.connect() + data = convert(data or {}) + query = convert(query) + if not isinstance(query, dict): + query = {'_id': query} + if '_id' in data: + del data['_id'] + return db[collection].update(query, {'$set': data}, multi=True) + def delete(self, collection, _id_or_query=None, force=False): """Remove one or more documents from a collection. @@ -240,8 +298,8 @@ class EventManDB(object): :param force: force the deletion of all documents, when `_id_or_query` is empty :type force: bool - :returns: how many documents were removed - :rtype: int + :returns: dictionary with the number or removed documents + :rtype: dict """ if not _id_or_query and not force: return @@ -250,4 +308,3 @@ class EventManDB(object): _id_or_query = {'_id': _id_or_query} _id_or_query = convert(_id_or_query) return db[collection].remove(_id_or_query) - diff --git a/utils.py b/utils.py index 16ac3bb..ea2f228 100644 --- a/utils.py +++ b/utils.py @@ -22,7 +22,7 @@ import string import random import hashlib import datetime -import StringIO +import io from bson.objectid import ObjectId @@ -39,7 +39,9 @@ def csvParse(csvStr, remap=None, merge=None): :returns: tuple with a dict of total and valid lines and the data :rtype: tuple """ - fd = StringIO.StringIO(csvStr) + if isinstance(csvStr, bytes): + csvStr = csvStr.decode('utf-8') + fd = io.StringIO(csvStr) reader = csv.reader(fd) remap = remap or {} merge = merge or {} @@ -47,7 +49,7 @@ def csvParse(csvStr, remap=None, merge=None): reply = dict(total=0, valid=0) results = [] try: - headers = reader.next() + headers = next(reader) fields = len(headers) except (StopIteration, csv.Error): return reply, {} @@ -63,8 +65,7 @@ def csvParse(csvStr, remap=None, merge=None): reply['total'] += 1 if len(row) != fields: continue - row = [unicode(cell, 'utf-8', 'replace') for cell in row] - values = dict(map(None, headers, row)) + values = dict(zip(headers, row)) values.update(merge) results.append(values) reply['valid'] += 1 @@ -88,19 +89,25 @@ def hash_password(password, salt=None): :rtype: str""" if salt is None: salt_pool = string.ascii_letters + string.digits - salt = ''.join(random.choice(salt_pool) for x in xrange(32)) - hash_ = hashlib.sha512('%s%s' % (salt, password)) + salt = ''.join(random.choice(salt_pool) for x in range(32)) + pwd = '%s%s' % (salt, password) + hash_ = hashlib.sha512(pwd.encode('utf-8')) return '$%s$%s' % (salt, hash_.hexdigest()) class ImprovedEncoder(json.JSONEncoder): """Enhance the default JSON encoder to serialize datetime and ObjectId instances.""" def default(self, o): - if isinstance(o, (datetime.datetime, datetime.date, + if isinstance(o, bytes): + try: + return o.decode('utf-8') + except: + pass + elif isinstance(o, (datetime.datetime, datetime.date, datetime.time, datetime.timedelta, ObjectId)): try: return str(o) - except Exception, e: + except Exception as e: pass elif isinstance(o, set): return list(o)