diffido.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """Diffido - because the F5 key is a terrible thing to waste.
  4. Copyright 2018 Davide Alberani <da@erlug.linux.it>
  5. Licensed under the Apache License, Version 2.0 (the "License");
  6. you may not use this file except in compliance with the License.
  7. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
  8. Unless required by applicable law or agreed to in writing, software
  9. distributed under the License is distributed on an "AS IS" BASIS,
  10. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11. See the License for the specific language governing permissions and
  12. limitations under the License.
  13. """
  14. import os
  15. import re
  16. import io
  17. import json
  18. import pytz
  19. import shutil
  20. import urllib
  21. import smtplib
  22. from email.mime.text import MIMEText
  23. import logging
  24. import datetime
  25. import requests
  26. import subprocess
  27. import multiprocessing
  28. from lxml import etree
  29. from xml.etree import ElementTree
  30. from tornado.ioloop import IOLoop
  31. from apscheduler.triggers.cron import CronTrigger
  32. from apscheduler.schedulers.tornado import TornadoScheduler
  33. from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
  34. import tornado.httpserver
  35. import tornado.ioloop
  36. import tornado.options
  37. from tornado.options import define, options
  38. import tornado.web
  39. from tornado import gen, escape
  40. JOBS_STORE = 'sqlite:///conf/jobs.db'
  41. API_VERSION = '1.0'
  42. SCHEDULES_FILE = 'conf/schedules.json'
  43. DEFAULT_CONF = 'conf/diffido.conf'
  44. EMAIL_FROM = 'diffido@localhost'
  45. SMTP_SETTINGS = {}
  46. GIT_CMD = 'git'
  47. re_commit = re.compile(r'^(?P<id>[0-9a-f]{40}) (?P<message>.*)\n(?: .* '
  48. '(?P<insertions>\d+) insertion.* (?P<deletions>\d+) deletion.*$)?', re.M)
  49. re_insertion = re.compile(r'(\d+) insertion')
  50. re_deletion = re.compile(r'(\d+) deletion')
  51. logger = logging.getLogger()
  52. logger.setLevel(logging.INFO)
  53. def read_schedules():
  54. """Return the schedules configuration.
  55. :returns: dictionary from the JSON object in conf/schedules.json
  56. :rtype: dict"""
  57. if not os.path.isfile(SCHEDULES_FILE):
  58. return {'schedules': {}}
  59. try:
  60. with open(SCHEDULES_FILE, 'r') as fd:
  61. schedules = json.loads(fd.read())
  62. for id_ in schedules.get('schedules', {}).keys():
  63. schedule = schedules['schedules'][id_]
  64. try:
  65. schedule['last_history'] = get_last_history(id_)
  66. except:
  67. schedule['last_history'] = {}
  68. continue
  69. return schedules
  70. except Exception as e:
  71. logger.error('unable to read %s: %s' % (SCHEDULES_FILE, e))
  72. return {'schedules': {}}
  73. def write_schedules(schedules):
  74. """Write the schedules configuration.
  75. :param schedules: the schedules to save
  76. :type schedules: dict
  77. :returns: True in case of success
  78. :rtype: bool"""
  79. try:
  80. with open(SCHEDULES_FILE, 'w') as fd:
  81. fd.write(json.dumps(schedules, indent=2))
  82. except Exception as e:
  83. logger.error('unable to write %s: %s' % (SCHEDULES_FILE, e))
  84. return False
  85. return True
  86. def next_id(schedules):
  87. """Return the next available integer (as a string) in the list of schedules keys (do not fills holes)
  88. :param schedules: the schedules
  89. :type schedules: dict
  90. :returns: the ID of the next schedule
  91. :rtype: str"""
  92. ids = schedules.get('schedules', {}).keys()
  93. if not ids:
  94. return '1'
  95. return str(max([int(i) for i in ids]) + 1)
  96. def get_schedule(id_, add_id=True, add_history=False):
  97. """Return information about a single schedule
  98. :param id_: ID of the schedule
  99. :type id_: str
  100. :param add_id: if True, add the ID in the dictionary
  101. :type add_id: bool
  102. :returns: the schedule
  103. :rtype: dict"""
  104. try:
  105. schedules = read_schedules()
  106. except Exception:
  107. return {}
  108. data = schedules.get('schedules', {}).get(id_, {})
  109. if add_history and data:
  110. data['last_history'] = get_last_history(id_)
  111. if add_id:
  112. data['id'] = str(id_)
  113. return data
  114. def select_xpath(content, xpath):
  115. """Select a portion of a HTML document
  116. :param content: the content of the document
  117. :type content: str
  118. :param xpath: the XPath selector
  119. :type xpath: str
  120. :returns: the selected document
  121. :rtype: str"""
  122. fd = io.StringIO(content)
  123. tree = etree.parse(fd)
  124. elems = tree.xpath(xpath)
  125. if not elems:
  126. return content
  127. selected_content = []
  128. for elem in elems:
  129. selected_content.append(''.join([elem.text] + [ElementTree.tostring(e).decode('utf-8', 'replace')
  130. for e in elem.getchildren()]))
  131. content = ''.join(selected_content)
  132. return content
  133. def run_job(id_=None, force=False, *args, **kwargs):
  134. """Run a job
  135. :param id_: ID of the schedule to run
  136. :type id_: str
  137. :param force: run even if disabled
  138. :type force: bool
  139. :param args: positional arguments
  140. :type args: tuple
  141. :param kwargs: named arguments
  142. :type kwargs: dict
  143. :returns: True in case of success
  144. :rtype: bool"""
  145. schedule = get_schedule(id_, add_id=False)
  146. url = schedule.get('url')
  147. if not url:
  148. return False
  149. logger.debug('running job id:%s title:%s url: %s' % (id_, schedule.get('title', ''), url))
  150. if not schedule.get('enabled') and not force:
  151. logger.info('not running job %s: disabled' % id_)
  152. return True
  153. req = requests.get(url, allow_redirects=True, timeout=(30.10, 240))
  154. content = req.text
  155. xpath = schedule.get('xpath')
  156. if xpath:
  157. try:
  158. content = select_xpath(content, xpath)
  159. except Exception as e:
  160. logger.warn('unable to extract XPath %s: %s' % (xpath, e))
  161. req_path = urllib.parse.urlparse(req.url).path
  162. base_name = os.path.basename(req_path) or 'index.html'
  163. def _commit(id_, filename, content, queue):
  164. os.chdir('storage/%s' % id_)
  165. current_lines = 0
  166. if os.path.isfile(filename):
  167. with open(filename, 'r') as fd:
  168. for line in fd:
  169. current_lines += 1
  170. with open(filename, 'w') as fd:
  171. fd.write(content)
  172. p = subprocess.Popen([GIT_CMD, 'add', filename])
  173. p.communicate()
  174. p = subprocess.Popen([GIT_CMD, 'commit', '-m', '%s' % datetime.datetime.utcnow(), '--allow-empty'],
  175. stdout=subprocess.PIPE)
  176. stdout, _ = p.communicate()
  177. stdout = stdout.decode('utf-8')
  178. insert = re_insertion.findall(stdout)
  179. if insert:
  180. insert = int(insert[0])
  181. else:
  182. insert = 0
  183. delete = re_deletion.findall(stdout)
  184. if delete:
  185. delete = int(delete[0])
  186. else:
  187. delete = 0
  188. queue.put({'insertions': insert, 'deletions': delete, 'previous_lines': current_lines,
  189. 'changes': max(insert, delete)})
  190. queue = multiprocessing.Queue()
  191. p = multiprocessing.Process(target=_commit, args=(id_, base_name, content, queue))
  192. p.start()
  193. res = queue.get()
  194. p.join()
  195. email = schedule.get('email')
  196. if not email:
  197. return True
  198. changes = res.get('changes')
  199. if not changes:
  200. return True
  201. min_change = schedule.get('minimum_change')
  202. previous_lines = res.get('previous_lines')
  203. if min_change and previous_lines:
  204. min_change = float(min_change)
  205. change_fraction = res.get('changes') / previous_lines
  206. if change_fraction < min_change:
  207. return True
  208. # send notification
  209. diff = get_diff(id_).get('diff')
  210. if not diff:
  211. return True
  212. send_email(to=email, subject='%s page changed' % schedule.get('title'),
  213. body='changes:\n\n%s' % diff)
  214. return True
  215. def safe_run_job(id_=None, *args, **kwargs):
  216. """Safely run a job, catching all the exceptions
  217. :param id_: ID of the schedule to run
  218. :type id_: str
  219. :param args: positional arguments
  220. :type args: tuple
  221. :param kwargs: named arguments
  222. :type kwargs: dict
  223. :returns: True in case of success
  224. :rtype: bool"""
  225. try:
  226. run_job(id_, *args, **kwargs)
  227. except Exception as e:
  228. send_email('error executing job %s: %s' % (id_, e))
  229. def send_email(to, subject='diffido', body='', from_=None):
  230. """Send an email
  231. :param to: destination address
  232. :type to: str
  233. :param subject: email subject
  234. :type subject: str
  235. :param body: body of the email
  236. :type body: str
  237. :param from_: sender address
  238. :type from_: str
  239. :returns: True in case of success
  240. :rtype: bool"""
  241. msg = MIMEText(body)
  242. msg['Subject'] = subject
  243. msg['From'] = from_ or EMAIL_FROM
  244. msg['To'] = to
  245. starttls = SMTP_SETTINGS.get('smtp-starttls')
  246. use_ssl = SMTP_SETTINGS.get('smtp-use-ssl')
  247. args = {}
  248. for key, value in SMTP_SETTINGS.items():
  249. if key in ('smtp-starttls', 'smtp-use-ssl'):
  250. continue
  251. if key in ('smtp-port'):
  252. value = int(value)
  253. key = key.replace('smtp-', '', 1).replace('-', '_')
  254. args[key] = value
  255. try:
  256. if use_ssl:
  257. with smtplib.SMTP_SSL(**args) as s:
  258. s.send_message(msg)
  259. else:
  260. tls_args = {}
  261. for key in ('ssl_keyfile', 'ssl_certfile', 'ssl_context'):
  262. if key in args:
  263. tls_args = args[key]
  264. del args[key]
  265. with smtplib.SMTP(**args) as s:
  266. if starttls:
  267. s.starttls(**tls_args)
  268. s.ehlo_or_helo_if_needed()
  269. s.send_message(msg)
  270. except Exception as e:
  271. logger.error('unable to send email to %s: %s' % (to, e))
  272. return False
  273. return True
  274. def get_history(id_, limit=None, add_info=False):
  275. """Read the history of a schedule
  276. :param id_: ID of the schedule
  277. :type id_: str
  278. :param limit: number of entries to fetch
  279. :type limit: int
  280. :param add_info: add information about the schedule itself
  281. :type add_info: int
  282. :returns: information about the schedule and its history
  283. :rtype: dict"""
  284. def _history(id_, limit, queue):
  285. try:
  286. os.chdir('storage/%s' % id_)
  287. except:
  288. return queue.put(b'')
  289. cmd = [GIT_CMD, 'log', '--pretty=oneline', '--shortstat']
  290. if limit is not None:
  291. cmd.append('-%s' % limit)
  292. p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
  293. stdout, _ = p.communicate()
  294. queue.put(stdout)
  295. queue = multiprocessing.Queue()
  296. p = multiprocessing.Process(target=_history, args=(id_, limit, queue))
  297. p.start()
  298. res = queue.get().decode('utf-8')
  299. p.join()
  300. history = []
  301. for match in re_commit.finditer(res):
  302. info = match.groupdict()
  303. info['insertions'] = int(info['insertions'] or 0)
  304. info['deletions'] = int(info['deletions'] or 0)
  305. info['changes'] = max(info['insertions'], info['deletions'])
  306. history.append(info)
  307. last_id = None
  308. if history and 'id' in history[0]:
  309. last_id = history[0]['id']
  310. for idx, item in enumerate(history):
  311. item['seq'] = idx + 1
  312. data = {'history': history, 'last_id': last_id}
  313. if add_info:
  314. data['schedule'] = get_schedule(id_)
  315. return data
  316. def get_last_history(id_):
  317. """Read the last history entry of a schedule
  318. :param id_: ID of the schedule
  319. :type id_: str
  320. :returns: information about the schedule and its history
  321. :rtype: dict"""
  322. history = get_history(id_, limit=1)
  323. hist = history.get('history') or [{}]
  324. return hist[0]
  325. def get_diff(id_, commit_id='HEAD', old_commit_id=None):
  326. """Return the diff between commits of a schedule
  327. :param id_: ID of the schedule
  328. :type id_: str
  329. :param commit_id: the most recent commit ID; HEAD by default
  330. :type commit_id: str
  331. :param old_commit_id: the older commit ID; if None, the previous commit is used
  332. :type old_commit_id: str
  333. :returns: information about the schedule and the diff between commits
  334. :rtype: dict"""
  335. def _history(id_, commit_id, old_commit_id, queue):
  336. os.chdir('storage/%s' % id_)
  337. p = subprocess.Popen([GIT_CMD, 'diff', old_commit_id or '%s~' % commit_id, commit_id],
  338. stdout=subprocess.PIPE)
  339. stdout, _ = p.communicate()
  340. queue.put(stdout)
  341. queue = multiprocessing.Queue()
  342. p = multiprocessing.Process(target=_history, args=(id_, commit_id, old_commit_id, queue))
  343. p.start()
  344. res = queue.get().decode('utf-8')
  345. p.join()
  346. schedule = get_schedule(id_)
  347. return {'diff': res, 'schedule': schedule}
  348. def scheduler_update(scheduler, id_):
  349. """Update a scheduler job, using information from the JSON object
  350. :param scheduler: the TornadoScheduler instance to modify
  351. :type scheduler: TornadoScheduler
  352. :param id_: ID of the schedule that must be updated
  353. :type id_: str
  354. :returns: True in case of success
  355. :rtype: bool"""
  356. schedule = get_schedule(id_, add_id=False)
  357. if not schedule:
  358. logger.warn('unable to update empty schedule %s' % id_)
  359. return False
  360. trigger = schedule.get('trigger')
  361. if trigger not in ('interval', 'cron'):
  362. logger.warn('unable to update empty schedule %s: trigger not in ("cron", "interval")' % id_)
  363. return False
  364. args = {}
  365. if trigger == 'interval':
  366. args['trigger'] = 'interval'
  367. for unit in 'weeks', 'days', 'hours', 'minutes', 'seconds':
  368. if 'interval_%s' % unit not in schedule:
  369. continue
  370. try:
  371. val = schedule['interval_%s' % unit]
  372. if not val:
  373. continue
  374. args[unit] = int(val)
  375. except Exception:
  376. logger.warn('invalid argument on schedule %s: %s parameter %s is not an integer' %
  377. (id_, 'interval_%s' % unit, schedule['interval_%s' % unit]))
  378. if len(args) == 1:
  379. logger.error('no valid interval specified, skipping schedule %s' % id_)
  380. return False
  381. elif trigger == 'cron':
  382. try:
  383. cron_trigger = CronTrigger.from_crontab(schedule['cron_crontab'])
  384. args['trigger'] = cron_trigger
  385. except Exception:
  386. logger.warn('invalid argument on schedule %s: cron_tab parameter %s is not a valid crontab' %
  387. (id_, schedule.get('cron_crontab')))
  388. return False
  389. git_create_repo(id_)
  390. try:
  391. scheduler.add_job(safe_run_job, id=id_, replace_existing=True, kwargs={'id_': id_}, **args)
  392. except Exception as e:
  393. logger.warn('unable to update job %s: %s' % (id_, e))
  394. return False
  395. return True
  396. def scheduler_delete(scheduler, id_):
  397. """Update a scheduler job, using information from the JSON object
  398. :param scheduler: the TornadoScheduler instance to modify
  399. :type scheduler: TornadoScheduler
  400. :param id_: ID of the schedule
  401. :type id_: str
  402. :returns: True in case of success
  403. :rtype: bool"""
  404. try:
  405. scheduler.remove_job(job_id=id_)
  406. except Exception as e:
  407. logger.warn('unable to delete job %s: %s' % (id_, e))
  408. return False
  409. return git_delete_repo(id_)
  410. def reset_from_schedules(scheduler):
  411. """"Reset all scheduler jobs, using information from the JSON object
  412. :param scheduler: the TornadoScheduler instance to modify
  413. :type scheduler: TornadoScheduler
  414. :returns: True in case of success
  415. :rtype: bool"""
  416. ret = False
  417. try:
  418. scheduler.remove_all_jobs()
  419. for key in read_schedules().get('schedules', {}).keys():
  420. ret |= scheduler_update(scheduler, id_=key)
  421. except Exception as e:
  422. logger.warn('unable to reset all jobs: %s' % e)
  423. return False
  424. return ret
  425. def git_init():
  426. """Initialize Git global settings"""
  427. p = subprocess.Popen([GIT_CMD, 'config', '--global', 'user.email', '"%s"' % EMAIL_FROM])
  428. p.communicate()
  429. p = subprocess.Popen([GIT_CMD, 'config', '--global', 'user.name', '"Diffido"'])
  430. p.communicate()
  431. def git_create_repo(id_):
  432. """Create a Git repository
  433. :param id_: ID of the schedule
  434. :type id_: str
  435. :returns: True in case of success
  436. :rtype: bool"""
  437. repo_dir = 'storage/%s' % id_
  438. if os.path.isdir(repo_dir):
  439. return True
  440. p = subprocess.Popen([GIT_CMD, 'init', repo_dir])
  441. p.communicate()
  442. return p.returncode == 0
  443. def git_delete_repo(id_):
  444. """Delete a Git repository
  445. :param id_: ID of the schedule
  446. :type id_: str
  447. :returns: True in case of success
  448. :rtype: bool"""
  449. repo_dir = 'storage/%s' % id_
  450. if not os.path.isdir(repo_dir):
  451. return False
  452. try:
  453. shutil.rmtree(repo_dir)
  454. except Exception as e:
  455. logger.warn('unable to delete Git repository %s: %s' % (id_, e))
  456. return False
  457. return True
  458. class DiffidoBaseException(Exception):
  459. """Base class for diffido custom exceptions.
  460. :param message: text message
  461. :type message: str
  462. :param status: numeric http status code
  463. :type status: int"""
  464. def __init__(self, message, status=400):
  465. super(DiffidoBaseException, self).__init__(message)
  466. self.message = message
  467. self.status = status
  468. class BaseHandler(tornado.web.RequestHandler):
  469. """Base class for request handlers."""
  470. # A property to access the first value of each argument.
  471. arguments = property(lambda self: dict([(k, v[0].decode('utf-8'))
  472. for k, v in self.request.arguments.items()]))
  473. @property
  474. def clean_body(self):
  475. """Return a clean dictionary from a JSON body, suitable for a query on MongoDB.
  476. :returns: a clean copy of the body arguments
  477. :rtype: dict"""
  478. return escape.json_decode(self.request.body or '{}')
  479. def write_error(self, status_code, **kwargs):
  480. """Default error handler."""
  481. if isinstance(kwargs.get('exc_info', (None, None))[1], DiffidoBaseException):
  482. exc = kwargs['exc_info'][1]
  483. status_code = exc.status
  484. message = exc.message
  485. else:
  486. message = 'internal error'
  487. self.build_error(message, status=status_code)
  488. def initialize(self, **kwargs):
  489. """Add every passed (key, value) as attributes of the instance."""
  490. for key, value in kwargs.items():
  491. setattr(self, key, value)
  492. def build_error(self, message='', status=400):
  493. """Build and write an error message.
  494. :param message: textual message
  495. :type message: str
  496. :param status: HTTP status code
  497. :type status: int
  498. """
  499. self.set_status(status)
  500. self.write({'error': True, 'message': message})
  501. def build_success(self, message='', status=200):
  502. """Build and write a success message.
  503. :param message: textual message
  504. :type message: str
  505. :param status: HTTP status code
  506. :type status: int
  507. """
  508. self.set_status(status)
  509. self.write({'error': False, 'message': message})
  510. class SchedulesHandler(BaseHandler):
  511. """Schedules handler."""
  512. @gen.coroutine
  513. def get(self, id_=None, *args, **kwargs):
  514. """Get a schedule."""
  515. if id_ is not None:
  516. return self.write({'schedule': get_schedule(id_, add_history=True)})
  517. schedules = read_schedules()
  518. self.write(schedules)
  519. @gen.coroutine
  520. def put(self, id_=None, *args, **kwargs):
  521. """Update a schedule."""
  522. if id_ is None:
  523. return self.build_error(message='update action requires an ID')
  524. data = self.clean_body
  525. schedules = read_schedules()
  526. if id_ not in schedules.get('schedules', {}):
  527. return self.build_error(message='schedule %s not found' % id_)
  528. schedules['schedules'][id_] = data
  529. write_schedules(schedules)
  530. scheduler_update(scheduler=self.scheduler, id_=id_)
  531. self.write(get_schedule(id_=id_))
  532. @gen.coroutine
  533. def post(self, *args, **kwargs):
  534. """Add a schedule."""
  535. data = self.clean_body
  536. schedules = read_schedules()
  537. id_ = next_id(schedules)
  538. schedules['schedules'][id_] = data
  539. write_schedules(schedules)
  540. scheduler_update(scheduler=self.scheduler, id_=id_)
  541. self.write(get_schedule(id_=id_))
  542. @gen.coroutine
  543. def delete(self, id_=None, *args, **kwargs):
  544. """Delete a schedule."""
  545. if id_ is None:
  546. return self.build_error(message='an ID must be specified')
  547. schedules = read_schedules()
  548. if id_ in schedules.get('schedules', {}):
  549. del schedules['schedules'][id_]
  550. write_schedules(schedules)
  551. scheduler_delete(scheduler=self.scheduler, id_=id_)
  552. self.build_success(message='removed schedule %s' % id_)
  553. class RunScheduleHandler(BaseHandler):
  554. """Reset schedules handler."""
  555. @gen.coroutine
  556. def post(self, id_, *args, **kwargs):
  557. if run_job(id_, force=True):
  558. return self.build_success('job run')
  559. self.build_error('job not run')
  560. class ResetSchedulesHandler(BaseHandler):
  561. """Reset schedules handler."""
  562. @gen.coroutine
  563. def post(self, *args, **kwargs):
  564. reset_from_schedules(self.scheduler)
  565. class HistoryHandler(BaseHandler):
  566. """History handler."""
  567. @gen.coroutine
  568. def get(self, id_, *args, **kwargs):
  569. self.write(get_history(id_, add_info=True))
  570. class DiffHandler(BaseHandler):
  571. """Diff handler."""
  572. @gen.coroutine
  573. def get(self, id_, commit_id, old_commit_id=None, *args, **kwargs):
  574. self.write(get_diff(id_, commit_id, old_commit_id))
  575. class TemplateHandler(BaseHandler):
  576. """Handler for the template files in the / path."""
  577. @gen.coroutine
  578. def get(self, *args, **kwargs):
  579. """Get a template file."""
  580. page = 'index.html'
  581. if args and args[0]:
  582. page = args[0].strip('/')
  583. arguments = self.arguments
  584. self.render(page, **arguments)
  585. def serve():
  586. """Read configuration and start the server."""
  587. global EMAIL_FROM, SMTP_SETTINGS
  588. jobstores = {'default': SQLAlchemyJobStore(url=JOBS_STORE)}
  589. scheduler = TornadoScheduler(jobstores=jobstores, timezone=pytz.utc)
  590. scheduler.start()
  591. define('port', default=3210, help='run on the given port', type=int)
  592. define('address', default='', help='bind the server at the given address', type=str)
  593. define('ssl_cert', default=os.path.join(os.path.dirname(__file__), 'ssl', 'diffido_cert.pem'),
  594. help='specify the SSL certificate to use for secure connections')
  595. define('ssl_key', default=os.path.join(os.path.dirname(__file__), 'ssl', 'diffido_key.pem'),
  596. help='specify the SSL private key to use for secure connections')
  597. define('admin-email', default='', help='email address of the site administrator', type=str)
  598. define('smtp-host', default='localhost', help='SMTP server address', type=str)
  599. define('smtp-port', default=0, help='SMTP server port', type=int)
  600. define('smtp-local-hostname', default=None, help='SMTP local hostname', type=str)
  601. define('smtp-use-ssl', default=False, help='Use SSL to connect to the SMTP server', type=bool)
  602. define('smtp-starttls', default=False, help='Use STARTTLS to connect to the SMTP server', type=bool)
  603. define('smtp-ssl-keyfile', default=None, help='SSL key file', type=str)
  604. define('smtp-ssl-certfile', default=None, help='SSL cert file', type=str)
  605. define('smtp-ssl-context', default=None, help='SSL context', type=str)
  606. define('debug', default=False, help='run in debug mode', type=bool)
  607. define('config', help='read configuration file',
  608. callback=lambda path: tornado.options.parse_config_file(path, final=False))
  609. if not options.config and os.path.isfile(DEFAULT_CONF):
  610. tornado.options.parse_config_file(DEFAULT_CONF, final=False)
  611. tornado.options.parse_command_line()
  612. if options.admin_email:
  613. EMAIL_FROM = options.admin_email
  614. for key, value in options.as_dict().items():
  615. if key.startswith('smtp-'):
  616. SMTP_SETTINGS[key] = value
  617. if options.debug:
  618. logger.setLevel(logging.DEBUG)
  619. ssl_options = {}
  620. if os.path.isfile(options.ssl_key) and os.path.isfile(options.ssl_cert):
  621. ssl_options = dict(certfile=options.ssl_cert, keyfile=options.ssl_key)
  622. init_params = dict(listen_port=options.port, logger=logger, ssl_options=ssl_options,
  623. scheduler=scheduler)
  624. git_init()
  625. _reset_schedules_path = r'schedules/reset'
  626. _schedule_run_path = r'schedules/(?P<id_>\d+)/run'
  627. _schedules_path = r'schedules/?(?P<id_>\d+)?'
  628. _history_path = r'schedules/?(?P<id_>\d+)/history'
  629. _diff_path = r'schedules/(?P<id_>\d+)/diff/(?P<commit_id>[0-9a-f]+)/?(?P<old_commit_id>[0-9a-f]+)?/?'
  630. application = tornado.web.Application([
  631. (r'/api/%s' % _reset_schedules_path, ResetSchedulesHandler, init_params),
  632. (r'/api/v%s/%s' % (API_VERSION, _reset_schedules_path), ResetSchedulesHandler, init_params),
  633. (r'/api/%s' % _schedule_run_path, RunScheduleHandler, init_params),
  634. (r'/api/v%s/%s' % (API_VERSION, _schedule_run_path), RunScheduleHandler, init_params),
  635. (r'/api/%s' % _history_path, HistoryHandler, init_params),
  636. (r'/api/v%s/%s' % (API_VERSION, _history_path), HistoryHandler, init_params),
  637. (r'/api/%s' % _diff_path, DiffHandler, init_params),
  638. (r'/api/v%s/%s' % (API_VERSION, _diff_path), DiffHandler, init_params),
  639. (r'/api/%s' % _schedules_path, SchedulesHandler, init_params),
  640. (r'/api/v%s/%s' % (API_VERSION, _schedules_path), SchedulesHandler, init_params),
  641. (r'/?(.*)', TemplateHandler, init_params),
  642. ],
  643. static_path=os.path.join(os.path.dirname(__file__), 'dist/static'),
  644. template_path=os.path.join(os.path.dirname(__file__), 'dist/'),
  645. debug=options.debug)
  646. http_server = tornado.httpserver.HTTPServer(application, ssl_options=ssl_options or None)
  647. logger.info('Start serving on %s://%s:%d', 'https' if ssl_options else 'http',
  648. options.address if options.address else '127.0.0.1',
  649. options.port)
  650. http_server.listen(options.port, options.address)
  651. try:
  652. IOLoop.instance().start()
  653. except (KeyboardInterrupt, SystemExit):
  654. pass
  655. if __name__ == '__main__':
  656. serve()