diffido.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """Diffido - because the F5 key is a terrible thing to waste.
  4. Copyright 2018 Davide Alberani <da@erlug.linux.it>
  5. Licensed under the Apache License, Version 2.0 (the "License");
  6. you may not use this file except in compliance with the License.
  7. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
  8. Unless required by applicable law or agreed to in writing, software
  9. distributed under the License is distributed on an "AS IS" BASIS,
  10. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11. See the License for the specific language governing permissions and
  12. limitations under the License.
  13. """
  14. import os
  15. import re
  16. import json
  17. import pytz
  18. import shutil
  19. import urllib
  20. import smtplib
  21. from email.mime.text import MIMEText
  22. import logging
  23. import datetime
  24. import requests
  25. import subprocess
  26. import multiprocessing
  27. from lxml import etree
  28. from xml.etree import ElementTree
  29. from tornado.ioloop import IOLoop
  30. from apscheduler.triggers.cron import CronTrigger
  31. from apscheduler.schedulers.tornado import TornadoScheduler
  32. from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
  33. import tornado.httpserver
  34. import tornado.ioloop
  35. import tornado.options
  36. from tornado.options import define, options
  37. import tornado.web
  38. from tornado import gen, escape
  39. JOBS_STORE = 'sqlite:///conf/jobs.db'
  40. API_VERSION = '1.0'
  41. SCHEDULES_FILE = 'conf/schedules.json'
  42. DEFAULT_CONF = 'conf/diffido.conf'
  43. EMAIL_FROM = 'diffido@localhost'
  44. SMTP_SETTINGS = {}
  45. GIT_CMD = 'git'
  46. re_commit = re.compile(r'^(?P<id>[0-9a-f]{40}) (?P<message>.*)\n(?: .* '
  47. '(?P<insertions>\d+) insertion.* (?P<deletions>\d+) deletion.*$)?', re.M)
  48. re_insertion = re.compile(r'(\d+) insertion')
  49. re_deletion = re.compile(r'(\d+) deletion')
  50. logger = logging.getLogger()
  51. logger.setLevel(logging.INFO)
  52. def read_schedules():
  53. """Return the schedules configuration.
  54. :returns: dictionary from the JSON object in conf/schedules.json
  55. :rtype: dict"""
  56. if not os.path.isfile(SCHEDULES_FILE):
  57. return {'schedules': {}}
  58. try:
  59. with open(SCHEDULES_FILE, 'r') as fd:
  60. schedules = json.loads(fd.read())
  61. for id_ in schedules.get('schedules', {}).keys():
  62. schedule = schedules['schedules'][id_]
  63. try:
  64. schedule['last_history'] = get_last_history(id_)
  65. except:
  66. schedule['last_history'] = {}
  67. continue
  68. return schedules
  69. except Exception as e:
  70. logger.error('unable to read %s: %s' % (SCHEDULES_FILE, e))
  71. return {'schedules': {}}
  72. def write_schedules(schedules):
  73. """Write the schedules configuration.
  74. :param schedules: the schedules to save
  75. :type schedules: dict
  76. :returns: True in case of success
  77. :rtype: bool"""
  78. try:
  79. with open(SCHEDULES_FILE, 'w') as fd:
  80. fd.write(json.dumps(schedules, indent=2))
  81. except Exception as e:
  82. logger.error('unable to write %s: %s' % (SCHEDULES_FILE, e))
  83. return False
  84. return True
  85. def next_id(schedules):
  86. """Return the next available integer (as a string) in the list of schedules keys (do not fills holes)
  87. :param schedules: the schedules
  88. :type schedules: dict
  89. :returns: the ID of the next schedule
  90. :rtype: str"""
  91. ids = schedules.get('schedules', {}).keys()
  92. if not ids:
  93. return '1'
  94. return str(max([int(i) for i in ids]) + 1)
  95. def get_schedule(id_, add_id=True, add_history=False):
  96. """Return information about a single schedule
  97. :param id_: ID of the schedule
  98. :type id_: str
  99. :param add_id: if True, add the ID in the dictionary
  100. :type add_id: bool
  101. :returns: the schedule
  102. :rtype: dict"""
  103. try:
  104. schedules = read_schedules()
  105. except Exception:
  106. return {}
  107. data = schedules.get('schedules', {}).get(id_, {})
  108. if add_history and data:
  109. data['last_history'] = get_last_history(id_)
  110. if add_id:
  111. data['id'] = str(id_)
  112. return data
  113. def select_xpath(content, xpath):
  114. """Select a portion of a HTML document
  115. :param content: the content of the document
  116. :type content: str
  117. :param xpath: the XPath selector
  118. :type xpath: str
  119. :returns: the selected document
  120. :rtype: str"""
  121. tree = etree.HTML(content)
  122. elems = tree.xpath(xpath)
  123. if not elems:
  124. return content
  125. selected_content = []
  126. for elem in elems:
  127. selected_content.append(''.join([elem.text] +
  128. [ElementTree.tostring(e, method='html').decode('utf-8', 'replace')
  129. for e in elem.getchildren()]))
  130. content = ''.join(selected_content).strip()
  131. return content
  132. def run_job(id_=None, force=False, *args, **kwargs):
  133. """Run a job
  134. :param id_: ID of the schedule to run
  135. :type id_: str
  136. :param force: run even if disabled
  137. :type force: bool
  138. :param args: positional arguments
  139. :type args: tuple
  140. :param kwargs: named arguments
  141. :type kwargs: dict
  142. :returns: True in case of success
  143. :rtype: bool"""
  144. schedule = get_schedule(id_, add_id=False)
  145. url = schedule.get('url')
  146. if not url:
  147. return False
  148. logger.debug('running job id:%s title:%s url: %s' % (id_, schedule.get('title', ''), url))
  149. if not schedule.get('enabled') and not force:
  150. logger.info('not running job %s: disabled' % id_)
  151. return True
  152. req = requests.get(url, allow_redirects=True, timeout=(30.10, 240))
  153. content = req.text
  154. xpath = schedule.get('xpath')
  155. if xpath:
  156. try:
  157. content = select_xpath(content, xpath)
  158. except Exception as e:
  159. logger.warn('unable to extract XPath %s: %s' % (xpath, e))
  160. req_path = urllib.parse.urlparse(req.url).path
  161. base_name = os.path.basename(req_path) or 'index.html'
  162. def _commit(id_, filename, content, queue):
  163. os.chdir('storage/%s' % id_)
  164. current_lines = 0
  165. if os.path.isfile(filename):
  166. with open(filename, 'r') as fd:
  167. for line in fd:
  168. current_lines += 1
  169. with open(filename, 'w') as fd:
  170. fd.write(content)
  171. p = subprocess.Popen([GIT_CMD, 'add', filename])
  172. p.communicate()
  173. p = subprocess.Popen([GIT_CMD, 'commit', '-m', '%s' % datetime.datetime.utcnow(), '--allow-empty'],
  174. stdout=subprocess.PIPE)
  175. stdout, _ = p.communicate()
  176. stdout = stdout.decode('utf-8')
  177. insert = re_insertion.findall(stdout)
  178. if insert:
  179. insert = int(insert[0])
  180. else:
  181. insert = 0
  182. delete = re_deletion.findall(stdout)
  183. if delete:
  184. delete = int(delete[0])
  185. else:
  186. delete = 0
  187. queue.put({'insertions': insert, 'deletions': delete, 'previous_lines': current_lines,
  188. 'changes': max(insert, delete)})
  189. queue = multiprocessing.Queue()
  190. p = multiprocessing.Process(target=_commit, args=(id_, base_name, content, queue))
  191. p.start()
  192. res = queue.get()
  193. p.join()
  194. email = schedule.get('email')
  195. if not email:
  196. return True
  197. changes = res.get('changes')
  198. if not changes:
  199. return True
  200. min_change = schedule.get('minimum_change')
  201. previous_lines = res.get('previous_lines')
  202. if min_change and previous_lines:
  203. min_change = float(min_change)
  204. change_fraction = res.get('changes') / previous_lines
  205. if change_fraction < min_change:
  206. return True
  207. # send notification
  208. diff = get_diff(id_).get('diff')
  209. if not diff:
  210. return True
  211. send_email(to=email, subject='%s page changed' % schedule.get('title'),
  212. body='changes:\n\n%s' % diff)
  213. return True
  214. def safe_run_job(id_=None, *args, **kwargs):
  215. """Safely run a job, catching all the exceptions
  216. :param id_: ID of the schedule to run
  217. :type id_: str
  218. :param args: positional arguments
  219. :type args: tuple
  220. :param kwargs: named arguments
  221. :type kwargs: dict
  222. :returns: True in case of success
  223. :rtype: bool"""
  224. try:
  225. run_job(id_, *args, **kwargs)
  226. except Exception as e:
  227. send_email('error executing job %s: %s' % (id_, e))
  228. def send_email(to, subject='diffido', body='', from_=None):
  229. """Send an email
  230. :param to: destination address
  231. :type to: str
  232. :param subject: email subject
  233. :type subject: str
  234. :param body: body of the email
  235. :type body: str
  236. :param from_: sender address
  237. :type from_: str
  238. :returns: True in case of success
  239. :rtype: bool"""
  240. msg = MIMEText(body)
  241. msg['Subject'] = subject
  242. msg['From'] = from_ or EMAIL_FROM
  243. msg['To'] = to
  244. starttls = SMTP_SETTINGS.get('smtp-starttls')
  245. use_ssl = SMTP_SETTINGS.get('smtp-use-ssl')
  246. args = {}
  247. for key, value in SMTP_SETTINGS.items():
  248. if key in ('smtp-starttls', 'smtp-use-ssl'):
  249. continue
  250. if key in ('smtp-port'):
  251. value = int(value)
  252. key = key.replace('smtp-', '', 1).replace('-', '_')
  253. args[key] = value
  254. try:
  255. if use_ssl:
  256. with smtplib.SMTP_SSL(**args) as s:
  257. s.send_message(msg)
  258. else:
  259. tls_args = {}
  260. for key in ('ssl_keyfile', 'ssl_certfile', 'ssl_context'):
  261. if key in args:
  262. tls_args = args[key]
  263. del args[key]
  264. with smtplib.SMTP(**args) as s:
  265. if starttls:
  266. s.starttls(**tls_args)
  267. s.ehlo_or_helo_if_needed()
  268. s.send_message(msg)
  269. except Exception as e:
  270. logger.error('unable to send email to %s: %s' % (to, e))
  271. return False
  272. return True
  273. def get_history(id_, limit=None, add_info=False):
  274. """Read the history of a schedule
  275. :param id_: ID of the schedule
  276. :type id_: str
  277. :param limit: number of entries to fetch
  278. :type limit: int
  279. :param add_info: add information about the schedule itself
  280. :type add_info: int
  281. :returns: information about the schedule and its history
  282. :rtype: dict"""
  283. def _history(id_, limit, queue):
  284. try:
  285. os.chdir('storage/%s' % id_)
  286. except:
  287. return queue.put(b'')
  288. cmd = [GIT_CMD, 'log', '--pretty=oneline', '--shortstat']
  289. if limit is not None:
  290. cmd.append('-%s' % limit)
  291. p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
  292. stdout, _ = p.communicate()
  293. queue.put(stdout)
  294. queue = multiprocessing.Queue()
  295. p = multiprocessing.Process(target=_history, args=(id_, limit, queue))
  296. p.start()
  297. res = queue.get().decode('utf-8')
  298. p.join()
  299. history = []
  300. for match in re_commit.finditer(res):
  301. info = match.groupdict()
  302. info['insertions'] = int(info['insertions'] or 0)
  303. info['deletions'] = int(info['deletions'] or 0)
  304. info['changes'] = max(info['insertions'], info['deletions'])
  305. history.append(info)
  306. last_id = None
  307. if history and 'id' in history[0]:
  308. last_id = history[0]['id']
  309. for idx, item in enumerate(history):
  310. item['seq'] = idx + 1
  311. data = {'history': history, 'last_id': last_id}
  312. if add_info:
  313. data['schedule'] = get_schedule(id_)
  314. return data
  315. def get_last_history(id_):
  316. """Read the last history entry of a schedule
  317. :param id_: ID of the schedule
  318. :type id_: str
  319. :returns: information about the schedule and its history
  320. :rtype: dict"""
  321. history = get_history(id_, limit=1)
  322. hist = history.get('history') or [{}]
  323. return hist[0]
  324. def get_diff(id_, commit_id='HEAD', old_commit_id=None):
  325. """Return the diff between commits of a schedule
  326. :param id_: ID of the schedule
  327. :type id_: str
  328. :param commit_id: the most recent commit ID; HEAD by default
  329. :type commit_id: str
  330. :param old_commit_id: the older commit ID; if None, the previous commit is used
  331. :type old_commit_id: str
  332. :returns: information about the schedule and the diff between commits
  333. :rtype: dict"""
  334. def _history(id_, commit_id, old_commit_id, queue):
  335. os.chdir('storage/%s' % id_)
  336. p = subprocess.Popen([GIT_CMD, 'diff', old_commit_id or '%s~' % commit_id, commit_id],
  337. stdout=subprocess.PIPE)
  338. stdout, _ = p.communicate()
  339. queue.put(stdout)
  340. queue = multiprocessing.Queue()
  341. p = multiprocessing.Process(target=_history, args=(id_, commit_id, old_commit_id, queue))
  342. p.start()
  343. res = queue.get().decode('utf-8')
  344. p.join()
  345. schedule = get_schedule(id_)
  346. return {'diff': res, 'schedule': schedule}
  347. def scheduler_update(scheduler, id_):
  348. """Update a scheduler job, using information from the JSON object
  349. :param scheduler: the TornadoScheduler instance to modify
  350. :type scheduler: TornadoScheduler
  351. :param id_: ID of the schedule that must be updated
  352. :type id_: str
  353. :returns: True in case of success
  354. :rtype: bool"""
  355. schedule = get_schedule(id_, add_id=False)
  356. if not schedule:
  357. logger.warn('unable to update empty schedule %s' % id_)
  358. return False
  359. trigger = schedule.get('trigger')
  360. if trigger not in ('interval', 'cron'):
  361. logger.warn('unable to update empty schedule %s: trigger not in ("cron", "interval")' % id_)
  362. return False
  363. args = {}
  364. if trigger == 'interval':
  365. args['trigger'] = 'interval'
  366. for unit in 'weeks', 'days', 'hours', 'minutes', 'seconds':
  367. if 'interval_%s' % unit not in schedule:
  368. continue
  369. try:
  370. val = schedule['interval_%s' % unit]
  371. if not val:
  372. continue
  373. args[unit] = int(val)
  374. except Exception:
  375. logger.warn('invalid argument on schedule %s: %s parameter %s is not an integer' %
  376. (id_, 'interval_%s' % unit, schedule['interval_%s' % unit]))
  377. if len(args) == 1:
  378. logger.error('no valid interval specified, skipping schedule %s' % id_)
  379. return False
  380. elif trigger == 'cron':
  381. try:
  382. cron_trigger = CronTrigger.from_crontab(schedule['cron_crontab'])
  383. args['trigger'] = cron_trigger
  384. except Exception:
  385. logger.warn('invalid argument on schedule %s: cron_tab parameter %s is not a valid crontab' %
  386. (id_, schedule.get('cron_crontab')))
  387. return False
  388. git_create_repo(id_)
  389. try:
  390. scheduler.add_job(safe_run_job, id=id_, replace_existing=True, kwargs={'id_': id_}, **args)
  391. except Exception as e:
  392. logger.warn('unable to update job %s: %s' % (id_, e))
  393. return False
  394. return True
  395. def scheduler_delete(scheduler, id_):
  396. """Update a scheduler job, using information from the JSON object
  397. :param scheduler: the TornadoScheduler instance to modify
  398. :type scheduler: TornadoScheduler
  399. :param id_: ID of the schedule
  400. :type id_: str
  401. :returns: True in case of success
  402. :rtype: bool"""
  403. try:
  404. scheduler.remove_job(job_id=id_)
  405. except Exception as e:
  406. logger.warn('unable to delete job %s: %s' % (id_, e))
  407. return False
  408. return git_delete_repo(id_)
  409. def reset_from_schedules(scheduler):
  410. """"Reset all scheduler jobs, using information from the JSON object
  411. :param scheduler: the TornadoScheduler instance to modify
  412. :type scheduler: TornadoScheduler
  413. :returns: True in case of success
  414. :rtype: bool"""
  415. ret = False
  416. try:
  417. scheduler.remove_all_jobs()
  418. for key in read_schedules().get('schedules', {}).keys():
  419. ret |= scheduler_update(scheduler, id_=key)
  420. except Exception as e:
  421. logger.warn('unable to reset all jobs: %s' % e)
  422. return False
  423. return ret
  424. def git_init():
  425. """Initialize Git global settings"""
  426. p = subprocess.Popen([GIT_CMD, 'config', '--global', 'user.email', '"%s"' % EMAIL_FROM])
  427. p.communicate()
  428. p = subprocess.Popen([GIT_CMD, 'config', '--global', 'user.name', '"Diffido"'])
  429. p.communicate()
  430. def git_create_repo(id_):
  431. """Create a Git repository
  432. :param id_: ID of the schedule
  433. :type id_: str
  434. :returns: True in case of success
  435. :rtype: bool"""
  436. repo_dir = 'storage/%s' % id_
  437. if os.path.isdir(repo_dir):
  438. return True
  439. p = subprocess.Popen([GIT_CMD, 'init', repo_dir])
  440. p.communicate()
  441. return p.returncode == 0
  442. def git_delete_repo(id_):
  443. """Delete a Git repository
  444. :param id_: ID of the schedule
  445. :type id_: str
  446. :returns: True in case of success
  447. :rtype: bool"""
  448. repo_dir = 'storage/%s' % id_
  449. if not os.path.isdir(repo_dir):
  450. return False
  451. try:
  452. shutil.rmtree(repo_dir)
  453. except Exception as e:
  454. logger.warn('unable to delete Git repository %s: %s' % (id_, e))
  455. return False
  456. return True
  457. class DiffidoBaseException(Exception):
  458. """Base class for diffido custom exceptions.
  459. :param message: text message
  460. :type message: str
  461. :param status: numeric http status code
  462. :type status: int"""
  463. def __init__(self, message, status=400):
  464. super(DiffidoBaseException, self).__init__(message)
  465. self.message = message
  466. self.status = status
  467. class BaseHandler(tornado.web.RequestHandler):
  468. """Base class for request handlers."""
  469. # A property to access the first value of each argument.
  470. arguments = property(lambda self: dict([(k, v[0].decode('utf-8'))
  471. for k, v in self.request.arguments.items()]))
  472. @property
  473. def clean_body(self):
  474. """Return a clean dictionary from a JSON body, suitable for a query on MongoDB.
  475. :returns: a clean copy of the body arguments
  476. :rtype: dict"""
  477. return escape.json_decode(self.request.body or '{}')
  478. def write_error(self, status_code, **kwargs):
  479. """Default error handler."""
  480. if isinstance(kwargs.get('exc_info', (None, None))[1], DiffidoBaseException):
  481. exc = kwargs['exc_info'][1]
  482. status_code = exc.status
  483. message = exc.message
  484. else:
  485. message = 'internal error'
  486. self.build_error(message, status=status_code)
  487. def initialize(self, **kwargs):
  488. """Add every passed (key, value) as attributes of the instance."""
  489. for key, value in kwargs.items():
  490. setattr(self, key, value)
  491. def build_error(self, message='', status=400):
  492. """Build and write an error message.
  493. :param message: textual message
  494. :type message: str
  495. :param status: HTTP status code
  496. :type status: int
  497. """
  498. self.set_status(status)
  499. self.write({'error': True, 'message': message})
  500. def build_success(self, message='', status=200):
  501. """Build and write a success message.
  502. :param message: textual message
  503. :type message: str
  504. :param status: HTTP status code
  505. :type status: int
  506. """
  507. self.set_status(status)
  508. self.write({'error': False, 'message': message})
  509. class SchedulesHandler(BaseHandler):
  510. """Schedules handler."""
  511. @gen.coroutine
  512. def get(self, id_=None, *args, **kwargs):
  513. """Get a schedule."""
  514. if id_ is not None:
  515. return self.write({'schedule': get_schedule(id_, add_history=True)})
  516. schedules = read_schedules()
  517. self.write(schedules)
  518. @gen.coroutine
  519. def put(self, id_=None, *args, **kwargs):
  520. """Update a schedule."""
  521. if id_ is None:
  522. return self.build_error(message='update action requires an ID')
  523. data = self.clean_body
  524. schedules = read_schedules()
  525. if id_ not in schedules.get('schedules', {}):
  526. return self.build_error(message='schedule %s not found' % id_)
  527. schedules['schedules'][id_] = data
  528. write_schedules(schedules)
  529. scheduler_update(scheduler=self.scheduler, id_=id_)
  530. self.write(get_schedule(id_=id_))
  531. @gen.coroutine
  532. def post(self, *args, **kwargs):
  533. """Add a schedule."""
  534. data = self.clean_body
  535. schedules = read_schedules()
  536. id_ = next_id(schedules)
  537. schedules['schedules'][id_] = data
  538. write_schedules(schedules)
  539. scheduler_update(scheduler=self.scheduler, id_=id_)
  540. self.write(get_schedule(id_=id_))
  541. @gen.coroutine
  542. def delete(self, id_=None, *args, **kwargs):
  543. """Delete a schedule."""
  544. if id_ is None:
  545. return self.build_error(message='an ID must be specified')
  546. schedules = read_schedules()
  547. if id_ in schedules.get('schedules', {}):
  548. del schedules['schedules'][id_]
  549. write_schedules(schedules)
  550. scheduler_delete(scheduler=self.scheduler, id_=id_)
  551. self.build_success(message='removed schedule %s' % id_)
  552. class RunScheduleHandler(BaseHandler):
  553. """Reset schedules handler."""
  554. @gen.coroutine
  555. def post(self, id_, *args, **kwargs):
  556. if run_job(id_, force=True):
  557. return self.build_success('job run')
  558. self.build_error('job not run')
  559. class ResetSchedulesHandler(BaseHandler):
  560. """Reset schedules handler."""
  561. @gen.coroutine
  562. def post(self, *args, **kwargs):
  563. reset_from_schedules(self.scheduler)
  564. class HistoryHandler(BaseHandler):
  565. """History handler."""
  566. @gen.coroutine
  567. def get(self, id_, *args, **kwargs):
  568. self.write(get_history(id_, add_info=True))
  569. class DiffHandler(BaseHandler):
  570. """Diff handler."""
  571. @gen.coroutine
  572. def get(self, id_, commit_id, old_commit_id=None, *args, **kwargs):
  573. self.write(get_diff(id_, commit_id, old_commit_id))
  574. class TemplateHandler(BaseHandler):
  575. """Handler for the template files in the / path."""
  576. @gen.coroutine
  577. def get(self, *args, **kwargs):
  578. """Get a template file."""
  579. page = 'index.html'
  580. if args and args[0]:
  581. page = args[0].strip('/')
  582. arguments = self.arguments
  583. self.render(page, **arguments)
  584. def serve():
  585. """Read configuration and start the server."""
  586. global EMAIL_FROM, SMTP_SETTINGS
  587. jobstores = {'default': SQLAlchemyJobStore(url=JOBS_STORE)}
  588. scheduler = TornadoScheduler(jobstores=jobstores, timezone=pytz.utc)
  589. scheduler.start()
  590. define('port', default=3210, help='run on the given port', type=int)
  591. define('address', default='', help='bind the server at the given address', type=str)
  592. define('ssl_cert', default=os.path.join(os.path.dirname(__file__), 'ssl', 'diffido_cert.pem'),
  593. help='specify the SSL certificate to use for secure connections')
  594. define('ssl_key', default=os.path.join(os.path.dirname(__file__), 'ssl', 'diffido_key.pem'),
  595. help='specify the SSL private key to use for secure connections')
  596. define('admin-email', default='', help='email address of the site administrator', type=str)
  597. define('smtp-host', default='localhost', help='SMTP server address', type=str)
  598. define('smtp-port', default=0, help='SMTP server port', type=int)
  599. define('smtp-local-hostname', default=None, help='SMTP local hostname', type=str)
  600. define('smtp-use-ssl', default=False, help='Use SSL to connect to the SMTP server', type=bool)
  601. define('smtp-starttls', default=False, help='Use STARTTLS to connect to the SMTP server', type=bool)
  602. define('smtp-ssl-keyfile', default=None, help='SSL key file', type=str)
  603. define('smtp-ssl-certfile', default=None, help='SSL cert file', type=str)
  604. define('smtp-ssl-context', default=None, help='SSL context', type=str)
  605. define('debug', default=False, help='run in debug mode', type=bool)
  606. define('config', help='read configuration file',
  607. callback=lambda path: tornado.options.parse_config_file(path, final=False))
  608. if not options.config and os.path.isfile(DEFAULT_CONF):
  609. tornado.options.parse_config_file(DEFAULT_CONF, final=False)
  610. tornado.options.parse_command_line()
  611. if options.admin_email:
  612. EMAIL_FROM = options.admin_email
  613. for key, value in options.as_dict().items():
  614. if key.startswith('smtp-'):
  615. SMTP_SETTINGS[key] = value
  616. if options.debug:
  617. logger.setLevel(logging.DEBUG)
  618. ssl_options = {}
  619. if os.path.isfile(options.ssl_key) and os.path.isfile(options.ssl_cert):
  620. ssl_options = dict(certfile=options.ssl_cert, keyfile=options.ssl_key)
  621. init_params = dict(listen_port=options.port, logger=logger, ssl_options=ssl_options,
  622. scheduler=scheduler)
  623. git_init()
  624. _reset_schedules_path = r'schedules/reset'
  625. _schedule_run_path = r'schedules/(?P<id_>\d+)/run'
  626. _schedules_path = r'schedules/?(?P<id_>\d+)?'
  627. _history_path = r'schedules/?(?P<id_>\d+)/history'
  628. _diff_path = r'schedules/(?P<id_>\d+)/diff/(?P<commit_id>[0-9a-f]+)/?(?P<old_commit_id>[0-9a-f]+)?/?'
  629. application = tornado.web.Application([
  630. (r'/api/%s' % _reset_schedules_path, ResetSchedulesHandler, init_params),
  631. (r'/api/v%s/%s' % (API_VERSION, _reset_schedules_path), ResetSchedulesHandler, init_params),
  632. (r'/api/%s' % _schedule_run_path, RunScheduleHandler, init_params),
  633. (r'/api/v%s/%s' % (API_VERSION, _schedule_run_path), RunScheduleHandler, init_params),
  634. (r'/api/%s' % _history_path, HistoryHandler, init_params),
  635. (r'/api/v%s/%s' % (API_VERSION, _history_path), HistoryHandler, init_params),
  636. (r'/api/%s' % _diff_path, DiffHandler, init_params),
  637. (r'/api/v%s/%s' % (API_VERSION, _diff_path), DiffHandler, init_params),
  638. (r'/api/%s' % _schedules_path, SchedulesHandler, init_params),
  639. (r'/api/v%s/%s' % (API_VERSION, _schedules_path), SchedulesHandler, init_params),
  640. (r'/?(.*)', TemplateHandler, init_params),
  641. ],
  642. static_path=os.path.join(os.path.dirname(__file__), 'dist/static'),
  643. template_path=os.path.join(os.path.dirname(__file__), 'dist/'),
  644. debug=options.debug)
  645. http_server = tornado.httpserver.HTTPServer(application, ssl_options=ssl_options or None)
  646. logger.info('Start serving on %s://%s:%d', 'https' if ssl_options else 'http',
  647. options.address if options.address else '127.0.0.1',
  648. options.port)
  649. http_server.listen(options.port, options.address)
  650. try:
  651. IOLoop.instance().start()
  652. except (KeyboardInterrupt, SystemExit):
  653. pass
  654. if __name__ == '__main__':
  655. serve()