diffido.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """Diffido - because the F5 key is a terrible thing to waste.
  4. Copyright 2018 Davide Alberani <da@erlug.linux.it>
  5. Licensed under the Apache License, Version 2.0 (the "License");
  6. you may not use this file except in compliance with the License.
  7. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
  8. Unless required by applicable law or agreed to in writing, software
  9. distributed under the License is distributed on an "AS IS" BASIS,
  10. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11. See the License for the specific language governing permissions and
  12. limitations under the License.
  13. """
  14. import os
  15. import re
  16. import json
  17. import pytz
  18. import shutil
  19. import urllib
  20. import smtplib
  21. from email.mime.text import MIMEText
  22. import logging
  23. import datetime
  24. import requests
  25. import subprocess
  26. import multiprocessing
  27. from lxml import etree
  28. from xml.etree import ElementTree
  29. from tornado.ioloop import IOLoop
  30. from apscheduler.triggers.cron import CronTrigger
  31. from apscheduler.schedulers.tornado import TornadoScheduler
  32. from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
  33. import tornado.httpserver
  34. import tornado.ioloop
  35. import tornado.options
  36. from tornado.options import define, options
  37. import tornado.web
  38. from tornado import gen, escape
  39. JOBS_STORE = 'sqlite:///conf/jobs.db'
  40. API_VERSION = '1.0'
  41. SCHEDULES_FILE = 'conf/schedules.json'
  42. DEFAULT_CONF = 'conf/diffido.conf'
  43. EMAIL_FROM = 'diffido@localhost'
  44. SMTP_SETTINGS = {}
  45. GIT_CMD = 'git'
  46. re_commit = re.compile(r'^(?P<id>[0-9a-f]{40}) (?P<message>.*)\n(?: .* '
  47. '(?P<insertions>\d+) insertion.* (?P<deletions>\d+) deletion.*$)?', re.M)
  48. re_insertion = re.compile(r'(\d+) insertion')
  49. re_deletion = re.compile(r'(\d+) deletion')
  50. logger = logging.getLogger()
  51. logger.setLevel(logging.INFO)
  52. def read_schedules():
  53. """Return the schedules configuration.
  54. :returns: dictionary from the JSON object in conf/schedules.json
  55. :rtype: dict"""
  56. if not os.path.isfile(SCHEDULES_FILE):
  57. return {'schedules': {}}
  58. try:
  59. with open(SCHEDULES_FILE, 'r') as fd:
  60. schedules = json.loads(fd.read())
  61. for id_ in schedules.get('schedules', {}).keys():
  62. schedule = schedules['schedules'][id_]
  63. try:
  64. schedule['last_history'] = get_last_history(id_)
  65. except:
  66. schedule['last_history'] = {}
  67. continue
  68. return schedules
  69. except Exception as e:
  70. logger.error('unable to read %s: %s' % (SCHEDULES_FILE, e))
  71. return {'schedules': {}}
  72. def write_schedules(schedules):
  73. """Write the schedules configuration.
  74. :param schedules: the schedules to save
  75. :type schedules: dict
  76. :returns: True in case of success
  77. :rtype: bool"""
  78. try:
  79. with open(SCHEDULES_FILE, 'w') as fd:
  80. fd.write(json.dumps(schedules, indent=2))
  81. except Exception as e:
  82. logger.error('unable to write %s: %s' % (SCHEDULES_FILE, e))
  83. return False
  84. return True
  85. def next_id(schedules):
  86. """Return the next available integer (as a string) in the list of schedules keys (do not fills holes)
  87. :param schedules: the schedules
  88. :type schedules: dict
  89. :returns: the ID of the next schedule
  90. :rtype: str"""
  91. ids = schedules.get('schedules', {}).keys()
  92. if not ids:
  93. return '1'
  94. return str(max([int(i) for i in ids]) + 1)
  95. def get_schedule(id_, add_id=True, add_history=False):
  96. """Return information about a single schedule
  97. :param id_: ID of the schedule
  98. :type id_: str
  99. :param add_id: if True, add the ID in the dictionary
  100. :type add_id: bool
  101. :returns: the schedule
  102. :rtype: dict"""
  103. try:
  104. schedules = read_schedules()
  105. except Exception:
  106. return {}
  107. data = schedules.get('schedules', {}).get(id_, {})
  108. if add_history and data:
  109. data['last_history'] = get_last_history(id_)
  110. if add_id:
  111. data['id'] = str(id_)
  112. return data
  113. def select_xpath(content, xpath):
  114. """Select a portion of a HTML document
  115. :param content: the content of the document
  116. :type content: str
  117. :param xpath: the XPath selector
  118. :type xpath: str
  119. :returns: the selected document
  120. :rtype: str"""
  121. tree = etree.HTML(content)
  122. elems = tree.xpath(xpath)
  123. if not elems:
  124. return content
  125. selected_content = []
  126. for elem in elems:
  127. pieces = []
  128. if elem.text:
  129. pieces.append(elem.text)
  130. for sub_el in elem.getchildren():
  131. try:
  132. sub_el_text = ElementTree.tostring(sub_el, method='html').decode('utf-8', 'replace')
  133. except:
  134. continue
  135. if sub_el_text:
  136. pieces.append(sub_el_text)
  137. selected_content.append(''.join(pieces))
  138. content = ''.join(selected_content).strip()
  139. return content
  140. def run_job(id_=None, force=False, *args, **kwargs):
  141. """Run a job
  142. :param id_: ID of the schedule to run
  143. :type id_: str
  144. :param force: run even if disabled
  145. :type force: bool
  146. :param args: positional arguments
  147. :type args: tuple
  148. :param kwargs: named arguments
  149. :type kwargs: dict
  150. :returns: True in case of success
  151. :rtype: bool"""
  152. schedule = get_schedule(id_, add_id=False)
  153. url = schedule.get('url')
  154. if not url:
  155. return False
  156. logger.debug('running job id:%s title:%s url: %s' % (id_, schedule.get('title', ''), url))
  157. if not schedule.get('enabled') and not force:
  158. logger.info('not running job %s: disabled' % id_)
  159. return True
  160. req = requests.get(url, allow_redirects=True, timeout=(30.10, 240))
  161. content = req.text
  162. xpath = schedule.get('xpath')
  163. if xpath:
  164. try:
  165. content = select_xpath(content, xpath)
  166. except Exception as e:
  167. logger.warn('unable to extract XPath %s: %s' % (xpath, e))
  168. req_path = urllib.parse.urlparse(req.url).path
  169. base_name = os.path.basename(req_path) or 'index.html'
  170. def _commit(id_, filename, content, queue):
  171. os.chdir('storage/%s' % id_)
  172. current_lines = 0
  173. if os.path.isfile(filename):
  174. with open(filename, 'r') as fd:
  175. for line in fd:
  176. current_lines += 1
  177. with open(filename, 'w') as fd:
  178. fd.write(content)
  179. p = subprocess.Popen([GIT_CMD, 'add', filename])
  180. p.communicate()
  181. p = subprocess.Popen([GIT_CMD, 'commit', '-m', '%s' % datetime.datetime.utcnow(), '--allow-empty'],
  182. stdout=subprocess.PIPE)
  183. stdout, _ = p.communicate()
  184. stdout = stdout.decode('utf-8')
  185. insert = re_insertion.findall(stdout)
  186. if insert:
  187. insert = int(insert[0])
  188. else:
  189. insert = 0
  190. delete = re_deletion.findall(stdout)
  191. if delete:
  192. delete = int(delete[0])
  193. else:
  194. delete = 0
  195. queue.put({'insertions': insert, 'deletions': delete, 'previous_lines': current_lines,
  196. 'changes': max(insert, delete)})
  197. queue = multiprocessing.Queue()
  198. p = multiprocessing.Process(target=_commit, args=(id_, base_name, content, queue))
  199. p.start()
  200. res = queue.get()
  201. p.join()
  202. email = schedule.get('email')
  203. if not email:
  204. return True
  205. changes = res.get('changes')
  206. if not changes:
  207. return True
  208. min_change = schedule.get('minimum_change')
  209. previous_lines = res.get('previous_lines')
  210. if min_change and previous_lines:
  211. min_change = float(min_change)
  212. change_fraction = res.get('changes') / previous_lines
  213. if change_fraction < min_change:
  214. return True
  215. # send notification
  216. diff = get_diff(id_).get('diff')
  217. if not diff:
  218. return True
  219. send_email(to=email, subject='%s page changed' % schedule.get('title'),
  220. body='changes:\n\n%s' % diff)
  221. return True
  222. def safe_run_job(id_=None, *args, **kwargs):
  223. """Safely run a job, catching all the exceptions
  224. :param id_: ID of the schedule to run
  225. :type id_: str
  226. :param args: positional arguments
  227. :type args: tuple
  228. :param kwargs: named arguments
  229. :type kwargs: dict
  230. :returns: True in case of success
  231. :rtype: bool"""
  232. try:
  233. run_job(id_, *args, **kwargs)
  234. except Exception as e:
  235. send_email('error executing job %s: %s' % (id_, e))
  236. def send_email(to, subject='diffido', body='', from_=None):
  237. """Send an email
  238. :param to: destination address
  239. :type to: str
  240. :param subject: email subject
  241. :type subject: str
  242. :param body: body of the email
  243. :type body: str
  244. :param from_: sender address
  245. :type from_: str
  246. :returns: True in case of success
  247. :rtype: bool"""
  248. msg = MIMEText(body)
  249. msg['Subject'] = subject
  250. msg['From'] = from_ or EMAIL_FROM
  251. msg['To'] = to
  252. starttls = SMTP_SETTINGS.get('smtp-starttls')
  253. use_ssl = SMTP_SETTINGS.get('smtp-use-ssl')
  254. args = {}
  255. for key, value in SMTP_SETTINGS.items():
  256. if key in ('smtp-starttls', 'smtp-use-ssl'):
  257. continue
  258. if key in ('smtp-port'):
  259. value = int(value)
  260. key = key.replace('smtp-', '', 1).replace('-', '_')
  261. args[key] = value
  262. try:
  263. if use_ssl:
  264. with smtplib.SMTP_SSL(**args) as s:
  265. s.send_message(msg)
  266. else:
  267. tls_args = {}
  268. for key in ('ssl_keyfile', 'ssl_certfile', 'ssl_context'):
  269. if key in args:
  270. tls_args = args[key]
  271. del args[key]
  272. with smtplib.SMTP(**args) as s:
  273. if starttls:
  274. s.starttls(**tls_args)
  275. s.ehlo_or_helo_if_needed()
  276. s.send_message(msg)
  277. except Exception as e:
  278. logger.error('unable to send email to %s: %s' % (to, e))
  279. return False
  280. return True
  281. def get_history(id_, limit=None, add_info=False):
  282. """Read the history of a schedule
  283. :param id_: ID of the schedule
  284. :type id_: str
  285. :param limit: number of entries to fetch
  286. :type limit: int
  287. :param add_info: add information about the schedule itself
  288. :type add_info: int
  289. :returns: information about the schedule and its history
  290. :rtype: dict"""
  291. def _history(id_, limit, queue):
  292. try:
  293. os.chdir('storage/%s' % id_)
  294. except:
  295. return queue.put(b'')
  296. cmd = [GIT_CMD, 'log', '--pretty=oneline', '--shortstat']
  297. if limit is not None:
  298. cmd.append('-%s' % limit)
  299. p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
  300. stdout, _ = p.communicate()
  301. queue.put(stdout)
  302. queue = multiprocessing.Queue()
  303. p = multiprocessing.Process(target=_history, args=(id_, limit, queue))
  304. p.start()
  305. res = queue.get().decode('utf-8')
  306. p.join()
  307. history = []
  308. for match in re_commit.finditer(res):
  309. info = match.groupdict()
  310. info['insertions'] = int(info['insertions'] or 0)
  311. info['deletions'] = int(info['deletions'] or 0)
  312. info['changes'] = max(info['insertions'], info['deletions'])
  313. history.append(info)
  314. last_id = None
  315. if history and 'id' in history[0]:
  316. last_id = history[0]['id']
  317. for idx, item in enumerate(history):
  318. item['seq'] = idx + 1
  319. data = {'history': history, 'last_id': last_id}
  320. if add_info:
  321. data['schedule'] = get_schedule(id_)
  322. return data
  323. def get_last_history(id_):
  324. """Read the last history entry of a schedule
  325. :param id_: ID of the schedule
  326. :type id_: str
  327. :returns: information about the schedule and its history
  328. :rtype: dict"""
  329. history = get_history(id_, limit=1)
  330. hist = history.get('history') or [{}]
  331. return hist[0]
  332. def get_diff(id_, commit_id='HEAD', old_commit_id=None):
  333. """Return the diff between commits of a schedule
  334. :param id_: ID of the schedule
  335. :type id_: str
  336. :param commit_id: the most recent commit ID; HEAD by default
  337. :type commit_id: str
  338. :param old_commit_id: the older commit ID; if None, the previous commit is used
  339. :type old_commit_id: str
  340. :returns: information about the schedule and the diff between commits
  341. :rtype: dict"""
  342. def _history(id_, commit_id, old_commit_id, queue):
  343. os.chdir('storage/%s' % id_)
  344. p = subprocess.Popen([GIT_CMD, 'diff', old_commit_id or '%s~' % commit_id, commit_id],
  345. stdout=subprocess.PIPE)
  346. stdout, _ = p.communicate()
  347. queue.put(stdout)
  348. queue = multiprocessing.Queue()
  349. p = multiprocessing.Process(target=_history, args=(id_, commit_id, old_commit_id, queue))
  350. p.start()
  351. res = queue.get().decode('utf-8')
  352. p.join()
  353. schedule = get_schedule(id_)
  354. return {'diff': res, 'schedule': schedule}
  355. def scheduler_update(scheduler, id_):
  356. """Update a scheduler job, using information from the JSON object
  357. :param scheduler: the TornadoScheduler instance to modify
  358. :type scheduler: TornadoScheduler
  359. :param id_: ID of the schedule that must be updated
  360. :type id_: str
  361. :returns: True in case of success
  362. :rtype: bool"""
  363. schedule = get_schedule(id_, add_id=False)
  364. if not schedule:
  365. logger.warn('unable to update empty schedule %s' % id_)
  366. return False
  367. trigger = schedule.get('trigger')
  368. if trigger not in ('interval', 'cron'):
  369. logger.warn('unable to update empty schedule %s: trigger not in ("cron", "interval")' % id_)
  370. return False
  371. args = {}
  372. if trigger == 'interval':
  373. args['trigger'] = 'interval'
  374. for unit in 'weeks', 'days', 'hours', 'minutes', 'seconds':
  375. if 'interval_%s' % unit not in schedule:
  376. continue
  377. try:
  378. val = schedule['interval_%s' % unit]
  379. if not val:
  380. continue
  381. args[unit] = int(val)
  382. except Exception:
  383. logger.warn('invalid argument on schedule %s: %s parameter %s is not an integer' %
  384. (id_, 'interval_%s' % unit, schedule['interval_%s' % unit]))
  385. if len(args) == 1:
  386. logger.error('no valid interval specified, skipping schedule %s' % id_)
  387. return False
  388. elif trigger == 'cron':
  389. try:
  390. cron_trigger = CronTrigger.from_crontab(schedule['cron_crontab'])
  391. args['trigger'] = cron_trigger
  392. except Exception:
  393. logger.warn('invalid argument on schedule %s: cron_tab parameter %s is not a valid crontab' %
  394. (id_, schedule.get('cron_crontab')))
  395. return False
  396. git_create_repo(id_)
  397. try:
  398. scheduler.add_job(safe_run_job, id=id_, replace_existing=True, kwargs={'id_': id_}, **args)
  399. except Exception as e:
  400. logger.warn('unable to update job %s: %s' % (id_, e))
  401. return False
  402. return True
  403. def scheduler_delete(scheduler, id_):
  404. """Update a scheduler job, using information from the JSON object
  405. :param scheduler: the TornadoScheduler instance to modify
  406. :type scheduler: TornadoScheduler
  407. :param id_: ID of the schedule
  408. :type id_: str
  409. :returns: True in case of success
  410. :rtype: bool"""
  411. try:
  412. scheduler.remove_job(job_id=id_)
  413. except Exception as e:
  414. logger.warn('unable to delete job %s: %s' % (id_, e))
  415. return False
  416. return git_delete_repo(id_)
  417. def reset_from_schedules(scheduler):
  418. """"Reset all scheduler jobs, using information from the JSON object
  419. :param scheduler: the TornadoScheduler instance to modify
  420. :type scheduler: TornadoScheduler
  421. :returns: True in case of success
  422. :rtype: bool"""
  423. ret = False
  424. try:
  425. scheduler.remove_all_jobs()
  426. for key in read_schedules().get('schedules', {}).keys():
  427. ret |= scheduler_update(scheduler, id_=key)
  428. except Exception as e:
  429. logger.warn('unable to reset all jobs: %s' % e)
  430. return False
  431. return ret
  432. def git_init():
  433. """Initialize Git global settings"""
  434. p = subprocess.Popen([GIT_CMD, 'config', '--global', 'user.email', '"%s"' % EMAIL_FROM])
  435. p.communicate()
  436. p = subprocess.Popen([GIT_CMD, 'config', '--global', 'user.name', '"Diffido"'])
  437. p.communicate()
  438. def git_create_repo(id_):
  439. """Create a Git repository
  440. :param id_: ID of the schedule
  441. :type id_: str
  442. :returns: True in case of success
  443. :rtype: bool"""
  444. repo_dir = 'storage/%s' % id_
  445. if os.path.isdir(repo_dir):
  446. return True
  447. p = subprocess.Popen([GIT_CMD, 'init', repo_dir])
  448. p.communicate()
  449. return p.returncode == 0
  450. def git_delete_repo(id_):
  451. """Delete a Git repository
  452. :param id_: ID of the schedule
  453. :type id_: str
  454. :returns: True in case of success
  455. :rtype: bool"""
  456. repo_dir = 'storage/%s' % id_
  457. if not os.path.isdir(repo_dir):
  458. return False
  459. try:
  460. shutil.rmtree(repo_dir)
  461. except Exception as e:
  462. logger.warn('unable to delete Git repository %s: %s' % (id_, e))
  463. return False
  464. return True
  465. class DiffidoBaseException(Exception):
  466. """Base class for diffido custom exceptions.
  467. :param message: text message
  468. :type message: str
  469. :param status: numeric http status code
  470. :type status: int"""
  471. def __init__(self, message, status=400):
  472. super(DiffidoBaseException, self).__init__(message)
  473. self.message = message
  474. self.status = status
  475. class BaseHandler(tornado.web.RequestHandler):
  476. """Base class for request handlers."""
  477. # A property to access the first value of each argument.
  478. arguments = property(lambda self: dict([(k, v[0].decode('utf-8'))
  479. for k, v in self.request.arguments.items()]))
  480. @property
  481. def clean_body(self):
  482. """Return a clean dictionary from a JSON body, suitable for a query on MongoDB.
  483. :returns: a clean copy of the body arguments
  484. :rtype: dict"""
  485. return escape.json_decode(self.request.body or '{}')
  486. def write_error(self, status_code, **kwargs):
  487. """Default error handler."""
  488. if isinstance(kwargs.get('exc_info', (None, None))[1], DiffidoBaseException):
  489. exc = kwargs['exc_info'][1]
  490. status_code = exc.status
  491. message = exc.message
  492. else:
  493. message = 'internal error'
  494. self.build_error(message, status=status_code)
  495. def initialize(self, **kwargs):
  496. """Add every passed (key, value) as attributes of the instance."""
  497. for key, value in kwargs.items():
  498. setattr(self, key, value)
  499. def build_error(self, message='', status=400):
  500. """Build and write an error message.
  501. :param message: textual message
  502. :type message: str
  503. :param status: HTTP status code
  504. :type status: int
  505. """
  506. self.set_status(status)
  507. self.write({'error': True, 'message': message})
  508. def build_success(self, message='', status=200):
  509. """Build and write a success message.
  510. :param message: textual message
  511. :type message: str
  512. :param status: HTTP status code
  513. :type status: int
  514. """
  515. self.set_status(status)
  516. self.write({'error': False, 'message': message})
  517. class SchedulesHandler(BaseHandler):
  518. """Schedules handler."""
  519. @gen.coroutine
  520. def get(self, id_=None, *args, **kwargs):
  521. """Get a schedule."""
  522. if id_ is not None:
  523. return self.write({'schedule': get_schedule(id_, add_history=True)})
  524. schedules = read_schedules()
  525. self.write(schedules)
  526. @gen.coroutine
  527. def put(self, id_=None, *args, **kwargs):
  528. """Update a schedule."""
  529. if id_ is None:
  530. return self.build_error(message='update action requires an ID')
  531. data = self.clean_body
  532. schedules = read_schedules()
  533. if id_ not in schedules.get('schedules', {}):
  534. return self.build_error(message='schedule %s not found' % id_)
  535. schedules['schedules'][id_] = data
  536. write_schedules(schedules)
  537. scheduler_update(scheduler=self.scheduler, id_=id_)
  538. self.write(get_schedule(id_=id_))
  539. @gen.coroutine
  540. def post(self, *args, **kwargs):
  541. """Add a schedule."""
  542. data = self.clean_body
  543. schedules = read_schedules()
  544. id_ = next_id(schedules)
  545. schedules['schedules'][id_] = data
  546. write_schedules(schedules)
  547. scheduler_update(scheduler=self.scheduler, id_=id_)
  548. self.write(get_schedule(id_=id_))
  549. @gen.coroutine
  550. def delete(self, id_=None, *args, **kwargs):
  551. """Delete a schedule."""
  552. if id_ is None:
  553. return self.build_error(message='an ID must be specified')
  554. schedules = read_schedules()
  555. if id_ in schedules.get('schedules', {}):
  556. del schedules['schedules'][id_]
  557. write_schedules(schedules)
  558. scheduler_delete(scheduler=self.scheduler, id_=id_)
  559. self.build_success(message='removed schedule %s' % id_)
  560. class RunScheduleHandler(BaseHandler):
  561. """Reset schedules handler."""
  562. @gen.coroutine
  563. def post(self, id_, *args, **kwargs):
  564. if run_job(id_, force=True):
  565. return self.build_success('job run')
  566. self.build_error('job not run')
  567. class ResetSchedulesHandler(BaseHandler):
  568. """Reset schedules handler."""
  569. @gen.coroutine
  570. def post(self, *args, **kwargs):
  571. reset_from_schedules(self.scheduler)
  572. class HistoryHandler(BaseHandler):
  573. """History handler."""
  574. @gen.coroutine
  575. def get(self, id_, *args, **kwargs):
  576. self.write(get_history(id_, add_info=True))
  577. class DiffHandler(BaseHandler):
  578. """Diff handler."""
  579. @gen.coroutine
  580. def get(self, id_, commit_id, old_commit_id=None, *args, **kwargs):
  581. self.write(get_diff(id_, commit_id, old_commit_id))
  582. class TemplateHandler(BaseHandler):
  583. """Handler for the template files in the / path."""
  584. @gen.coroutine
  585. def get(self, *args, **kwargs):
  586. """Get a template file."""
  587. page = 'index.html'
  588. if args and args[0]:
  589. page = args[0].strip('/')
  590. arguments = self.arguments
  591. self.render(page, **arguments)
  592. def serve():
  593. """Read configuration and start the server."""
  594. global EMAIL_FROM, SMTP_SETTINGS
  595. jobstores = {'default': SQLAlchemyJobStore(url=JOBS_STORE)}
  596. scheduler = TornadoScheduler(jobstores=jobstores, timezone=pytz.utc)
  597. scheduler.start()
  598. define('port', default=3210, help='run on the given port', type=int)
  599. define('address', default='', help='bind the server at the given address', type=str)
  600. define('ssl_cert', default=os.path.join(os.path.dirname(__file__), 'ssl', 'diffido_cert.pem'),
  601. help='specify the SSL certificate to use for secure connections')
  602. define('ssl_key', default=os.path.join(os.path.dirname(__file__), 'ssl', 'diffido_key.pem'),
  603. help='specify the SSL private key to use for secure connections')
  604. define('admin-email', default='', help='email address of the site administrator', type=str)
  605. define('smtp-host', default='localhost', help='SMTP server address', type=str)
  606. define('smtp-port', default=0, help='SMTP server port', type=int)
  607. define('smtp-local-hostname', default=None, help='SMTP local hostname', type=str)
  608. define('smtp-use-ssl', default=False, help='Use SSL to connect to the SMTP server', type=bool)
  609. define('smtp-starttls', default=False, help='Use STARTTLS to connect to the SMTP server', type=bool)
  610. define('smtp-ssl-keyfile', default=None, help='SSL key file', type=str)
  611. define('smtp-ssl-certfile', default=None, help='SSL cert file', type=str)
  612. define('smtp-ssl-context', default=None, help='SSL context', type=str)
  613. define('debug', default=False, help='run in debug mode', type=bool)
  614. define('config', help='read configuration file',
  615. callback=lambda path: tornado.options.parse_config_file(path, final=False))
  616. if not options.config and os.path.isfile(DEFAULT_CONF):
  617. tornado.options.parse_config_file(DEFAULT_CONF, final=False)
  618. tornado.options.parse_command_line()
  619. if options.admin_email:
  620. EMAIL_FROM = options.admin_email
  621. for key, value in options.as_dict().items():
  622. if key.startswith('smtp-'):
  623. SMTP_SETTINGS[key] = value
  624. if options.debug:
  625. logger.setLevel(logging.DEBUG)
  626. ssl_options = {}
  627. if os.path.isfile(options.ssl_key) and os.path.isfile(options.ssl_cert):
  628. ssl_options = dict(certfile=options.ssl_cert, keyfile=options.ssl_key)
  629. init_params = dict(listen_port=options.port, logger=logger, ssl_options=ssl_options,
  630. scheduler=scheduler)
  631. git_init()
  632. _reset_schedules_path = r'schedules/reset'
  633. _schedule_run_path = r'schedules/(?P<id_>\d+)/run'
  634. _schedules_path = r'schedules/?(?P<id_>\d+)?'
  635. _history_path = r'schedules/?(?P<id_>\d+)/history'
  636. _diff_path = r'schedules/(?P<id_>\d+)/diff/(?P<commit_id>[0-9a-f]+)/?(?P<old_commit_id>[0-9a-f]+)?/?'
  637. application = tornado.web.Application([
  638. (r'/api/%s' % _reset_schedules_path, ResetSchedulesHandler, init_params),
  639. (r'/api/v%s/%s' % (API_VERSION, _reset_schedules_path), ResetSchedulesHandler, init_params),
  640. (r'/api/%s' % _schedule_run_path, RunScheduleHandler, init_params),
  641. (r'/api/v%s/%s' % (API_VERSION, _schedule_run_path), RunScheduleHandler, init_params),
  642. (r'/api/%s' % _history_path, HistoryHandler, init_params),
  643. (r'/api/v%s/%s' % (API_VERSION, _history_path), HistoryHandler, init_params),
  644. (r'/api/%s' % _diff_path, DiffHandler, init_params),
  645. (r'/api/v%s/%s' % (API_VERSION, _diff_path), DiffHandler, init_params),
  646. (r'/api/%s' % _schedules_path, SchedulesHandler, init_params),
  647. (r'/api/v%s/%s' % (API_VERSION, _schedules_path), SchedulesHandler, init_params),
  648. (r'/?(.*)', TemplateHandler, init_params),
  649. ],
  650. static_path=os.path.join(os.path.dirname(__file__), 'dist/static'),
  651. template_path=os.path.join(os.path.dirname(__file__), 'dist/'),
  652. debug=options.debug)
  653. http_server = tornado.httpserver.HTTPServer(application, ssl_options=ssl_options or None)
  654. logger.info('Start serving on %s://%s:%d', 'https' if ssl_options else 'http',
  655. options.address if options.address else '127.0.0.1',
  656. options.port)
  657. http_server.listen(options.port, options.address)
  658. try:
  659. IOLoop.instance().start()
  660. except (KeyboardInterrupt, SystemExit):
  661. pass
  662. if __name__ == '__main__':
  663. serve()