diffido.py 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """Diffido - because the F5 key is a terrible thing to waste.
  4. Copyright 2018 Davide Alberani <da@erlug.linux.it>
  5. Licensed under the Apache License, Version 2.0 (the "License");
  6. you may not use this file except in compliance with the License.
  7. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
  8. Unless required by applicable law or agreed to in writing, software
  9. distributed under the License is distributed on an "AS IS" BASIS,
  10. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11. See the License for the specific language governing permissions and
  12. limitations under the License.
  13. """
  14. import os
  15. import re
  16. import json
  17. import pytz
  18. import shutil
  19. import urllib
  20. import smtplib
  21. from email.mime.text import MIMEText
  22. import logging
  23. import datetime
  24. import requests
  25. import subprocess
  26. import multiprocessing
  27. from lxml import etree
  28. from xml.etree import ElementTree
  29. from tornado.ioloop import IOLoop
  30. from apscheduler.triggers.cron import CronTrigger
  31. from apscheduler.schedulers.tornado import TornadoScheduler
  32. from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
  33. import tornado.httpserver
  34. import tornado.ioloop
  35. import tornado.options
  36. from tornado.options import define, options
  37. import tornado.web
  38. from tornado import gen, escape
  39. JOBS_STORE = 'sqlite:///conf/jobs.db'
  40. API_VERSION = '1.0'
  41. SCHEDULES_FILE = 'conf/schedules.json'
  42. DEFAULT_CONF = 'conf/diffido.conf'
  43. EMAIL_FROM = 'diffido@localhost'
  44. SMTP_SETTINGS = {}
  45. GIT_CMD = 'git'
  46. re_commit = re.compile(r'^(?P<id>[0-9a-f]{40}) (?P<message>.*)\n(?: .* '
  47. '(?P<insertions>\d+) insertion.* (?P<deletions>\d+) deletion.*$)?', re.M)
  48. re_insertion = re.compile(r'(\d+) insertion')
  49. re_deletion = re.compile(r'(\d+) deletion')
  50. logger = logging.getLogger()
  51. logger.setLevel(logging.INFO)
  52. def read_schedules():
  53. """Return the schedules configuration.
  54. :returns: dictionary from the JSON object in conf/schedules.json
  55. :rtype: dict"""
  56. if not os.path.isfile(SCHEDULES_FILE):
  57. return {'schedules': {}}
  58. try:
  59. with open(SCHEDULES_FILE, 'r') as fd:
  60. schedules = json.loads(fd.read())
  61. for id_ in schedules.get('schedules', {}).keys():
  62. schedule = schedules['schedules'][id_]
  63. try:
  64. schedule['last_history'] = get_last_history(id_)
  65. except:
  66. schedule['last_history'] = {}
  67. continue
  68. return schedules
  69. except Exception as e:
  70. logger.error('unable to read %s: %s' % (SCHEDULES_FILE, e))
  71. return {'schedules': {}}
  72. def write_schedules(schedules):
  73. """Write the schedules configuration.
  74. :param schedules: the schedules to save
  75. :type schedules: dict
  76. :returns: True in case of success
  77. :rtype: bool"""
  78. try:
  79. with open(SCHEDULES_FILE, 'w') as fd:
  80. fd.write(json.dumps(schedules, indent=2))
  81. except Exception as e:
  82. logger.error('unable to write %s: %s' % (SCHEDULES_FILE, e))
  83. return False
  84. return True
  85. def next_id(schedules):
  86. """Return the next available integer (as a string) in the list of schedules keys (do not fills holes)
  87. :param schedules: the schedules
  88. :type schedules: dict
  89. :returns: the ID of the next schedule
  90. :rtype: str"""
  91. ids = schedules.get('schedules', {}).keys()
  92. if not ids:
  93. return '1'
  94. return str(max([int(i) for i in ids]) + 1)
  95. def get_schedule(id_, add_id=True, add_history=False):
  96. """Return information about a single schedule
  97. :param id_: ID of the schedule
  98. :type id_: str
  99. :param add_id: if True, add the ID in the dictionary
  100. :type add_id: bool
  101. :returns: the schedule
  102. :rtype: dict"""
  103. try:
  104. schedules = read_schedules()
  105. except Exception:
  106. return {}
  107. data = schedules.get('schedules', {}).get(id_, {})
  108. if add_history and data:
  109. data['last_history'] = get_last_history(id_)
  110. if add_id:
  111. data['id'] = str(id_)
  112. return data
  113. def select_xpath(content, xpath):
  114. """Select a portion of a HTML document
  115. :param content: the content of the document
  116. :type content: str
  117. :param xpath: the XPath selector
  118. :type xpath: str
  119. :returns: the selected document
  120. :rtype: str"""
  121. tree = etree.HTML(content)
  122. elems = tree.xpath(xpath)
  123. if not elems:
  124. return content
  125. selected_content = []
  126. for elem in elems:
  127. pieces = []
  128. if elem.text:
  129. pieces.append(elem.text)
  130. for sub_el in elem.getchildren():
  131. try:
  132. sub_el_text = ElementTree.tostring(sub_el, method='html').decode('utf-8', 'replace')
  133. except:
  134. continue
  135. if sub_el_text:
  136. pieces.append(sub_el_text)
  137. selected_content.append(''.join(pieces))
  138. content = ''.join(selected_content).strip()
  139. return content
  140. def run_job(id_=None, force=False, *args, **kwargs):
  141. """Run a job
  142. :param id_: ID of the schedule to run
  143. :type id_: str
  144. :param force: run even if disabled
  145. :type force: bool
  146. :param args: positional arguments
  147. :type args: tuple
  148. :param kwargs: named arguments
  149. :type kwargs: dict
  150. :returns: True in case of success
  151. :rtype: bool"""
  152. schedule = get_schedule(id_, add_id=False)
  153. url = schedule.get('url')
  154. if not url:
  155. return False
  156. logger.debug('running job id:%s title:%s url: %s' % (id_, schedule.get('title', ''), url))
  157. if not schedule.get('enabled') and not force:
  158. logger.info('not running job %s: disabled' % id_)
  159. return True
  160. req = requests.get(url, allow_redirects=True, timeout=(30.10, 240))
  161. content = req.text
  162. xpath = schedule.get('xpath')
  163. if xpath:
  164. try:
  165. content = select_xpath(content, xpath)
  166. except Exception as e:
  167. logger.warn('unable to extract XPath %s: %s' % (xpath, e))
  168. req_path = urllib.parse.urlparse(req.url).path
  169. base_name = os.path.basename(req_path) or 'index.html'
  170. def _commit(id_, filename, content, queue):
  171. try:
  172. os.chdir('storage/%s' % id_)
  173. except Exception as e:
  174. logger.info('unable to move to storage/%s directory: %s; trying to create it...' % (id_, e))
  175. _created = False
  176. try:
  177. _created = git_create_repo(id_)
  178. except Exception as e:
  179. logger.info('unable to move to storage/%s directory: %s; unable to create it' % (id_, e))
  180. if not _created:
  181. return False
  182. current_lines = 0
  183. if os.path.isfile(filename):
  184. with open(filename, 'r') as fd:
  185. for line in fd:
  186. current_lines += 1
  187. with open(filename, 'w') as fd:
  188. fd.write(content)
  189. p = subprocess.Popen([GIT_CMD, 'add', filename])
  190. p.communicate()
  191. p = subprocess.Popen([GIT_CMD, 'commit', '-m', '%s' % datetime.datetime.utcnow(), '--allow-empty'],
  192. stdout=subprocess.PIPE)
  193. stdout, _ = p.communicate()
  194. stdout = stdout.decode('utf-8')
  195. insert = re_insertion.findall(stdout)
  196. if insert:
  197. insert = int(insert[0])
  198. else:
  199. insert = 0
  200. delete = re_deletion.findall(stdout)
  201. if delete:
  202. delete = int(delete[0])
  203. else:
  204. delete = 0
  205. queue.put({'insertions': insert, 'deletions': delete, 'previous_lines': current_lines,
  206. 'changes': max(insert, delete)})
  207. queue = multiprocessing.Queue()
  208. p = multiprocessing.Process(target=_commit, args=(id_, base_name, content, queue))
  209. p.start()
  210. res = queue.get()
  211. p.join()
  212. email = schedule.get('email')
  213. if not email:
  214. return True
  215. changes = res.get('changes')
  216. if not changes:
  217. return True
  218. min_change = schedule.get('minimum_change')
  219. previous_lines = res.get('previous_lines')
  220. if min_change and previous_lines:
  221. min_change = float(min_change)
  222. change_fraction = res.get('changes') / previous_lines
  223. if change_fraction < min_change:
  224. return True
  225. # send notification
  226. diff = get_diff(id_).get('diff')
  227. if not diff:
  228. return True
  229. send_email(to=email, subject='%s page changed' % schedule.get('title'),
  230. body='changes:\n\n%s' % diff)
  231. return True
  232. def safe_run_job(id_=None, *args, **kwargs):
  233. """Safely run a job, catching all the exceptions
  234. :param id_: ID of the schedule to run
  235. :type id_: str
  236. :param args: positional arguments
  237. :type args: tuple
  238. :param kwargs: named arguments
  239. :type kwargs: dict
  240. :returns: True in case of success
  241. :rtype: bool"""
  242. try:
  243. run_job(id_, *args, **kwargs)
  244. except Exception as e:
  245. send_email('error executing job %s: %s' % (id_, e))
  246. def send_email(to, subject='diffido', body='', from_=None):
  247. """Send an email
  248. :param to: destination address
  249. :type to: str
  250. :param subject: email subject
  251. :type subject: str
  252. :param body: body of the email
  253. :type body: str
  254. :param from_: sender address
  255. :type from_: str
  256. :returns: True in case of success
  257. :rtype: bool"""
  258. msg = MIMEText(body)
  259. msg['Subject'] = subject
  260. msg['From'] = from_ or EMAIL_FROM
  261. msg['To'] = to
  262. starttls = SMTP_SETTINGS.get('smtp-starttls')
  263. use_ssl = SMTP_SETTINGS.get('smtp-use-ssl')
  264. args = {}
  265. for key, value in SMTP_SETTINGS.items():
  266. if key in ('smtp-starttls', 'smtp-use-ssl'):
  267. continue
  268. if key in ('smtp-port'):
  269. value = int(value)
  270. key = key.replace('smtp-', '', 1).replace('-', '_')
  271. args[key] = value
  272. try:
  273. if use_ssl:
  274. with smtplib.SMTP_SSL(**args) as s:
  275. if args.get('username'):
  276. s.login(args.get('username', ''), args.get('password', ''))
  277. s.send_message(msg)
  278. else:
  279. tls_args = {}
  280. for key in ('ssl_keyfile', 'ssl_certfile', 'ssl_context'):
  281. if key in args:
  282. tls_args[key] = args[key].replace('ssl_', '')
  283. del args[key]
  284. with smtplib.SMTP(**args) as s:
  285. if starttls:
  286. s.starttls(**tls_args)
  287. s.ehlo_or_helo_if_needed()
  288. if args.get('username'):
  289. s.login(args.get('username', ''), args.get('password', ''))
  290. s.send_message(msg)
  291. except Exception as e:
  292. logger.error('unable to send email to %s: %s' % (to, e))
  293. return False
  294. return True
  295. def get_history(id_, limit=None, add_info=False):
  296. """Read the history of a schedule
  297. :param id_: ID of the schedule
  298. :type id_: str
  299. :param limit: number of entries to fetch
  300. :type limit: int
  301. :param add_info: add information about the schedule itself
  302. :type add_info: int
  303. :returns: information about the schedule and its history
  304. :rtype: dict"""
  305. def _history(id_, limit, queue):
  306. try:
  307. os.chdir('storage/%s' % id_)
  308. except Exception as e:
  309. logger.info('unable to move to storage/%s directory: %s' % (id_, e))
  310. return queue.put(b'')
  311. cmd = [GIT_CMD, 'log', '--pretty=oneline', '--shortstat']
  312. if limit is not None:
  313. cmd.append('-%s' % limit)
  314. p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
  315. stdout, _ = p.communicate()
  316. queue.put(stdout)
  317. queue = multiprocessing.Queue()
  318. p = multiprocessing.Process(target=_history, args=(id_, limit, queue))
  319. p.start()
  320. res = queue.get().decode('utf-8')
  321. p.join()
  322. history = []
  323. for match in re_commit.finditer(res):
  324. info = match.groupdict()
  325. info['insertions'] = int(info['insertions'] or 0)
  326. info['deletions'] = int(info['deletions'] or 0)
  327. info['changes'] = max(info['insertions'], info['deletions'])
  328. history.append(info)
  329. last_id = None
  330. if history and 'id' in history[0]:
  331. last_id = history[0]['id']
  332. for idx, item in enumerate(history):
  333. item['seq'] = idx + 1
  334. data = {'history': history, 'last_id': last_id}
  335. if add_info:
  336. data['schedule'] = get_schedule(id_)
  337. return data
  338. def get_last_history(id_):
  339. """Read the last history entry of a schedule
  340. :param id_: ID of the schedule
  341. :type id_: str
  342. :returns: information about the schedule and its history
  343. :rtype: dict"""
  344. history = get_history(id_, limit=1)
  345. hist = history.get('history') or [{}]
  346. return hist[0]
  347. def get_diff(id_, commit_id='HEAD', old_commit_id=None):
  348. """Return the diff between commits of a schedule
  349. :param id_: ID of the schedule
  350. :type id_: str
  351. :param commit_id: the most recent commit ID; HEAD by default
  352. :type commit_id: str
  353. :param old_commit_id: the older commit ID; if None, the previous commit is used
  354. :type old_commit_id: str
  355. :returns: information about the schedule and the diff between commits
  356. :rtype: dict"""
  357. def _history(id_, commit_id, old_commit_id, queue):
  358. try:
  359. os.chdir('storage/%s' % id_)
  360. except Exception as e:
  361. logger.info('unable to move to storage/%s directory: %s' % (id_, e))
  362. return queue.put(b'')
  363. p = subprocess.Popen([GIT_CMD, 'diff', old_commit_id or '%s~' % commit_id, commit_id],
  364. stdout=subprocess.PIPE)
  365. stdout, _ = p.communicate()
  366. queue.put(stdout)
  367. queue = multiprocessing.Queue()
  368. p = multiprocessing.Process(target=_history, args=(id_, commit_id, old_commit_id, queue))
  369. p.start()
  370. res = queue.get().decode('utf-8')
  371. p.join()
  372. schedule = get_schedule(id_)
  373. return {'diff': res, 'schedule': schedule}
  374. def scheduler_update(scheduler, id_):
  375. """Update a scheduler job, using information from the JSON object
  376. :param scheduler: the TornadoScheduler instance to modify
  377. :type scheduler: TornadoScheduler
  378. :param id_: ID of the schedule that must be updated
  379. :type id_: str
  380. :returns: True in case of success
  381. :rtype: bool"""
  382. schedule = get_schedule(id_, add_id=False)
  383. if not schedule:
  384. logger.warn('unable to update empty schedule %s' % id_)
  385. return False
  386. trigger = schedule.get('trigger')
  387. if trigger not in ('interval', 'cron'):
  388. logger.warn('unable to update empty schedule %s: trigger not in ("cron", "interval")' % id_)
  389. return False
  390. args = {}
  391. if trigger == 'interval':
  392. args['trigger'] = 'interval'
  393. for unit in 'weeks', 'days', 'hours', 'minutes', 'seconds':
  394. if 'interval_%s' % unit not in schedule:
  395. continue
  396. try:
  397. val = schedule['interval_%s' % unit]
  398. if not val:
  399. continue
  400. args[unit] = int(val)
  401. except Exception:
  402. logger.warn('invalid argument on schedule %s: %s parameter %s is not an integer' %
  403. (id_, 'interval_%s' % unit, schedule['interval_%s' % unit]))
  404. if len(args) == 1:
  405. logger.error('no valid interval specified, skipping schedule %s' % id_)
  406. return False
  407. elif trigger == 'cron':
  408. try:
  409. cron_trigger = CronTrigger.from_crontab(schedule['cron_crontab'])
  410. args['trigger'] = cron_trigger
  411. except Exception:
  412. logger.warn('invalid argument on schedule %s: cron_tab parameter %s is not a valid crontab' %
  413. (id_, schedule.get('cron_crontab')))
  414. return False
  415. git_create_repo(id_)
  416. try:
  417. scheduler.add_job(safe_run_job, id=id_, replace_existing=True, kwargs={'id_': id_}, **args)
  418. except Exception as e:
  419. logger.warn('unable to update job %s: %s' % (id_, e))
  420. return False
  421. return True
  422. def scheduler_delete(scheduler, id_):
  423. """Update a scheduler job, using information from the JSON object
  424. :param scheduler: the TornadoScheduler instance to modify
  425. :type scheduler: TornadoScheduler
  426. :param id_: ID of the schedule
  427. :type id_: str
  428. :returns: True in case of success
  429. :rtype: bool"""
  430. try:
  431. scheduler.remove_job(job_id=id_)
  432. except Exception as e:
  433. logger.warn('unable to delete job %s: %s' % (id_, e))
  434. return False
  435. return git_delete_repo(id_)
  436. def reset_from_schedules(scheduler):
  437. """"Reset all scheduler jobs, using information from the JSON object
  438. :param scheduler: the TornadoScheduler instance to modify
  439. :type scheduler: TornadoScheduler
  440. :returns: True in case of success
  441. :rtype: bool"""
  442. ret = False
  443. try:
  444. scheduler.remove_all_jobs()
  445. for key in read_schedules().get('schedules', {}).keys():
  446. ret |= scheduler_update(scheduler, id_=key)
  447. except Exception as e:
  448. logger.warn('unable to reset all jobs: %s' % e)
  449. return False
  450. return ret
  451. def git_init():
  452. """Initialize Git global settings"""
  453. p = subprocess.Popen([GIT_CMD, 'config', '--global', 'user.email', '"%s"' % EMAIL_FROM])
  454. p.communicate()
  455. p = subprocess.Popen([GIT_CMD, 'config', '--global', 'user.name', '"Diffido"'])
  456. p.communicate()
  457. def git_create_repo(id_):
  458. """Create a Git repository
  459. :param id_: ID of the schedule
  460. :type id_: str
  461. :returns: True in case of success
  462. :rtype: bool"""
  463. repo_dir = 'storage/%s' % id_
  464. if os.path.isdir(repo_dir):
  465. return True
  466. p = subprocess.Popen([GIT_CMD, 'init', repo_dir])
  467. p.communicate()
  468. return p.returncode == 0
  469. def git_delete_repo(id_):
  470. """Delete a Git repository
  471. :param id_: ID of the schedule
  472. :type id_: str
  473. :returns: True in case of success
  474. :rtype: bool"""
  475. repo_dir = 'storage/%s' % id_
  476. if not os.path.isdir(repo_dir):
  477. return False
  478. try:
  479. shutil.rmtree(repo_dir)
  480. except Exception as e:
  481. logger.warn('unable to delete Git repository %s: %s' % (id_, e))
  482. return False
  483. return True
  484. class DiffidoBaseException(Exception):
  485. """Base class for diffido custom exceptions.
  486. :param message: text message
  487. :type message: str
  488. :param status: numeric http status code
  489. :type status: int"""
  490. def __init__(self, message, status=400):
  491. super(DiffidoBaseException, self).__init__(message)
  492. self.message = message
  493. self.status = status
  494. class BaseHandler(tornado.web.RequestHandler):
  495. """Base class for request handlers."""
  496. # A property to access the first value of each argument.
  497. arguments = property(lambda self: dict([(k, v[0].decode('utf-8'))
  498. for k, v in self.request.arguments.items()]))
  499. @property
  500. def clean_body(self):
  501. """Return a clean dictionary from a JSON body, suitable for a query on MongoDB.
  502. :returns: a clean copy of the body arguments
  503. :rtype: dict"""
  504. return escape.json_decode(self.request.body or '{}')
  505. def write_error(self, status_code, **kwargs):
  506. """Default error handler."""
  507. if isinstance(kwargs.get('exc_info', (None, None))[1], DiffidoBaseException):
  508. exc = kwargs['exc_info'][1]
  509. status_code = exc.status
  510. message = exc.message
  511. else:
  512. message = 'internal error'
  513. self.build_error(message, status=status_code)
  514. def initialize(self, **kwargs):
  515. """Add every passed (key, value) as attributes of the instance."""
  516. for key, value in kwargs.items():
  517. setattr(self, key, value)
  518. def build_error(self, message='', status=400):
  519. """Build and write an error message.
  520. :param message: textual message
  521. :type message: str
  522. :param status: HTTP status code
  523. :type status: int
  524. """
  525. self.set_status(status)
  526. self.write({'error': True, 'message': message})
  527. def build_success(self, message='', status=200):
  528. """Build and write a success message.
  529. :param message: textual message
  530. :type message: str
  531. :param status: HTTP status code
  532. :type status: int
  533. """
  534. self.set_status(status)
  535. self.write({'error': False, 'message': message})
  536. class SchedulesHandler(BaseHandler):
  537. """Schedules handler."""
  538. @gen.coroutine
  539. def get(self, id_=None, *args, **kwargs):
  540. """Get a schedule."""
  541. if id_ is not None:
  542. return self.write({'schedule': get_schedule(id_, add_history=True)})
  543. schedules = read_schedules()
  544. self.write(schedules)
  545. @gen.coroutine
  546. def put(self, id_=None, *args, **kwargs):
  547. """Update a schedule."""
  548. if id_ is None:
  549. return self.build_error(message='update action requires an ID')
  550. data = self.clean_body
  551. schedules = read_schedules()
  552. if id_ not in schedules.get('schedules', {}):
  553. return self.build_error(message='schedule %s not found' % id_)
  554. schedules['schedules'][id_] = data
  555. write_schedules(schedules)
  556. scheduler_update(scheduler=self.scheduler, id_=id_)
  557. self.write(get_schedule(id_=id_))
  558. @gen.coroutine
  559. def post(self, *args, **kwargs):
  560. """Add a schedule."""
  561. data = self.clean_body
  562. schedules = read_schedules()
  563. id_ = next_id(schedules)
  564. schedules['schedules'][id_] = data
  565. write_schedules(schedules)
  566. scheduler_update(scheduler=self.scheduler, id_=id_)
  567. self.write(get_schedule(id_=id_))
  568. @gen.coroutine
  569. def delete(self, id_=None, *args, **kwargs):
  570. """Delete a schedule."""
  571. if id_ is None:
  572. return self.build_error(message='an ID must be specified')
  573. schedules = read_schedules()
  574. if id_ in schedules.get('schedules', {}):
  575. del schedules['schedules'][id_]
  576. write_schedules(schedules)
  577. scheduler_delete(scheduler=self.scheduler, id_=id_)
  578. self.build_success(message='removed schedule %s' % id_)
  579. class RunScheduleHandler(BaseHandler):
  580. """Reset schedules handler."""
  581. @gen.coroutine
  582. def post(self, id_, *args, **kwargs):
  583. if run_job(id_, force=True):
  584. return self.build_success('job run')
  585. self.build_error('job not run')
  586. class ResetSchedulesHandler(BaseHandler):
  587. """Reset schedules handler."""
  588. @gen.coroutine
  589. def post(self, *args, **kwargs):
  590. reset_from_schedules(self.scheduler)
  591. class HistoryHandler(BaseHandler):
  592. """History handler."""
  593. @gen.coroutine
  594. def get(self, id_, *args, **kwargs):
  595. self.write(get_history(id_, add_info=True))
  596. class DiffHandler(BaseHandler):
  597. """Diff handler."""
  598. @gen.coroutine
  599. def get(self, id_, commit_id, old_commit_id=None, *args, **kwargs):
  600. self.write(get_diff(id_, commit_id, old_commit_id))
  601. class TemplateHandler(BaseHandler):
  602. """Handler for the template files in the / path."""
  603. @gen.coroutine
  604. def get(self, *args, **kwargs):
  605. """Get a template file."""
  606. page = 'index.html'
  607. if args and args[0]:
  608. page = args[0].strip('/')
  609. arguments = self.arguments
  610. self.render(page, **arguments)
  611. def serve():
  612. """Read configuration and start the server."""
  613. global EMAIL_FROM, SMTP_SETTINGS
  614. jobstores = {'default': SQLAlchemyJobStore(url=JOBS_STORE)}
  615. scheduler = TornadoScheduler(jobstores=jobstores, timezone=pytz.utc)
  616. scheduler.start()
  617. define('port', default=3210, help='run on the given port', type=int)
  618. define('address', default='', help='bind the server at the given address', type=str)
  619. define('ssl_cert', default=os.path.join(os.path.dirname(__file__), 'ssl', 'diffido_cert.pem'),
  620. help='specify the SSL certificate to use for secure connections')
  621. define('ssl_key', default=os.path.join(os.path.dirname(__file__), 'ssl', 'diffido_key.pem'),
  622. help='specify the SSL private key to use for secure connections')
  623. define('admin-email', default='', help='email address of the site administrator', type=str)
  624. define('smtp-host', default='localhost', help='SMTP server address', type=str)
  625. define('smtp-port', default=0, help='SMTP server port', type=int)
  626. define('smtp-local-hostname', default=None, help='SMTP local hostname', type=str)
  627. define('smtp-use-ssl', default=False, help='Use SSL to connect to the SMTP server', type=bool)
  628. define('smtp-starttls', default=False, help='Use STARTTLS to connect to the SMTP server', type=bool)
  629. define('smtp-ssl-keyfile', default=None, help='SMTP SSL key file', type=str)
  630. define('smtp-ssl-certfile', default=None, help='SMTP SSL cert file', type=str)
  631. define('smtp-ssl-context', default=None, help='SMTP SSL context', type=str)
  632. define('smtp-username', default=None, help='SMTP username', type=str)
  633. define('smtp-password', default=None, help='SMTP password', type=str)
  634. define('debug', default=False, help='run in debug mode', type=bool)
  635. define('config', help='read configuration file',
  636. callback=lambda path: tornado.options.parse_config_file(path, final=False))
  637. if not options.config and os.path.isfile(DEFAULT_CONF):
  638. tornado.options.parse_config_file(DEFAULT_CONF, final=False)
  639. tornado.options.parse_command_line()
  640. if options.admin_email:
  641. EMAIL_FROM = options.admin_email
  642. for key, value in options.as_dict().items():
  643. if key.startswith('smtp-'):
  644. SMTP_SETTINGS[key] = value
  645. if options.debug:
  646. logger.setLevel(logging.DEBUG)
  647. ssl_options = {}
  648. if os.path.isfile(options.ssl_key) and os.path.isfile(options.ssl_cert):
  649. ssl_options = dict(certfile=options.ssl_cert, keyfile=options.ssl_key)
  650. init_params = dict(listen_port=options.port, logger=logger, ssl_options=ssl_options,
  651. scheduler=scheduler)
  652. git_init()
  653. _reset_schedules_path = r'schedules/reset'
  654. _schedule_run_path = r'schedules/(?P<id_>\d+)/run'
  655. _schedules_path = r'schedules/?(?P<id_>\d+)?'
  656. _history_path = r'schedules/?(?P<id_>\d+)/history'
  657. _diff_path = r'schedules/(?P<id_>\d+)/diff/(?P<commit_id>[0-9a-f]+)/?(?P<old_commit_id>[0-9a-f]+)?/?'
  658. application = tornado.web.Application([
  659. (r'/api/%s' % _reset_schedules_path, ResetSchedulesHandler, init_params),
  660. (r'/api/v%s/%s' % (API_VERSION, _reset_schedules_path), ResetSchedulesHandler, init_params),
  661. (r'/api/%s' % _schedule_run_path, RunScheduleHandler, init_params),
  662. (r'/api/v%s/%s' % (API_VERSION, _schedule_run_path), RunScheduleHandler, init_params),
  663. (r'/api/%s' % _history_path, HistoryHandler, init_params),
  664. (r'/api/v%s/%s' % (API_VERSION, _history_path), HistoryHandler, init_params),
  665. (r'/api/%s' % _diff_path, DiffHandler, init_params),
  666. (r'/api/v%s/%s' % (API_VERSION, _diff_path), DiffHandler, init_params),
  667. (r'/api/%s' % _schedules_path, SchedulesHandler, init_params),
  668. (r'/api/v%s/%s' % (API_VERSION, _schedules_path), SchedulesHandler, init_params),
  669. (r'/?(.*)', TemplateHandler, init_params),
  670. ],
  671. static_path=os.path.join(os.path.dirname(__file__), 'dist/static'),
  672. template_path=os.path.join(os.path.dirname(__file__), 'dist/'),
  673. debug=options.debug)
  674. http_server = tornado.httpserver.HTTPServer(application, ssl_options=ssl_options or None)
  675. logger.info('Start serving on %s://%s:%d', 'https' if ssl_options else 'http',
  676. options.address if options.address else '127.0.0.1',
  677. options.port)
  678. http_server.listen(options.port, options.address)
  679. try:
  680. IOLoop.instance().start()
  681. except (KeyboardInterrupt, SystemExit):
  682. pass
  683. if __name__ == '__main__':
  684. serve()