diffido.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """Diffido - because the F5 key is a terrible thing to waste.
  4. Copyright 2018 Davide Alberani <da@erlug.linux.it>
  5. Licensed under the Apache License, Version 2.0 (the "License");
  6. you may not use this file except in compliance with the License.
  7. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
  8. Unless required by applicable law or agreed to in writing, software
  9. distributed under the License is distributed on an "AS IS" BASIS,
  10. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11. See the License for the specific language governing permissions and
  12. limitations under the License.
  13. """
  14. import os
  15. import re
  16. import json
  17. import pytz
  18. import shutil
  19. import urllib
  20. import smtplib
  21. from email.mime.text import MIMEText
  22. from email.utils import formatdate
  23. import logging
  24. import datetime
  25. import requests
  26. import subprocess
  27. import multiprocessing
  28. from lxml import etree
  29. from xml.etree import ElementTree
  30. from tornado.ioloop import IOLoop
  31. from apscheduler.triggers.cron import CronTrigger
  32. from apscheduler.schedulers.tornado import TornadoScheduler
  33. from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
  34. import tornado.httpserver
  35. import tornado.ioloop
  36. import tornado.options
  37. from tornado.options import define, options
  38. import tornado.web
  39. from tornado import gen, escape
  40. JOBS_STORE = 'sqlite:///conf/jobs.db'
  41. API_VERSION = '1.0'
  42. SCHEDULES_FILE = 'conf/schedules.json'
  43. DEFAULT_CONF = 'conf/diffido.conf'
  44. EMAIL_FROM = 'diffido@localhost'
  45. SMTP_SETTINGS = {}
  46. GIT_CMD = 'git'
  47. re_commit = re.compile(r'^(?P<id>[0-9a-f]{40}) (?P<message>.*)\n(?: .* '
  48. '(?P<insertions>\d+) insertion.* (?P<deletions>\d+) deletion.*$)?', re.M)
  49. re_insertion = re.compile(r'(\d+) insertion')
  50. re_deletion = re.compile(r'(\d+) deletion')
  51. logger = logging.getLogger()
  52. logger.setLevel(logging.INFO)
  53. def read_schedules():
  54. """Return the schedules configuration.
  55. :returns: dictionary from the JSON object in conf/schedules.json
  56. :rtype: dict"""
  57. if not os.path.isfile(SCHEDULES_FILE):
  58. return {'schedules': {}}
  59. try:
  60. with open(SCHEDULES_FILE, 'r') as fd:
  61. schedules = json.loads(fd.read())
  62. for id_ in schedules.get('schedules', {}).keys():
  63. schedule = schedules['schedules'][id_]
  64. try:
  65. schedule['last_history'] = get_last_history(id_)
  66. except:
  67. schedule['last_history'] = {}
  68. continue
  69. return schedules
  70. except Exception as e:
  71. logger.error('unable to read %s: %s' % (SCHEDULES_FILE, e))
  72. return {'schedules': {}}
  73. def write_schedules(schedules):
  74. """Write the schedules configuration.
  75. :param schedules: the schedules to save
  76. :type schedules: dict
  77. :returns: True in case of success
  78. :rtype: bool"""
  79. try:
  80. with open(SCHEDULES_FILE, 'w') as fd:
  81. fd.write(json.dumps(schedules, indent=2))
  82. except Exception as e:
  83. logger.error('unable to write %s: %s' % (SCHEDULES_FILE, e))
  84. return False
  85. return True
  86. def next_id(schedules):
  87. """Return the next available integer (as a string) in the list of schedules keys (do not fills holes)
  88. :param schedules: the schedules
  89. :type schedules: dict
  90. :returns: the ID of the next schedule
  91. :rtype: str"""
  92. ids = schedules.get('schedules', {}).keys()
  93. if not ids:
  94. return '1'
  95. return str(max([int(i) for i in ids]) + 1)
  96. def get_schedule(id_, add_id=True, add_history=False):
  97. """Return information about a single schedule
  98. :param id_: ID of the schedule
  99. :type id_: str
  100. :param add_id: if True, add the ID in the dictionary
  101. :type add_id: bool
  102. :returns: the schedule
  103. :rtype: dict"""
  104. try:
  105. schedules = read_schedules()
  106. except Exception:
  107. return {}
  108. data = schedules.get('schedules', {}).get(id_, {})
  109. if add_history and data:
  110. data['last_history'] = get_last_history(id_)
  111. if add_id:
  112. data['id'] = str(id_)
  113. return data
  114. def select_xpath(content, xpath):
  115. """Select a portion of a HTML document
  116. :param content: the content of the document
  117. :type content: str
  118. :param xpath: the XPath selector
  119. :type xpath: str
  120. :returns: the selected document
  121. :rtype: str"""
  122. tree = etree.HTML(content)
  123. elems = tree.xpath(xpath)
  124. if not elems:
  125. return content
  126. selected_content = []
  127. for elem in elems:
  128. pieces = []
  129. if elem.text:
  130. pieces.append(elem.text)
  131. for sub_el in elem.getchildren():
  132. try:
  133. sub_el_text = ElementTree.tostring(sub_el, method='html').decode('utf-8', 'replace')
  134. except:
  135. continue
  136. if sub_el_text:
  137. pieces.append(sub_el_text)
  138. selected_content.append(''.join(pieces))
  139. content = ''.join(selected_content).strip()
  140. return content
  141. def run_job(id_=None, force=False, *args, **kwargs):
  142. """Run a job
  143. :param id_: ID of the schedule to run
  144. :type id_: str
  145. :param force: run even if disabled
  146. :type force: bool
  147. :param args: positional arguments
  148. :type args: tuple
  149. :param kwargs: named arguments
  150. :type kwargs: dict
  151. :returns: True in case of success
  152. :rtype: bool"""
  153. schedule = get_schedule(id_, add_id=False)
  154. url = schedule.get('url')
  155. if not url:
  156. return False
  157. logger.debug('running job id:%s title:%s url: %s' % (id_, schedule.get('title', ''), url))
  158. if not schedule.get('enabled') and not force:
  159. logger.info('not running job %s: disabled' % id_)
  160. return True
  161. req = requests.get(url, allow_redirects=True, timeout=(30.10, 240))
  162. content = req.text
  163. xpath = schedule.get('xpath')
  164. if xpath:
  165. try:
  166. content = select_xpath(content, xpath)
  167. except Exception as e:
  168. logger.warn('unable to extract XPath %s: %s' % (xpath, e))
  169. req_path = urllib.parse.urlparse(req.url).path
  170. base_name = os.path.basename(req_path) or 'index.html'
  171. def _commit(id_, filename, content, queue):
  172. try:
  173. os.chdir('storage/%s' % id_)
  174. except Exception as e:
  175. logger.info('unable to move to storage/%s directory: %s; trying to create it...' % (id_, e))
  176. _created = False
  177. try:
  178. _created = git_create_repo(id_)
  179. except Exception as e:
  180. logger.info('unable to move to storage/%s directory: %s; unable to create it' % (id_, e))
  181. if not _created:
  182. return False
  183. current_lines = 0
  184. if os.path.isfile(filename):
  185. with open(filename, 'r') as fd:
  186. for line in fd:
  187. current_lines += 1
  188. with open(filename, 'w') as fd:
  189. fd.write(content)
  190. p = subprocess.Popen([GIT_CMD, 'add', filename])
  191. p.communicate()
  192. p = subprocess.Popen([GIT_CMD, 'commit', '-m', '%s' % datetime.datetime.utcnow(), '--allow-empty'],
  193. stdout=subprocess.PIPE)
  194. stdout, _ = p.communicate()
  195. stdout = stdout.decode('utf-8')
  196. insert = re_insertion.findall(stdout)
  197. if insert:
  198. insert = int(insert[0])
  199. else:
  200. insert = 0
  201. delete = re_deletion.findall(stdout)
  202. if delete:
  203. delete = int(delete[0])
  204. else:
  205. delete = 0
  206. queue.put({'insertions': insert, 'deletions': delete, 'previous_lines': current_lines,
  207. 'changes': max(insert, delete)})
  208. queue = multiprocessing.Queue()
  209. p = multiprocessing.Process(target=_commit, args=(id_, base_name, content, queue))
  210. p.start()
  211. res = queue.get()
  212. p.join()
  213. email = schedule.get('email')
  214. if not email:
  215. return True
  216. changes = res.get('changes')
  217. if not changes:
  218. return True
  219. min_change = schedule.get('minimum_change')
  220. previous_lines = res.get('previous_lines')
  221. if min_change and previous_lines:
  222. min_change = float(min_change)
  223. change_fraction = res.get('changes') / previous_lines
  224. if change_fraction < min_change:
  225. return True
  226. # send notification
  227. diff = get_diff(id_).get('diff')
  228. if not diff:
  229. return True
  230. send_email(to=email, subject='%s page changed' % schedule.get('title'),
  231. body='changes:\n\n%s' % diff)
  232. return True
  233. def safe_run_job(id_=None, *args, **kwargs):
  234. """Safely run a job, catching all the exceptions
  235. :param id_: ID of the schedule to run
  236. :type id_: str
  237. :param args: positional arguments
  238. :type args: tuple
  239. :param kwargs: named arguments
  240. :type kwargs: dict
  241. :returns: True in case of success
  242. :rtype: bool"""
  243. try:
  244. run_job(id_, *args, **kwargs)
  245. except Exception as e:
  246. send_email('error executing job %s: %s' % (id_, e))
  247. def send_email(to, subject='diffido', body='', from_=None):
  248. """Send an email
  249. :param to: destination address
  250. :type to: str
  251. :param subject: email subject
  252. :type subject: str
  253. :param body: body of the email
  254. :type body: str
  255. :param from_: sender address
  256. :type from_: str
  257. :returns: True in case of success
  258. :rtype: bool"""
  259. msg = MIMEText(body)
  260. msg['Subject'] = subject
  261. msg['From'] = from_ or EMAIL_FROM
  262. msg['To'] = to
  263. msg["Date"] = formatdate(localtime=True)
  264. starttls = SMTP_SETTINGS.get('smtp-starttls')
  265. use_ssl = SMTP_SETTINGS.get('smtp-use-ssl')
  266. username = SMTP_SETTINGS.get('smtp-username')
  267. password = SMTP_SETTINGS.get('smtp-password')
  268. args = {}
  269. for key, value in SMTP_SETTINGS.items():
  270. if key in ('smtp-starttls', 'smtp-use-ssl', 'smtp-username', 'smtp-password'):
  271. continue
  272. if key in ('smtp-port'):
  273. value = int(value)
  274. key = key.replace('smtp-', '', 1).replace('-', '_')
  275. args[key] = value
  276. try:
  277. if use_ssl:
  278. for key in ('ssl_keyfile', 'ssl_certfile', 'ssl_context'):
  279. if key in args:
  280. args[key.replace('ssl_', '')] = args[key]
  281. del args[key]
  282. logger.debug('STMP SSL connection with args: %s' % repr(args))
  283. with smtplib.SMTP_SSL(**args) as s:
  284. if username:
  285. logger.debug('STMP LOGIN for username %s and password of length %d' % (username, len(password)))
  286. s.login(username, password)
  287. s.send_message(msg)
  288. else:
  289. tls_args = {}
  290. for key in ('ssl_keyfile', 'ssl_certfile', 'ssl_context'):
  291. if key in args:
  292. tls_args[key.replace('ssl_', '')] = args[key]
  293. del args[key]
  294. logger.debug('STMP connection with args: %s' % repr(args))
  295. with smtplib.SMTP(**args) as s:
  296. if starttls:
  297. logger.debug('STMP STARTTLS connection with args: %s' % repr(tls_args))
  298. s.ehlo_or_helo_if_needed()
  299. s.starttls(**tls_args)
  300. if username:
  301. logger.debug('STMP LOGIN for username %s and password of length %d' % (username, len(password)))
  302. s.login(username, password)
  303. s.send_message(msg)
  304. except Exception as e:
  305. logger.error('unable to send email to %s: %s' % (to, e))
  306. return False
  307. return True
  308. def get_history(id_, limit=None, add_info=False):
  309. """Read the history of a schedule
  310. :param id_: ID of the schedule
  311. :type id_: str
  312. :param limit: number of entries to fetch
  313. :type limit: int
  314. :param add_info: add information about the schedule itself
  315. :type add_info: int
  316. :returns: information about the schedule and its history
  317. :rtype: dict"""
  318. def _history(id_, limit, queue):
  319. try:
  320. os.chdir('storage/%s' % id_)
  321. except Exception as e:
  322. logger.info('unable to move to storage/%s directory: %s' % (id_, e))
  323. return queue.put(b'')
  324. cmd = [GIT_CMD, 'log', '--pretty=oneline', '--shortstat']
  325. if limit is not None:
  326. cmd.append('-%s' % limit)
  327. p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
  328. stdout, _ = p.communicate()
  329. queue.put(stdout)
  330. queue = multiprocessing.Queue()
  331. p = multiprocessing.Process(target=_history, args=(id_, limit, queue))
  332. p.start()
  333. res = queue.get().decode('utf-8')
  334. p.join()
  335. history = []
  336. for match in re_commit.finditer(res):
  337. info = match.groupdict()
  338. info['insertions'] = int(info['insertions'] or 0)
  339. info['deletions'] = int(info['deletions'] or 0)
  340. info['changes'] = max(info['insertions'], info['deletions'])
  341. history.append(info)
  342. last_id = None
  343. if history and 'id' in history[0]:
  344. last_id = history[0]['id']
  345. for idx, item in enumerate(history):
  346. item['seq'] = idx + 1
  347. data = {'history': history, 'last_id': last_id}
  348. if add_info:
  349. data['schedule'] = get_schedule(id_)
  350. return data
  351. def get_last_history(id_):
  352. """Read the last history entry of a schedule
  353. :param id_: ID of the schedule
  354. :type id_: str
  355. :returns: information about the schedule and its history
  356. :rtype: dict"""
  357. history = get_history(id_, limit=1)
  358. hist = history.get('history') or [{}]
  359. return hist[0]
  360. def get_diff(id_, commit_id='HEAD', old_commit_id=None):
  361. """Return the diff between commits of a schedule
  362. :param id_: ID of the schedule
  363. :type id_: str
  364. :param commit_id: the most recent commit ID; HEAD by default
  365. :type commit_id: str
  366. :param old_commit_id: the older commit ID; if None, the previous commit is used
  367. :type old_commit_id: str
  368. :returns: information about the schedule and the diff between commits
  369. :rtype: dict"""
  370. def _history(id_, commit_id, old_commit_id, queue):
  371. try:
  372. os.chdir('storage/%s' % id_)
  373. except Exception as e:
  374. logger.info('unable to move to storage/%s directory: %s' % (id_, e))
  375. return queue.put(b'')
  376. p = subprocess.Popen([GIT_CMD, 'diff', old_commit_id or '%s~' % commit_id, commit_id],
  377. stdout=subprocess.PIPE)
  378. stdout, _ = p.communicate()
  379. queue.put(stdout)
  380. queue = multiprocessing.Queue()
  381. p = multiprocessing.Process(target=_history, args=(id_, commit_id, old_commit_id, queue))
  382. p.start()
  383. res = queue.get().decode('utf-8')
  384. p.join()
  385. schedule = get_schedule(id_)
  386. return {'diff': res, 'schedule': schedule}
  387. def scheduler_update(scheduler, id_):
  388. """Update a scheduler job, using information from the JSON object
  389. :param scheduler: the TornadoScheduler instance to modify
  390. :type scheduler: TornadoScheduler
  391. :param id_: ID of the schedule that must be updated
  392. :type id_: str
  393. :returns: True in case of success
  394. :rtype: bool"""
  395. schedule = get_schedule(id_, add_id=False)
  396. if not schedule:
  397. logger.warn('unable to update empty schedule %s' % id_)
  398. return False
  399. trigger = schedule.get('trigger')
  400. if trigger not in ('interval', 'cron'):
  401. logger.warn('unable to update empty schedule %s: trigger not in ("cron", "interval")' % id_)
  402. return False
  403. args = {}
  404. if trigger == 'interval':
  405. args['trigger'] = 'interval'
  406. for unit in 'weeks', 'days', 'hours', 'minutes', 'seconds':
  407. if 'interval_%s' % unit not in schedule:
  408. continue
  409. try:
  410. val = schedule['interval_%s' % unit]
  411. if not val:
  412. continue
  413. args[unit] = int(val)
  414. except Exception:
  415. logger.warn('invalid argument on schedule %s: %s parameter %s is not an integer' %
  416. (id_, 'interval_%s' % unit, schedule['interval_%s' % unit]))
  417. if len(args) == 1:
  418. logger.error('no valid interval specified, skipping schedule %s' % id_)
  419. return False
  420. elif trigger == 'cron':
  421. try:
  422. cron_trigger = CronTrigger.from_crontab(schedule['cron_crontab'])
  423. args['trigger'] = cron_trigger
  424. except Exception:
  425. logger.warn('invalid argument on schedule %s: cron_tab parameter %s is not a valid crontab' %
  426. (id_, schedule.get('cron_crontab')))
  427. return False
  428. git_create_repo(id_)
  429. try:
  430. scheduler.add_job(safe_run_job, id=id_, replace_existing=True, kwargs={'id_': id_}, **args)
  431. except Exception as e:
  432. logger.warn('unable to update job %s: %s' % (id_, e))
  433. return False
  434. return True
  435. def scheduler_delete(scheduler, id_):
  436. """Update a scheduler job, using information from the JSON object
  437. :param scheduler: the TornadoScheduler instance to modify
  438. :type scheduler: TornadoScheduler
  439. :param id_: ID of the schedule
  440. :type id_: str
  441. :returns: True in case of success
  442. :rtype: bool"""
  443. try:
  444. scheduler.remove_job(job_id=id_)
  445. except Exception as e:
  446. logger.warn('unable to delete job %s: %s' % (id_, e))
  447. return False
  448. return git_delete_repo(id_)
  449. def reset_from_schedules(scheduler):
  450. """"Reset all scheduler jobs, using information from the JSON object
  451. :param scheduler: the TornadoScheduler instance to modify
  452. :type scheduler: TornadoScheduler
  453. :returns: True in case of success
  454. :rtype: bool"""
  455. ret = False
  456. try:
  457. scheduler.remove_all_jobs()
  458. for key in read_schedules().get('schedules', {}).keys():
  459. ret |= scheduler_update(scheduler, id_=key)
  460. except Exception as e:
  461. logger.warn('unable to reset all jobs: %s' % e)
  462. return False
  463. return ret
  464. def git_init():
  465. """Initialize Git global settings"""
  466. p = subprocess.Popen([GIT_CMD, 'config', '--global', 'user.email', '"%s"' % EMAIL_FROM])
  467. p.communicate()
  468. p = subprocess.Popen([GIT_CMD, 'config', '--global', 'user.name', '"Diffido"'])
  469. p.communicate()
  470. def git_create_repo(id_):
  471. """Create a Git repository
  472. :param id_: ID of the schedule
  473. :type id_: str
  474. :returns: True in case of success
  475. :rtype: bool"""
  476. repo_dir = 'storage/%s' % id_
  477. if os.path.isdir(repo_dir):
  478. return True
  479. p = subprocess.Popen([GIT_CMD, 'init', repo_dir])
  480. p.communicate()
  481. return p.returncode == 0
  482. def git_delete_repo(id_):
  483. """Delete a Git repository
  484. :param id_: ID of the schedule
  485. :type id_: str
  486. :returns: True in case of success
  487. :rtype: bool"""
  488. repo_dir = 'storage/%s' % id_
  489. if not os.path.isdir(repo_dir):
  490. return False
  491. try:
  492. shutil.rmtree(repo_dir)
  493. except Exception as e:
  494. logger.warn('unable to delete Git repository %s: %s' % (id_, e))
  495. return False
  496. return True
  497. class DiffidoBaseException(Exception):
  498. """Base class for diffido custom exceptions.
  499. :param message: text message
  500. :type message: str
  501. :param status: numeric http status code
  502. :type status: int"""
  503. def __init__(self, message, status=400):
  504. super(DiffidoBaseException, self).__init__(message)
  505. self.message = message
  506. self.status = status
  507. class BaseHandler(tornado.web.RequestHandler):
  508. """Base class for request handlers."""
  509. # A property to access the first value of each argument.
  510. arguments = property(lambda self: dict([(k, v[0].decode('utf-8'))
  511. for k, v in self.request.arguments.items()]))
  512. @property
  513. def clean_body(self):
  514. """Return a clean dictionary from a JSON body, suitable for a query on MongoDB.
  515. :returns: a clean copy of the body arguments
  516. :rtype: dict"""
  517. return escape.json_decode(self.request.body or '{}')
  518. def write_error(self, status_code, **kwargs):
  519. """Default error handler."""
  520. if isinstance(kwargs.get('exc_info', (None, None))[1], DiffidoBaseException):
  521. exc = kwargs['exc_info'][1]
  522. status_code = exc.status
  523. message = exc.message
  524. else:
  525. message = 'internal error'
  526. self.build_error(message, status=status_code)
  527. def initialize(self, **kwargs):
  528. """Add every passed (key, value) as attributes of the instance."""
  529. for key, value in kwargs.items():
  530. setattr(self, key, value)
  531. def build_error(self, message='', status=400):
  532. """Build and write an error message.
  533. :param message: textual message
  534. :type message: str
  535. :param status: HTTP status code
  536. :type status: int
  537. """
  538. self.set_status(status)
  539. self.write({'error': True, 'message': message})
  540. def build_success(self, message='', status=200):
  541. """Build and write a success message.
  542. :param message: textual message
  543. :type message: str
  544. :param status: HTTP status code
  545. :type status: int
  546. """
  547. self.set_status(status)
  548. self.write({'error': False, 'message': message})
  549. class SchedulesHandler(BaseHandler):
  550. """Schedules handler."""
  551. @gen.coroutine
  552. def get(self, id_=None, *args, **kwargs):
  553. """Get a schedule."""
  554. if id_ is not None:
  555. return self.write({'schedule': get_schedule(id_, add_history=True)})
  556. schedules = read_schedules()
  557. self.write(schedules)
  558. @gen.coroutine
  559. def put(self, id_=None, *args, **kwargs):
  560. """Update a schedule."""
  561. if id_ is None:
  562. return self.build_error(message='update action requires an ID')
  563. data = self.clean_body
  564. schedules = read_schedules()
  565. if id_ not in schedules.get('schedules', {}):
  566. return self.build_error(message='schedule %s not found' % id_)
  567. schedules['schedules'][id_] = data
  568. write_schedules(schedules)
  569. scheduler_update(scheduler=self.scheduler, id_=id_)
  570. self.write(get_schedule(id_=id_))
  571. @gen.coroutine
  572. def post(self, *args, **kwargs):
  573. """Add a schedule."""
  574. data = self.clean_body
  575. schedules = read_schedules()
  576. id_ = next_id(schedules)
  577. schedules['schedules'][id_] = data
  578. write_schedules(schedules)
  579. scheduler_update(scheduler=self.scheduler, id_=id_)
  580. self.write(get_schedule(id_=id_))
  581. @gen.coroutine
  582. def delete(self, id_=None, *args, **kwargs):
  583. """Delete a schedule."""
  584. if id_ is None:
  585. return self.build_error(message='an ID must be specified')
  586. schedules = read_schedules()
  587. if id_ in schedules.get('schedules', {}):
  588. del schedules['schedules'][id_]
  589. write_schedules(schedules)
  590. scheduler_delete(scheduler=self.scheduler, id_=id_)
  591. self.build_success(message='removed schedule %s' % id_)
  592. class RunScheduleHandler(BaseHandler):
  593. """Reset schedules handler."""
  594. @gen.coroutine
  595. def post(self, id_, *args, **kwargs):
  596. if run_job(id_, force=True):
  597. return self.build_success('job run')
  598. self.build_error('job not run')
  599. class ResetSchedulesHandler(BaseHandler):
  600. """Reset schedules handler."""
  601. @gen.coroutine
  602. def post(self, *args, **kwargs):
  603. reset_from_schedules(self.scheduler)
  604. class HistoryHandler(BaseHandler):
  605. """History handler."""
  606. @gen.coroutine
  607. def get(self, id_, *args, **kwargs):
  608. self.write(get_history(id_, add_info=True))
  609. class DiffHandler(BaseHandler):
  610. """Diff handler."""
  611. @gen.coroutine
  612. def get(self, id_, commit_id, old_commit_id=None, *args, **kwargs):
  613. self.write(get_diff(id_, commit_id, old_commit_id))
  614. class TemplateHandler(BaseHandler):
  615. """Handler for the template files in the / path."""
  616. @gen.coroutine
  617. def get(self, *args, **kwargs):
  618. """Get a template file."""
  619. page = 'index.html'
  620. if args and args[0]:
  621. page = args[0].strip('/')
  622. arguments = self.arguments
  623. self.render(page, **arguments)
  624. def serve():
  625. """Read configuration and start the server."""
  626. global EMAIL_FROM, SMTP_SETTINGS
  627. jobstores = {'default': SQLAlchemyJobStore(url=JOBS_STORE)}
  628. scheduler = TornadoScheduler(jobstores=jobstores, timezone=pytz.utc)
  629. scheduler.start()
  630. define('port', default=3210, help='run on the given port', type=int)
  631. define('address', default='', help='bind the server at the given address', type=str)
  632. define('ssl_cert', default=os.path.join(os.path.dirname(__file__), 'ssl', 'diffido_cert.pem'),
  633. help='specify the SSL certificate to use for secure connections')
  634. define('ssl_key', default=os.path.join(os.path.dirname(__file__), 'ssl', 'diffido_key.pem'),
  635. help='specify the SSL private key to use for secure connections')
  636. define('admin-email', default='', help='email address of the site administrator', type=str)
  637. define('smtp-host', default='localhost', help='SMTP server address', type=str)
  638. define('smtp-port', default=0, help='SMTP server port', type=int)
  639. define('smtp-local-hostname', default=None, help='SMTP local hostname', type=str)
  640. define('smtp-use-ssl', default=False, help='Use SSL to connect to the SMTP server', type=bool)
  641. define('smtp-starttls', default=False, help='Use STARTTLS to connect to the SMTP server', type=bool)
  642. define('smtp-ssl-keyfile', default=None, help='SMTP SSL key file', type=str)
  643. define('smtp-ssl-certfile', default=None, help='SMTP SSL cert file', type=str)
  644. define('smtp-ssl-context', default=None, help='SMTP SSL context', type=str)
  645. define('smtp-username', default='', help='SMTP username', type=str)
  646. define('smtp-password', default='', help='SMTP password', type=str)
  647. define('debug', default=False, help='run in debug mode', type=bool)
  648. define('config', help='read configuration file',
  649. callback=lambda path: tornado.options.parse_config_file(path, final=False))
  650. if not options.config and os.path.isfile(DEFAULT_CONF):
  651. tornado.options.parse_config_file(DEFAULT_CONF, final=False)
  652. tornado.options.parse_command_line()
  653. if options.admin_email:
  654. EMAIL_FROM = options.admin_email
  655. for key, value in options.as_dict().items():
  656. if key.startswith('smtp-'):
  657. SMTP_SETTINGS[key] = value
  658. if options.debug:
  659. logger.setLevel(logging.DEBUG)
  660. ssl_options = {}
  661. if os.path.isfile(options.ssl_key) and os.path.isfile(options.ssl_cert):
  662. ssl_options = dict(certfile=options.ssl_cert, keyfile=options.ssl_key)
  663. init_params = dict(listen_port=options.port, logger=logger, ssl_options=ssl_options,
  664. scheduler=scheduler)
  665. git_init()
  666. _reset_schedules_path = r'schedules/reset'
  667. _schedule_run_path = r'schedules/(?P<id_>\d+)/run'
  668. _schedules_path = r'schedules/?(?P<id_>\d+)?'
  669. _history_path = r'schedules/?(?P<id_>\d+)/history'
  670. _diff_path = r'schedules/(?P<id_>\d+)/diff/(?P<commit_id>[0-9a-f]+)/?(?P<old_commit_id>[0-9a-f]+)?/?'
  671. application = tornado.web.Application([
  672. (r'/api/%s' % _reset_schedules_path, ResetSchedulesHandler, init_params),
  673. (r'/api/v%s/%s' % (API_VERSION, _reset_schedules_path), ResetSchedulesHandler, init_params),
  674. (r'/api/%s' % _schedule_run_path, RunScheduleHandler, init_params),
  675. (r'/api/v%s/%s' % (API_VERSION, _schedule_run_path), RunScheduleHandler, init_params),
  676. (r'/api/%s' % _history_path, HistoryHandler, init_params),
  677. (r'/api/v%s/%s' % (API_VERSION, _history_path), HistoryHandler, init_params),
  678. (r'/api/%s' % _diff_path, DiffHandler, init_params),
  679. (r'/api/v%s/%s' % (API_VERSION, _diff_path), DiffHandler, init_params),
  680. (r'/api/%s' % _schedules_path, SchedulesHandler, init_params),
  681. (r'/api/v%s/%s' % (API_VERSION, _schedules_path), SchedulesHandler, init_params),
  682. (r'/?(.*)', TemplateHandler, init_params),
  683. ],
  684. static_path=os.path.join(os.path.dirname(__file__), 'dist/static'),
  685. template_path=os.path.join(os.path.dirname(__file__), 'dist/'),
  686. debug=options.debug)
  687. http_server = tornado.httpserver.HTTPServer(application, ssl_options=ssl_options or None)
  688. logger.info('Start serving on %s://%s:%d', 'https' if ssl_options else 'http',
  689. options.address if options.address else '127.0.0.1',
  690. options.port)
  691. http_server.listen(options.port, options.address)
  692. try:
  693. IOLoop.instance().start()
  694. except (KeyboardInterrupt, SystemExit):
  695. pass
  696. if __name__ == '__main__':
  697. serve()