serious packaging
This commit is contained in:
parent
fc0c73b0dd
commit
9e77197ec4
6 changed files with 225 additions and 176 deletions
0
bandcavall/__init__.py
Normal file
0
bandcavall/__init__.py
Normal file
35
bandcavall/download.py
Normal file
35
bandcavall/download.py
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
import os
|
||||
from os.path import join, exists
|
||||
import requests
|
||||
import mutagen.mp3
|
||||
import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def download_track(track, args):
|
||||
log.debug('Downloading "%s"' % track['title'])
|
||||
if 'album' in track:
|
||||
fdir = join(track['artist'], track['album']).replace(' ', '_')
|
||||
else:
|
||||
fdir = join(track['artist'])
|
||||
fname = track['title'].replace(' ', '_') + '.mp3'
|
||||
fpath = join(args.download_base, fdir, fname)
|
||||
os.makedirs(join(args.download_base, fdir), exist_ok=True)
|
||||
if exists(fpath):
|
||||
log.info('file %s exists, skipping' % fpath)
|
||||
log.debug('download in ' + fpath)
|
||||
resp = requests.get(track['url'], stream=True)
|
||||
if not resp.ok:
|
||||
log.error('Error downloading track ', track['title'])
|
||||
return
|
||||
with open(fpath, 'wb') as buf:
|
||||
log.debug('writing ' + fname)
|
||||
for block in resp.iter_content():
|
||||
buf.write(block)
|
||||
audio = mutagen.mp3.EasyMP3(fpath) # TODO: is it always mp3?
|
||||
audio['title'] = track['title']
|
||||
audio['artist'] = track['artist']
|
||||
if 'album' in track:
|
||||
audio['album'] = track['album']
|
||||
audio.save()
|
||||
log.info('track %(title)s downloaded' % track)
|
||||
98
bandcavall/main.py
Normal file
98
bandcavall/main.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
from os.path import join
|
||||
import argparse
|
||||
from urllib.parse import urlparse
|
||||
import logging
|
||||
import json
|
||||
import multiprocessing
|
||||
import functools
|
||||
|
||||
from splinter import Browser
|
||||
|
||||
from .download import download_track
|
||||
from .visit import visit_track, visit_album, visit_artist
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_parser():
|
||||
p = argparse.ArgumentParser(
|
||||
description='Download tracks from bandcamp, sharing is caring!',
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
p.add_argument('url')
|
||||
p.add_argument('--loglevel', default='INFO', help='Log level',
|
||||
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'])
|
||||
info = p.add_argument_group('fetch track info')
|
||||
info.add_argument('--driver', default='firefox',
|
||||
help='Internal splinter option',
|
||||
choices=['firefox', 'chrome'])
|
||||
info.add_argument('--mode', default=None,
|
||||
choices=['album', 'track', 'artist'])
|
||||
do = p.add_argument_group('what to do with fetched urls')
|
||||
do.add_argument('--do', default='print',
|
||||
help='What to do with retrieved informations',
|
||||
choices=['print', 'wgetprint', 'download'])
|
||||
do.add_argument('--download-base', default='.',
|
||||
help='Only applies to wget')
|
||||
do.add_argument('-j', '--jobs', default=3, type=int,
|
||||
help='Parallel downloads')
|
||||
|
||||
return p
|
||||
|
||||
|
||||
def get_mode(url):
|
||||
path = urlparse(url).path
|
||||
if path.startswith('/track/'):
|
||||
return 'track'
|
||||
if path.startswith('/album/'):
|
||||
return 'album'
|
||||
raise ValueError('unsupported url: "{}"'.format(path))
|
||||
|
||||
|
||||
def main():
|
||||
args = get_parser().parse_args()
|
||||
logging.basicConfig(level=args.loglevel)
|
||||
logging.getLogger('selenium.webdriver').setLevel(logging.WARNING)
|
||||
logging.getLogger('requests').setLevel(logging.WARNING)
|
||||
|
||||
if args.mode is None:
|
||||
args.mode = get_mode(args.url)
|
||||
log.info('Going in %s mode' % args.mode)
|
||||
|
||||
prefs = {
|
||||
'dom.audiochannel.mutedByDefault': True
|
||||
}
|
||||
with Browser(args.driver, profile_preferences=prefs) as br:
|
||||
# get track info
|
||||
if args.mode == 'track':
|
||||
tracks = visit_track(br, args.url)
|
||||
elif args.mode == 'album':
|
||||
tracks = visit_album(br, args.url)
|
||||
else:
|
||||
tracks = visit_artist(br, args.url)
|
||||
|
||||
# do something
|
||||
if args.do == 'print':
|
||||
for track in tracks:
|
||||
print(json.dumps(track))
|
||||
elif args.do == 'wgetprint':
|
||||
for track in tracks:
|
||||
cmd = ['wget', '-nv', "'%s'" % track['url'],
|
||||
'-O', "'%s'" %
|
||||
join(args.download_base,
|
||||
'-'.join((track['artist'], track['album'],
|
||||
track['title']))).replace(' ', '_')
|
||||
]
|
||||
print(' '.join(cmd))
|
||||
elif args.do == 'download':
|
||||
pool = multiprocessing.Pool(processes=args.jobs)
|
||||
for res in pool.imap_unordered(functools.partial(download_track,
|
||||
args=args),
|
||||
tracks):
|
||||
print('R', res)
|
||||
pool.close()
|
||||
pool.join()
|
||||
else:
|
||||
raise NotImplementedError('action ' + args.do)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
54
bandcavall/visit.py
Normal file
54
bandcavall/visit.py
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
import time
|
||||
import logging
|
||||
|
||||
from splinter.exceptions import ElementDoesNotExist
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def visit_track(b, url):
|
||||
log.debug('Visiting track %s' % url)
|
||||
b.visit(url)
|
||||
b.find_by_css('.playbutton').first.click()
|
||||
info = {}
|
||||
info['artist'] = b.find_by_xpath('//span[@itemprop="byArtist"]') \
|
||||
.first.value
|
||||
try:
|
||||
info['album'] = b.find_by_xpath('//span[@itemprop="inAlbum"]') \
|
||||
.first.value
|
||||
except ElementDoesNotExist: # non-album tracks
|
||||
pass
|
||||
info['title'] = b.find_by_css('#name-section h2.trackTitle').first.value
|
||||
time.sleep(0.2) # how nice!
|
||||
b.find_by_css('.playbutton').first.click()
|
||||
info['url'] = b.find_by_css('audio').first['src']
|
||||
return info
|
||||
|
||||
|
||||
def visit_album(b, url):
|
||||
log.debug('Visiting album %s' % url)
|
||||
b.visit(url)
|
||||
track_urls = [link['href'] for link in
|
||||
b.find_by_css('.track_row_view.linked .title a')
|
||||
if link['href'] and '/track/' in link['href']]
|
||||
for track_url in track_urls:
|
||||
yield visit_track(b, track_url)
|
||||
|
||||
|
||||
def visit_artist(b, url):
|
||||
log.debug('Visiting artist %s' % url)
|
||||
b.visit(url)
|
||||
album_urls = [link['href'] for link in
|
||||
b.find_by_css('.square a')
|
||||
if link['href'] and '/album/' in link['href']]
|
||||
nonalbumtrack_urls = [link['href'] for link in
|
||||
b.find_by_css('.square a')
|
||||
if link['href'] and '/track/' in link['href']]
|
||||
log.debug('Found %d album + %d non-album tracks' % (len(album_urls),
|
||||
len(nonalbumtrack_urls)
|
||||
))
|
||||
for track_url in nonalbumtrack_urls:
|
||||
yield visit_track(b, track_url)
|
||||
for album_url in album_urls:
|
||||
for track in visit_album(b, album_url):
|
||||
yield track
|
||||
176
fetchcose.py
176
fetchcose.py
|
|
@ -1,176 +0,0 @@
|
|||
import time
|
||||
import os
|
||||
from os.path import join, exists
|
||||
import argparse
|
||||
from urllib.parse import urlparse
|
||||
import logging
|
||||
import json
|
||||
import multiprocessing
|
||||
|
||||
from splinter import Browser
|
||||
from splinter.exceptions import ElementDoesNotExist
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def visit_track(b, url):
|
||||
log.debug('Visiting track %s' % url)
|
||||
b.visit(url)
|
||||
b.find_by_css('.playbutton').first.click()
|
||||
info = {}
|
||||
info['artist'] = b.find_by_xpath('//span[@itemprop="byArtist"]') \
|
||||
.first.value
|
||||
try:
|
||||
info['album'] = b.find_by_xpath('//span[@itemprop="inAlbum"]') \
|
||||
.first.value
|
||||
except ElementDoesNotExist: # non-album tracks
|
||||
pass
|
||||
info['title'] = b.find_by_css('#name-section h2.trackTitle').first.value
|
||||
time.sleep(0.2) # how nice!
|
||||
b.find_by_css('.playbutton').first.click()
|
||||
info['url'] = b.find_by_css('audio').first['src']
|
||||
return info
|
||||
|
||||
|
||||
def visit_album(b, url):
|
||||
log.debug('Visiting album %s' % url)
|
||||
b.visit(url)
|
||||
track_urls = [link['href'] for link in
|
||||
b.find_by_css('.track_row_view.linked .title a')
|
||||
if link['href'] and '/track/' in link['href']]
|
||||
for track_url in track_urls:
|
||||
yield visit_track(b, track_url)
|
||||
|
||||
|
||||
def visit_artist(b, url):
|
||||
log.debug('Visiting artist %s' % url)
|
||||
b.visit(url)
|
||||
album_urls = [link['href'] for link in
|
||||
b.find_by_css('.square a')
|
||||
if link['href'] and '/album/' in link['href']]
|
||||
nonalbumtrack_urls = [link['href'] for link in
|
||||
b.find_by_css('.square a')
|
||||
if link['href'] and '/track/' in link['href']]
|
||||
log.debug('Found %d album + %d non-album tracks' % (len(album_urls),
|
||||
len(nonalbumtrack_urls)
|
||||
))
|
||||
for track_url in nonalbumtrack_urls:
|
||||
yield visit_track(b, track_url)
|
||||
for album_url in album_urls:
|
||||
for track in visit_album(b, album_url):
|
||||
yield track
|
||||
|
||||
|
||||
def download_track(track):
|
||||
log.debug('Downloading "%s"' % track['title'])
|
||||
if 'album' in track:
|
||||
fdir = join(track['artist'], track['album']).replace(' ', '_')
|
||||
else:
|
||||
fdir = join(track['artist'])
|
||||
fname = track['title'].replace(' ', '_') + '.mp3'
|
||||
fpath = join(args.download_base, fdir, fname)
|
||||
os.makedirs(join(args.download_base, fdir), exist_ok=True)
|
||||
if exists(fpath):
|
||||
log.info('file %s exists, skipping' % fpath)
|
||||
log.debug('download in ' + fpath)
|
||||
resp = requests.get(track['url'], stream=True)
|
||||
if not resp.ok:
|
||||
log.error('Error downloading track ', track['title'])
|
||||
return
|
||||
with open(fpath, 'wb') as buf:
|
||||
log.debug('writing ' + fname)
|
||||
for block in resp.iter_content():
|
||||
buf.write(block)
|
||||
audio = mutagen.mp3.EasyMP3(fpath) # TODO: is it always mp3?
|
||||
audio['title'] = track['title']
|
||||
audio['artist'] = track['artist']
|
||||
if 'album' in track:
|
||||
audio['album'] = track['album']
|
||||
audio.save()
|
||||
log.info('track %(title)s downloaded' % track)
|
||||
return fpath
|
||||
|
||||
|
||||
def get_parser():
|
||||
p = argparse.ArgumentParser(
|
||||
description='Download tracks from bandcamp, sharing is caring!',
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
p.add_argument('url')
|
||||
p.add_argument('--loglevel', default='INFO', help='Log level',
|
||||
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'])
|
||||
info = p.add_argument_group('fetch track info')
|
||||
info.add_argument('--driver', default='firefox',
|
||||
help='Internal splinter option',
|
||||
choices=['firefox', 'chrome'])
|
||||
info.add_argument('--mode', default=None,
|
||||
choices=['album', 'track', 'artist'])
|
||||
do = p.add_argument_group('what to do with fetched urls')
|
||||
do.add_argument('--do', default='print',
|
||||
help='What to do with retrieved informations',
|
||||
choices=['print', 'wgetprint', 'download'])
|
||||
do.add_argument('--download-base', default='.',
|
||||
help='Only applies to wget')
|
||||
do.add_argument('-j', '--jobs', default=3, type=int,
|
||||
help='Parallel downloads')
|
||||
|
||||
return p
|
||||
|
||||
|
||||
def get_mode(url):
|
||||
path = urlparse(url).path
|
||||
if path.startswith('/track/'):
|
||||
return 'track'
|
||||
if path.startswith('/album/'):
|
||||
return 'album'
|
||||
raise ValueError('unsupported url')
|
||||
|
||||
|
||||
args = get_parser().parse_args()
|
||||
logging.basicConfig(level=args.loglevel)
|
||||
logging.getLogger('selenium.webdriver').setLevel(logging.WARNING)
|
||||
logging.getLogger('requests').setLevel(logging.WARNING)
|
||||
if args.do == 'download':
|
||||
try:
|
||||
import requests
|
||||
import mutagen.mp3
|
||||
except ImportError:
|
||||
log.critical('--do=download requires requests and download libraries')
|
||||
raise
|
||||
|
||||
if args.mode is None:
|
||||
args.mode = get_mode(args.url)
|
||||
log.info('Going in %s mode' % args.mode)
|
||||
|
||||
prefs = {
|
||||
'dom.audiochannel.mutedByDefault': True
|
||||
}
|
||||
with Browser(args.driver, profile_preferences=prefs) as br:
|
||||
# get track info
|
||||
if args.mode == 'track':
|
||||
tracks = visit_track(br, args.url)
|
||||
elif args.mode == 'album':
|
||||
tracks = visit_album(br, args.url)
|
||||
else:
|
||||
tracks = visit_artist(br, args.url)
|
||||
|
||||
# do something
|
||||
if args.do == 'print':
|
||||
for track in tracks:
|
||||
print(json.dumps(track))
|
||||
elif args.do == 'wgetprint':
|
||||
for track in tracks:
|
||||
cmd = ['wget', '-nv', "'%s'" % track['url'],
|
||||
'-O', "'%s'" %
|
||||
join(args.download_base,
|
||||
'-'.join((track['artist'], track['album'],
|
||||
track['title']))).replace(' ', '_')
|
||||
]
|
||||
print(' '.join(cmd))
|
||||
elif args.do == 'download':
|
||||
pool = multiprocessing.Pool(processes=args.jobs)
|
||||
for res in pool.imap_unordered(download_track, tracks):
|
||||
print('R', res)
|
||||
pool.close()
|
||||
pool.join()
|
||||
else:
|
||||
raise NotImplementedError('action ' + args.do)
|
||||
38
setup.py
Normal file
38
setup.py
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
import os
|
||||
|
||||
from setuptools import setup
|
||||
|
||||
|
||||
def read(fname):
|
||||
with open(os.path.join(os.path.dirname(__file__), fname)) as buf:
|
||||
return buf.read()
|
||||
|
||||
|
||||
conf = dict(
|
||||
name='bandcavall',
|
||||
version='0.1',
|
||||
description='Download music from bandcamp',
|
||||
long_description=read('README.md'),
|
||||
author='boyska',
|
||||
author_email='piuttosto@logorroici.org',
|
||||
url='https://git.lattuga.net/boyska/bandcavall',
|
||||
license='AGPL',
|
||||
packages=['bandcavall'],
|
||||
install_requires=[
|
||||
'splinter',
|
||||
'requests',
|
||||
'mutagen',
|
||||
],
|
||||
zip_safe=False,
|
||||
entry_points={'console_scripts': [
|
||||
'bandcavall=bandcavall.main:main',
|
||||
]},
|
||||
classifiers=[
|
||||
"License :: OSI Approved :: GNU Affero General Public License v3",
|
||||
"Operating System :: POSIX :: Linux",
|
||||
"Programming Language :: Python :: 3",
|
||||
])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
setup(**conf)
|
||||
Loading…
Reference in a new issue