ewa.clisupport

Package ewa :: Module clisupport |
[hide private]
[frames] | no frames]

# Source Code for Module ewa.clisupport

      1  import errno
      2  import logging
      3  from optparse import OptionParser
      4  import os
      5  from os import path
      6  import re
      7  import sys
      8  import time
      9
     10  try:
     11      import grp
     12      import pwd
     13      have_unix = 1
     14  except ImportError:
     15      have_unix = 0
     16
     17  try:
     18      from functools import partial
     19  except ImportError:
     20
     21 -    def partial(func, *args, **kw):
     22
     23          def inner(*_args, **_kw):
     24              d = kw.copy()
     25              d.update(_kw)
     26              return func(*(args + _args), **d)
     27          return inner
     28
     29  try:
     30      from flup.server.scgi_fork import WSGIServer as SCGIServer
     31      from flup.server.fcgi_fork import WSGIServer as FCGIServer
     32      from flup.server.scgi import WSGIServer as SCGIThreadServer
     33      from flup.server.fcgi import WSGIServer as FCGIThreadServer
     34      haveflup = True
     35  except ImportError:
     36      haveflup = False
     37  try:
     38      from paste import httpserver
     39      havepaste = True
     40  except ImportError:
     41      havepaste = False
     42
     43  import ewa.mp3
     44  import ewa.audio
     45  from ewa.config import Config, initConfig
     46  from ewa.lighttpd_hack_middleware import LighttpdHackMiddleware
     47  from ewa.logutil import (debug, error, exception, info, logger,
     48                           initLogging, warn)
     49  from ewa.wsgiapp import EwaApp
     50  from ewa.rules import FileRule
     51  from ewa import __version__
     52
     53  VERSION_TEXT = """\
     54  %%s %s
     55
     56  Copyright (C) 2007, 2010 WNYC New York Public Radio.
     57  This is free software; see the source for copying conditions.  There is NO
     58  warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
     59
     60  Written by Jacob Smullyan and others, not necessarily in that order.
     61  """ % __version__
     62
     63
     64  SERVE_DESCRIPTION = """\
     65  Starts a WSGI application that produces combined MP3 files
     66  according to the specified rules.
     67  """
     68
     69  RULE_DESCRIPTION = """\
     70  Produces a combined MP3 file according to the specified rules.
     71  """
     72
     73  SPLICE_DESCRIPTION = """\
     74  Splices MP3 files together.
     75  """
     76
     77
     78 -def get_serve_parser():
     79      protocols = []
     80      if haveflup:
     81          protocols.extend(['fcgi', 'scgi'])
     82      if havepaste:
     83          protocols.append('http')
     84      if not protocols:
     85          print >> sys.stderr, ("no protocols available; "
     86                                "please install paste and/or flup")
     87          sys.exit(1)
     88      usage = "usage: %prog [options]"
     89      parser = OptionParser(usage, description=SERVE_DESCRIPTION)
     90      parser.add_option('-c',
     91                        '--config',
     92                        dest='configfile',
     93                        default=None,
     94                        help="path to ewa config file")
     95      parser.add_option('-D',
     96                        '--nodaemonize',
     97                        action='store_true',
     98                        default=False,
     99                        dest='nodaemonize',
    100                        help="don't daemonize, regardless of config settings")
    101      parser.add_option('--version',
    102                        action="store_true",
    103                        dest="version",
    104                        help="show version and exit")
    105      parser.add_option('--lighttpd-hack',
    106                        action='store_true',
    107                        default=False,
    108                        dest='lighttpd_hack',
    109                        help=("hack for some versions of lighttpd "
    110                              "to force SCRIPT_NAME to ''"))
    111      return parser
    112
    113
    114 -def get_splice_parser():
    115      usage = "%prog [options] files"
    116      parser = OptionParser(usage, description=SPLICE_DESCRIPTION)
    117      parser.add_option('-o',
    118                        '--output',
    119                        dest='output',
    120                        help="output file (default: stdout)",
    121                        default='-',
    122                        metavar='OUT')
    123      parser.add_option('-t',
    124                        '--tagfile',
    125                        dest="tagfile",
    126                        help="tag file",
    127                        default=None,
    128                        metavar="TAGFILE")
    129      parser.add_option('-d',
    130                        '--debug',
    131                        action='store_true',
    132                        default=False,
    133                        help="print debugging information",
    134                        dest='debugmode')
    135      parser.add_option('-s',
    136                        '--sanitycheck',
    137                        action='store_true',
    138                        default=False,
    139                        help="sanity check the input mp3 files",
    140                        dest='sanitycheck')
    141      parser.add_option('-e',
    142                        '--engine',
    143                        default='default',
    144                        dest='engine',
    145                        metavar='ENGINE',
    146                        choices=('default', 'mp3cat', 'sox'),
    147                        help=("which splicing engine to use (default ewa "
    148                              "splicer, mp3cat, or sox)"))
    149      parser.add_option('--version',
    150                        action="store_true",
    151                        dest="version",
    152                        help="show version and exit")
    153      return parser
    154
    155
    156 -def get_ap_parser():
    157      usage = "%prog [options] [files]"
    158      parser = OptionParser(usage, description=RULE_DESCRIPTION)
    159
    160      parser.add_option('-c',
    161                        '--config',
    162                        dest='configfile',
    163                        default=None,
    164                        help="path to ewa config file")
    165      parser.add_option('-r',
    166                        '--recursive',
    167                        dest='recursive',
    168                        help="recurse through directories",
    169                        action='store_true',
    170                        default=False)
    171      parser.add_option('--rulefile',
    172                        dest='rulefile',
    173                        help="specify a rulefile",
    174                        metavar='RULEFILE')
    175      parser.add_option('-d',
    176                        '--debug',
    177                        action='store_true',
    178                        default=False,
    179                        help="print debugging information",
    180                        dest='debugmode')
    181      parser.add_option('-n',
    182                        '--dry-run',
    183                        action='store_true',
    184                        default=False,
    185                        dest='dryrun',
    186                        help="don't do anything, just print what would be done")
    187      parser.add_option('-e',
    188                        '--engine',
    189                        default=None,
    190                        dest='engine',
    191                        metavar='ENGINE',
    192                        choices=('default', 'mp3cat', 'sox'),
    193                        help=("which splicing engine to use (default ewa "
    194                              "splicer, mp3cat, or sox)"))
    195      parser.add_option('-a',
    196                        '--absolute',
    197                        default=False,
    198                        dest='absolute',
    199                        action='store_true',
    200                        help=("interpret file paths relative to the filesystem "
    201                              "rather than the basedir (default: no)"))
    202
    203      parser.add_option('-t',
    204                        '--configtest',
    205                        default=False,
    206                        dest='configtest',
    207                        action='store_true',
    208                        help="just test the config file for syntax errors")
    209
    210      parser.add_option('-x',
    211                        '--max-age',
    212                        default=0,
    213                        dest='max_age',
    214                        type=int,
    215                        help=('max age before generated files expire and '
    216                              'are regenerated. Default is 0, which means no '
    217                              'expiration.  -1 will force regeneration.'))
    218
    219      parser.add_option('-D',
    220                        '--delete',
    221                        default=False,
    222                        action='store_true',
    223                        help=('delete files in combined directory that are '
    224                              'not in the the main directory'))
    225
    226      parser.add_option('--version',
    227                        action="store_true",
    228                        dest="version",
    229                        help="show version and exit")
    230      return parser
    231
    232
    233 -def resolve_engine(enginename):
    234      if enginename == 'default':
    235          return ewa.mp3._default_splicer
    236      elif enginename == 'mp3cat':
    237          return ewa.mp3._mp3cat_splicer
    238      elif enginename == 'sox':
    239          return ewa.mp3._sox_splicer
    240
    241
    242 -def do_splice(args):
    243      parser = get_splice_parser()
    244      opts, args = parser.parse_args(args)
    245      if opts.version:
    246          print VERSION_TEXT % path.basename(sys.argv[0])
    247          sys.exit(0)
    248      if opts.debugmode:
    249          Config.loglevel = logging.DEBUG
    250      initLogging(level=Config.loglevel,
    251                  filename=Config.logfile)
    252
    253      engine = resolve_engine(opts.engine)
    254      if opts.sanitycheck:
    255          try:
    256              ewa.mp3.mp3_sanity_check(args)
    257          except Exception, e:
    258              print >> sys.stderr, 'sanity check failed: %s' % str(e)
    259              sys.exit(1)
    260
    261      use_stdout = opts.output == '-'
    262      if use_stdout:
    263          fp = sys.stdout
    264      else:
    265          fp = open(options.output, 'wb')
    266      for chunk in ewa.mp3.splice(args, opts.tagfile, splicer=engine):
    267          fp.write(chunk)
    268      if not use_stdout:
    269          fp.close()
    270
    271
    272 -def do_audioprovider(args):
    273      parser = gs = get_ap_parser()
    274      opts, args = parser.parse_args(args)
    275      if opts.version:
    276          print VERSION_TEXT % path.basename(sys.argv[0])
    277          sys.exit(0)
    278      configfile = _find_config(opts.configfile)
    279      if not configfile:
    280          parser.error('no config file specified or found in the usual places')
    281      if not path.exists(configfile):
    282          parser.error("config file does not exist: %s" % configfile)
    283      initConfig(configfile)
    284
    285      # override config
    286      if opts.debugmode:
    287          Config.loglevel = 'debug'
    288      if opts.rulefile:
    289          Config.rulefile = opts.rulefile
    290      if opts.engine:
    291          Config.engine = engine
    292      engine = resolve_engine(Config.engine)
    293      initLogging(level=Config.loglevel)
    294      rule = FileRule(Config.rulefile)
    295
    296      if opts.configtest:
    297          print "Config OK"
    298          sys.exit(0)
    299
    300      provider = ewa.audio.FSAudioProvider(Config.basedir,
    301                                           Config.targetdir)
    302      mainpath = provider.get_main_path("")
    303      if not mainpath.endswith('/'):
    304          # currently this won't happen,
    305          # but better safe than sorry
    306          mainpath += '/'
    307      if opts.absolute:
    308          abs = [path.abspath(f) for f in args]
    309          stripped_main = mainpath[:-1]
    310          if not path.commonprefix([stripped_main] + abs) == stripped_main:
    311              debug("absolute paths of files: %s", abs)
    312              debug("mainpath: %s", mainpath)
    313              parser.error("files outside managed directory")
    314          idx = len(mainpath)
    315          args = [x[idx:] for x in abs]
    316      else:
    317          args = [re.sub('/*(.*)', r'\1', x) for x in args]
    318
    319      if opts.delete:
    320          if not opts.recursive:
    321              parser.error("delete only works in recursive mode")
    322          delete_finder = DeleteFinder(args, mainpath)
    323          for file, isdir in delete_finder:
    324              info('deleting %s', file)
    325              if not opts.dryrun:
    326                  try:
    327                      if isdir:
    328                          os.rmdir(file)
    329                      else:
    330                          os.unlink(file)
    331                  except Exception, e:
    332                      error("couldn't unlink %s: %s", file, e)
    333
    334      if opts.recursive:
    335          # replace args with an iterator that finds mp3 files
    336          if opts.max_age == -1:
    337              args = RecursiveMp3FileIterator(args, mainpath)
    338          else:
    339              args = RecursiveChangedMp3FileIterator(args,
    340                                                     mainpath,
    341                                                     opts.max_age)
    342      if not args:
    343          parser.error("no files specified")
    344
    345      if opts.dryrun:
    346          for file in args:
    347              debug("mp3 file: %s", file)
    348              print "playlist for %s:" % file
    349              for part in provider.get_playlist(file, rule, False):
    350                  print "\t%s" % part
    351          sys.exit(0)
    352      else:
    353          _change_user_group()
    354          for file in args:
    355              try:
    356                  target = provider.create_combined(file, rule, splicer=engine)
    357              except:
    358                  exception("error creating combined file for %s", file)
    359              else:
    360                  debug('created %s', target)
    361      sys.exit(0)
    362
    363
    364 -class DeleteFinder(object):
    365 -    def __init__(self, files, basedir):
    366          self.files = files
    367          self.basedir = basedir
    368          self.targetdir = Config.targetdir
    369          if self.targetdir is None:
    370              self.targetdir = path.join(Config.basedir, 'combined')
    371
    372 -    def _yielder(self, root, apath, isdir):
    373          targetpath = os.path.join(root, apath)
    374          mainpath = path.join(self.basedir,
    375                               path.relpath(targetpath, self.targetdir))
    376
    377          debug('mainpath for %s is %s', apath, mainpath)
    378          if not os.path.exists(mainpath):
    379              yield targetpath, isdir
    380
    381 -    def __iter__(self):
    382          for f in (path.join(self.targetdir, x) for x in self.files):
    383              if path.isdir(f):
    384                  debug('found directory: %s', f)
    385                  for root, dirs, files in os.walk(f, topdown=False):
    386                      for thing in files:
    387                          if thing.endswith('~'):
    388                              # looks like an ewa temp file.  Let it be.
    389                              continue
    390                          for res in self._yielder(root, thing, 0):
    391                              yield res
    392                      for thing in dirs:
    393                          for res in self._yielder(root, thing, 1):
    394                              yield res
    395
    396
    397 -class RecursiveMp3FileIterator(object):
    398 -    def __init__(self, files, basedir):
    399          self.files = files
    400          self.basedir = basedir
    401
    402 -    def __iter__(self):
    403          length = len(self.basedir)
    404          for f in (path.join(self.basedir, x) for x in self.files):
    405              if path.isdir(f):
    406                  debug('found directory: %s', f)
    407                  for mp3 in self._walk(f):
    408                      debug('raw value: %s', mp3)
    409                      yield mp3[length:]
    410              else:
    411                  debug('found non-directory: %s', f)
    412                  yield f[length:]
    413
    414 -    def _walk(self, f):
    415          for root, dirs, files in os.walk(f):
    416              for f in files:
    417                  if f.endswith('.mp3') or f.endswith('MP3'):
    418                      yield path.join(root, f)
    419
    420
    421 -class RecursiveChangedMp3FileIterator(RecursiveMp3FileIterator):
    422
    423 -    def __init__(self, files, basedir, max_age=0):
    424          super(RecursiveChangedMp3FileIterator, self).__init__(files, basedir)
    425          self.targetdir = Config.targetdir
    426          if self.targetdir is None:
    427              self.targetdir = path.join(Config.basedir, 'combined')
    428          self.max_age = max_age
    429
    430 -    def _walk(self, f):
    431          for root, dirs, files in os.walk(f):
    432              for f in files:
    433                  if f.endswith('.mp3') or f.endswith('MP3'):
    434                      fullpath = path.join(root, f)
    435                      debug('fullpath is %s', fullpath)
    436                      debug('basedir: %s; targetdir: %s',
    437                            self.basedir, self.targetdir)
    438                      targetpath = os.path.join(self.targetdir,
    439                                                path.relpath(fullpath,
    440                                                             self.basedir))
    441                      try:
    442                          stats = os.stat(targetpath)
    443                      except OSError, ozzie:
    444                          if ozzie.errno == errno.ENOENT:
    445                              # target doesn't exist
    446                              debug("target doesn't exist: %s", targetpath)
    447                              yield fullpath
    448                          else:
    449                              exception("error statting targetfile")
    450                              # should we return this, or continue? ????
    451                      else:
    452                          # we know the file exists
    453                          target_mtime = stats.st_mtime
    454                          try:
    455                              original_mtime = path.getmtime(fullpath)
    456                          except OSError:
    457                              exception('peculiar error getting mtime of %s',
    458                                        fullpath)
    459                              continue
    460
    461                          if original_mtime > target_mtime:
    462                              # if original file has changed, we need to
    463                              # regenerate regardless
    464                              yield fullpath
    465                          elif self.max_age <= 0:
    466                              # max_age of zero or less means we never
    467                              # regenerate the files
    468                              continue
    469                          else:
    470                              # regenerate if the file is older than
    471                              # max_age minutes
    472                              mtime = stats.st_mtime
    473                              age = time.time() - mtime
    474                              if age > self.max_age * 60:
    475                                  yield fullpath
    476
    477
    478 -def _find_config(givenpath):
    479      for pth in (p for p in (
    480          givenpath,
    481          path.expanduser('~/.ewa.conf'),
    482          path.expanduser('~/.ewa/ewa.conf'),
    483          '/etc/ewa.conf',
    484          '/etc/ewa/ewa.conf',
    485          ) if p):
    486          if path.exists(pth):
    487              return pth
    488
    489
    490 -def do_serve(args):
    491      parser = get_serve_parser()
    492      opts, args = parser.parse_args(args)
    493      if opts.version:
    494          print VERSION_TEXT % path.basename(sys.argv[0])
    495          sys.exit(0)
    496      configfile = _find_config(opts.configfile)
    497      if not configfile:
    498          parser.error('no config file specified or found in the usual places')
    499      if not path.exists(configfile):
    500          parser.error("config file does not exist: %s" % configfile)
    501      initConfig(configfile)
    502      if opts.nodaemonize:
    503          Config.daemonize = False
    504
    505      if not Config.rulefile:
    506          parser.error("a rulefile needs to be specified")
    507      rule = FileRule(Config.rulefile)
    508      if Config.logfile:
    509          initLogging(level=Config.loglevel,
    510                      filename=Config.logfile,
    511                      rotate=Config.logrotate)
    512      # check for incompatible options
    513      if Config.protocol == 'http' and Config.unixsocket:
    514          parser.error('unix sockets not supported for http server')
    515      if Config.umask and not Config.unixsocket:
    516          parser.error('umask only applicable for unix sockets')
    517      if Config.unixsocket and (Config.interface or Config.port):
    518          parser.error('incompatible mixture of unix socket and tcp options')
    519      engine = resolve_engine(Config.engine)
    520
    521      app = EwaApp(rule=rule,
    522                   basedir=Config.basedir,
    523                   targetdir=Config.targetdir,
    524                   stream=Config.stream,
    525                   refresh_rate=Config.refresh_rate,
    526                   use_xsendfile=Config.use_xsendfile,
    527                   sendfile_header=Config.sendfile_header,
    528                   content_disposition=Config.content_disposition,
    529                   splicer=engine)
    530
    531      if opts.lighttpd_hack:
    532          app = LighttpdHackMiddleware(app)
    533
    534      if Config.protocol == 'http':
    535          runner = partial(httpserver.serve,
    536                           app,
    537                           Config.interface,
    538                           Config.port)
    539      else:
    540          if Config.interface:
    541              bindAddress = (Config.interface, Config.port)
    542              debug('bindAddress: %s', bindAddress)
    543          elif Config.unixsocket:
    544              bindAddress = Config.unixsocket
    545          if Config.protocol == 'scgi':
    546              if Config.use_threads:
    547                  serverclass = SCGIThreadServer
    548              else:
    549                  serverclass = SCGIServer
    550          elif Config.protocol == 'fcgi':
    551              if Config.use_threads:
    552                  serverclass = FCGIThreadServer
    553              else:
    554                  serverclass = FCGIServer
    555          if Config.protocol in ('fcgi', 'scgi'):
    556              if Config.use_threads:
    557                  kw = dict(maxSpare=Config.max_spare,
    558                            minSpare=Config.min_spare,
    559                            maxThreads=Config.max_threads)
    560              else:
    561                  kw = dict(maxSpare=Config.max_spare,
    562                            minSpare=Config.min_spare,
    563                            maxChildren=Config.max_children,
    564                            maxRequests=Config.max_requests)
    565              # clean out Nones
    566              kw = dict(i for i in kw.items() if not i[1] is None)
    567          else:
    568              kw = {}
    569
    570          runner = serverclass(app,
    571                               bindAddress=bindAddress,
    572                               umask=Config.umask,
    573                               **kw).run
    574
    575      try:
    576          run_server(runner,
    577                     Config.pidfile,
    578                     Config.daemonize)
    579      except SystemExit:
    580          raise
    581      except:
    582          exception('exception caught')
    583          sys.exit(1)
    584      else:
    585          sys.exit(0)
    586
    587
    588 -def _change_user_group():
    589      if not have_unix:
    590          # maybe do something else on Windows someday...
    591          return
    592
    593      user = Config.user
    594      group = Config.group
    595      if user or group:
    596          # in case we are seteuid something else, which would
    597          # cause setuid or getuid to fail, undo any existing
    598          # seteuid. (The only reason to do this is for the case
    599          # os.getuid()==0, AFAIK).
    600          try:
    601              seteuid = os.seteuid
    602          except AttributeError:
    603              # the OS may not support seteuid, in which
    604              # case everything is hotsy-totsy.
    605              pass
    606          else:
    607              seteuid(os.getuid())
    608          if group:
    609              gid = grp.getgrnam(group)[2]
    610              os.setgid(gid)
    611          if user:
    612              uid = pwd.getpwnam(user)[2]
    613              os.setuid(uid)
    614
    615
    616 -def run_server(func, pidfile=None, daemonize=True):
    617      debug("daemonize: %s, pidfile: %s", daemonize, pidfile)
    618      if daemonize:
    619          try:
    620              os.fork
    621          except NameError:
    622              info("not daemonizing, as the fork() call is not available")
    623              daemonize = False
    624      if daemonize:
    625          if os.fork():
    626              os._exit(0)
    627          if os.fork():
    628              os._exit(0)
    629          os.setsid()
    630          os.chdir('/')
    631          os.umask(0)
    632          os.open(os.devnull, os.O_RDWR)
    633          os.dup2(0, 1)
    634          os.dup2(0, 2)
    635          if pidfile:
    636              pidfp = open(pidfile, 'w')
    637              pidfp.write('%s' % os.getpid())
    638              pidfp.close()
    639
    640      try:
    641          _change_user_group()
    642          try:
    643              func()
    644          except KeyboardInterrupt:
    645              pass
    646      finally:
    647          if daemonize and pidfile:
    648              try:
    649                  os.unlink(pidfile)
    650              except OSError, e:
    651                  # if it doesn't exist, that's OK
    652                  if e.errno != errno.ENOENT:
    653                      warn("problem unlinking pid file %s", pidfile)
    654