translate.storage.xpi

Package translate :: Package storage :: Module xpi |
[hide private]
[frames] | no frames]

# Source Code for Module translate.storage.xpi

      1  #!/usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3  #
      4  # Copyright 2004, 2005 Zuza Software Foundation
      5  #
      6  # This file is part of translate.
      7  #
      8  # translate is free software; you can redistribute it and/or modify
      9  # it under the terms of the GNU General Public License as published by
     10  # the Free Software Foundation; either version 2 of the License, or
     11  # (at your option) any later version.
     12  #
     13  # translate is distributed in the hope that it will be useful,
     14  # but WITHOUT ANY WARRANTY; without even the implied warranty of
     15  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     16  # GNU General Public License for more details.
     17  #
     18  # You should have received a copy of the GNU General Public License
     19  # along with translate; if not, write to the Free Software
     20  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
     21
     22  """module for accessing mozilla xpi packages"""
     23
     24  from __future__ import generators
     25  import zipfile
     26  import os.path
     27  from translate import __version__
     28  import StringIO
     29  import re
     30
     31  # we have some enhancements to zipfile in a file called zipfileext
     32  # hopefully they will be included in a future version of python
     33  from translate.misc import zipfileext
     34  ZipFileBase = zipfileext.ZipFileExt
     35
     36  from translate.misc import wStringIO
     37  # this is a fix to the StringIO in Python 2.3.3
     38  # submitted as patch 951915 on sourceforge
     39 -class FixedStringIO(wStringIO.StringIO):
     40 -    def truncate(self, size=None):
     41          StringIO.StringIO.truncate(self, size)
     42          self.len = len(self.buf)
     43
     44  NamedStringInput = wStringIO.StringIO
     45  NamedStringOutput = wStringIO.StringIO
     46
     47 -def _commonprefix(itemlist):
     48      def cp(a, b):
     49          l = min(len(a), len(b))
     50          for n in range(l):
     51              if a[n] != b[n]: return a[:n]
     52          return a[:l]
     53      if itemlist:
     54          return reduce(cp, itemlist)
     55      else:
     56          return ''
     57
     58 -def rememberchanged(self, method):
     59      def changed(*args, **kwargs):
     60          self.changed = True
     61          method(*args, **kwargs)
     62      return changed
     63
     64 -class CatchPotentialOutput(NamedStringInput, object):
     65      """catches output if there has been, before closing"""
     66 -    def __init__(self, contents, onclose):
     67          """Set up the output stream, and remember a method to call on closing"""
     68          NamedStringInput.__init__(self, contents)
     69          self.onclose = onclose
     70          self.changed = False
     71          s = super(CatchPotentialOutput, self)
     72          self.write = rememberchanged(self, s.write)
     73          self.writelines = rememberchanged(self, s.writelines)
     74          self.truncate = rememberchanged(self, s.truncate)
     75
     76 -    def close(self):
     77          """wrap the underlying close method, to pass the value to onclose before it goes"""
     78          if self.changed:
     79              value = self.getvalue()
     80              self.onclose(value)
     81          NamedStringInput.close(self)
     82
     83 -    def flush(self):
     84          """zip files call flush, not close, on file-like objects"""
     85          value = self.getvalue()
     86          self.onclose(value)
     87          NamedStringInput.flush(self)
     88
     89 -    def slam(self):
     90          """use this method to force the closing of the stream if it isn't closed yet"""
     91          if not self.closed:
     92              self.close()
     93
     94 -class ZipFileCatcher(ZipFileBase, object):
     95      """a ZipFile that calls any methods its instructed to before closing (useful for catching stream output)"""
     96 -    def __init__(self, *args, **kwargs):
     97          """initialize the ZipFileCatcher"""
     98          # storing oldclose as attribute, since if close is called from __del__ it has no access to external variables
     99          self.oldclose = super(ZipFileCatcher, self).close
    100          super(ZipFileCatcher, self).__init__(*args, **kwargs)
    101
    102 -    def addcatcher(self, pendingsave):
    103          """remember to call the given method before closing"""
    104          if hasattr(self, "pendingsaves"):
    105              if not pendingsave in self.pendingsaves:
    106                  self.pendingsaves.append(pendingsave)
    107          else:
    108              self.pendingsaves = [pendingsave]
    109
    110 -    def close(self):
    111          """close the stream, remembering to call any addcatcher methods first"""
    112          if hasattr(self, "pendingsaves"):
    113              for pendingsave in self.pendingsaves:
    114                  pendingsave()
    115          # if close is called from __del__, it somehow can't see ZipFileCatcher, so we've cached oldclose...
    116          if ZipFileCatcher is None:
    117              self.oldclose()
    118          else:
    119              super(ZipFileCatcher, self).close()
    120
    121 -    def overwritestr(self, zinfo_or_arcname, bytes):
    122          """writes the string into the archive, overwriting the file if it exists..."""
    123          if isinstance(zinfo_or_arcname, zipfile.ZipInfo):
    124              filename = zinfo_or_arcname.filename
    125          else:
    126              filename = zinfo_or_arcname
    127          if filename in self.NameToInfo:
    128              self.delete(filename)
    129          self.writestr(zinfo_or_arcname, bytes)
    130          self.writeendrec()
    131
    132 -class XpiFile(ZipFileCatcher):
    133 -    def __init__(self, *args, **kwargs):
    134          """sets up the xpi file"""
    135          self.includenonloc = kwargs.get("includenonloc", True)
    136          if "includenonloc" in kwargs:
    137              del kwargs["includenonloc"]
    138          if "compression" not in kwargs:
    139              kwargs["compression"] = zipfile.ZIP_DEFLATED
    140          self.locale = kwargs.pop("locale", None)
    141          self.region = kwargs.pop("region", None)
    142          super(XpiFile, self).__init__(*args, **kwargs)
    143          self.jarfiles = {}
    144          self.findlangreg()
    145          self.jarprefixes = self.findjarprefixes()
    146          self.reverseprefixes = dict([
    147              (prefix,jarfilename) for jarfilename, prefix in self.jarprefixes.iteritems() if prefix])
    148          self.reverseprefixes["package/"] = None
    149
    150 -    def iterjars(self):
    151          """iterate through the jar files in the xpi as ZipFile objects"""
    152          for filename in self.namelist():
    153              if filename.lower().endswith('.jar'):
    154                  if filename not in self.jarfiles:
    155                      jarstream = self.openinputstream(None, filename)
    156                      jarfile = ZipFileCatcher(jarstream, mode=self.mode)
    157                      self.jarfiles[filename] = jarfile
    158                  else:
    159                      jarfile = self.jarfiles[filename]
    160                  yield filename, jarfile
    161
    162 -    def islocfile(self, filename):
    163          """returns whether the given file is needed for localization (basically .dtd and .properties)"""
    164          base, ext = os.path.splitext(filename)
    165          return ext in (os.extsep + "dtd", os.extsep + "properties")
    166
    167 -    def findlangreg(self):
    168          """finds the common prefix of all the files stored in the jar files"""
    169          dirstructure = {}
    170          locale = self.locale
    171          region = self.region
    172          localematch = re.compile("^[a-z]{2,3}(-[a-zA-Z]{2,3}|)$")
    173          regionmatch = re.compile("^[a-zA-Z]{2,3}$")
    174          # exclude en-mac, en-win, en-unix for seamonkey
    175          osmatch = re.compile("^[a-z]{2,3}-(mac|unix|win)$")
    176          for jarfilename, jarfile in self.iterjars():
    177              jarname = "".join(jarfilename.split('/')[-1:]).replace(".jar", "", 1)
    178              if localematch.match(jarname) and not osmatch.match(jarname):
    179                  if locale is None:
    180                      locale = jarname
    181                  elif locale != jarname:
    182                      locale = 0
    183              elif regionmatch.match(jarname):
    184                  if region is None:
    185                      region = jarname
    186                  elif region != jarname:
    187                      region = 0
    188              for filename in jarfile.namelist():
    189                  if filename.endswith('/'): continue
    190                  if not self.islocfile(filename) and not self.includenonloc: continue
    191                  parts = filename.split('/')[:-1]
    192                  treepoint = dirstructure
    193                  for partnum in range(len(parts)):
    194                      part = parts[partnum]
    195                      if part in treepoint:
    196                          treepoint = treepoint[part]
    197                      else:
    198                          treepoint[part] = {}
    199                          treepoint = treepoint[part]
    200          localeentries = {}
    201          if 'locale' in dirstructure:
    202              for dirname in dirstructure['locale']:
    203                  localeentries[dirname] = 1
    204                  if localematch.match(dirname) and not osmatch.match(dirname):
    205                      if locale is None:
    206                          locale = dirname
    207                      elif locale != dirname:
    208                          print "locale dir mismatch - ", dirname, "but locale is", locale, "setting to 0"
    209                          locale = 0
    210                  elif regionmatch.match(dirname):
    211                      if region is None:
    212                          region = dirname
    213                      elif region != dirname:
    214                          region = 0
    215          if locale and locale in localeentries:
    216              del localeentries[locale]
    217          if region and region in localeentries:
    218              del localeentries[region]
    219          if locale and not region:
    220              if "-" in locale:
    221                  region = locale.split("-", 1)[1]
    222              else:
    223                  region = ""
    224          self.setlangreg(locale, region)
    225
    226 -    def setlangreg(self, locale, region):
    227          """set the locale and region of this xpi"""
    228          if locale == 0 or locale is None:
    229              raise ValueError("unable to determine locale")
    230          self.locale = locale
    231          self.region = region
    232          self.dirmap = {}
    233          if self.locale is not None:
    234              self.dirmap[('locale', self.locale)] = ('lang-reg',)
    235          if self.region:
    236              self.dirmap[('locale', self.region)] = ('reg',)
    237
    238 -    def findjarprefixes(self):
    239          """checks the uniqueness of the jar files contents"""
    240          uniquenames = {}
    241          jarprefixes = {}
    242          for jarfilename, jarfile in self.iterjars():
    243              jarprefixes[jarfilename] = ""
    244              for filename in jarfile.namelist():
    245                  if filename.endswith('/'): continue
    246                  if filename in uniquenames:
    247                      jarprefixes[jarfilename] = True
    248                      jarprefixes[uniquenames[filename]] = True
    249                  else:
    250                      uniquenames[filename] = jarfilename
    251          for jarfilename, hasconflicts in jarprefixes.items():
    252              if hasconflicts:
    253                  shortjarfilename = os.path.split(jarfilename)[1]
    254                  shortjarfilename = os.path.splitext(shortjarfilename)[0]
    255                  jarprefixes[jarfilename] = shortjarfilename+'/'
    256          # this is a clever trick that will e.g. remove zu- from zu-win, zu-mac, zu-unix
    257          commonjarprefix = _commonprefix([prefix for prefix in jarprefixes.itervalues() if prefix])
    258          if commonjarprefix:
    259              for jarfilename, prefix in jarprefixes.items():
    260                  if prefix:
    261                      jarprefixes[jarfilename] = prefix.replace(commonjarprefix, '', 1)
    262          return jarprefixes
    263
    264 -    def ziptoospath(self, zippath):
    265          """converts a zipfile filepath to an os-style filepath"""
    266          return os.path.join(*zippath.split('/'))
    267
    268 -    def ostozippath(self, ospath):
    269          """converts an os-style filepath to a zipfile filepath"""
    270          return '/'.join(ospath.split(os.sep))
    271
    272 -    def mapfilename(self, filename):
    273          """uses a map to simplify the directory structure"""
    274          parts = tuple(filename.split('/'))
    275          possiblematch = None
    276          for prefix, mapto in self.dirmap.iteritems():
    277              if parts[:len(prefix)] == prefix:
    278                  if possiblematch is None or len(possiblematch[0]) < len(prefix):
    279                      possiblematch = prefix, mapto
    280          if possiblematch is not None:
    281              prefix, mapto = possiblematch
    282              mapped = mapto + parts[len(prefix):]
    283              return '/'.join(mapped)
    284          return filename
    285
    286 -    def mapxpifilename(self, filename):
    287          """uses a map to rename files that occur straight in the xpi"""
    288          if filename.startswith('bin/chrome/') and filename.endswith(".manifest"):
    289              return 'bin/chrome/lang-reg.manifest'
    290          return filename
    291
    292 -    def reversemapfile(self, filename):
    293          """unmaps the filename..."""
    294          possiblematch = None
    295          parts = tuple(filename.split('/'))
    296          for prefix, mapto in self.dirmap.iteritems():
    297              if parts[:len(mapto)] == mapto:
    298                  if possiblematch is None or len(possiblematch[0]) < len(mapto):
    299                      possiblematch = (mapto, prefix)
    300          if possiblematch is None:
    301              return filename
    302          mapto, prefix = possiblematch
    303          reversemapped = prefix + parts[len(mapto):]
    304          return '/'.join(reversemapped)
    305
    306 -    def reversemapxpifilename(self, filename):
    307          """uses a map to rename files that occur straight in the xpi"""
    308          if filename == 'bin/chrome/lang-reg.manifest':
    309              if self.locale:
    310                  return '/'.join(('bin', 'chrome', self.locale + '.manifest'))
    311              else:
    312                  for otherfilename in self.namelist():
    313                      if otherfilename.startswith("bin/chrome/") and otherfilename.endswith(".manifest"):
    314                          return otherfilename
    315          return filename
    316
    317 -    def jartoospath(self, jarfilename, filename):
    318          """converts a filename from within a jarfile to an os-style filepath"""
    319          if jarfilename:
    320              jarprefix = self.jarprefixes[jarfilename]
    321              return self.ziptoospath(jarprefix+self.mapfilename(filename))
    322          else:
    323              return self.ziptoospath(os.path.join("package", self.mapxpifilename(filename)))
    324
    325 -    def ostojarpath(self, ospath):
    326          """converts an extracted os-style filepath to a jarfilename and filename"""
    327          zipparts = ospath.split(os.sep)
    328          prefix = zipparts[0] + '/'
    329          if prefix in self.reverseprefixes:
    330              jarfilename = self.reverseprefixes[prefix]
    331              filename = self.reversemapfile('/'.join(zipparts[1:]))
    332              if jarfilename is None:
    333                  filename = self.reversemapxpifilename(filename)
    334              return jarfilename, filename
    335          else:
    336              filename = self.ostozippath(ospath)
    337              if filename in self.namelist():
    338                  return None, filename
    339              filename = self.reversemapfile('/'.join(zipparts))
    340              possiblejarfilenames = [jarfilename for jarfilename, prefix in self.jarprefixes.iteritems() if not prefix]
    341              for jarfilename in possiblejarfilenames:
    342                  jarfile = self.jarfiles[jarfilename]
    343                  if filename in jarfile.namelist():
    344                      return jarfilename, filename
    345              raise IndexError("ospath not found in xpi file, could not guess location: %r" % ospath)
    346
    347 -    def jarfileexists(self, jarfilename, filename):
    348          """checks whether the given file exists inside the xpi"""
    349          if jarfilename is None:
    350              return filename in self.namelist()
    351          else:
    352              jarfile = self.jarfiles[jarfilename]
    353              return filename in jarfile.namelist()
    354
    355 -    def ospathexists(self, ospath):
    356          """checks whether the given file exists inside the xpi"""
    357          jarfilename, filename = self.ostojarpath(ospath)
    358          if jarfilename is None:
    359              return filename in self.namelist()
    360          else:
    361              jarfile = self.jarfiles[jarfilename]
    362              return filename in jarfile.namelist()
    363
    364 -    def openinputstream(self, jarfilename, filename):
    365          """opens a file (possibly inside a jarfile as a StringIO"""
    366          if jarfilename is None:
    367              contents = self.read(filename)
    368              def onclose(contents):
    369                  if contents != self.read(filename):
    370                      self.overwritestr(filename, contents)
    371              inputstream = CatchPotentialOutput(contents, onclose)
    372              self.addcatcher(inputstream.slam)
    373          else:
    374              jarfile = self.jarfiles[jarfilename]
    375              contents = jarfile.read(filename)
    376              inputstream = NamedStringInput(contents)
    377          inputstream.name = self.jartoospath(jarfilename, filename)
    378          if hasattr(self.fp, 'name'):
    379              inputstream.name = "%s:%s" % (self.fp.name, inputstream.name)
    380          return inputstream
    381
    382 -    def openoutputstream(self, jarfilename, filename):
    383          """opens a file for writing (possibly inside a jarfile as a StringIO"""
    384          if jarfilename is None:
    385              def onclose(contents):
    386                  self.overwritestr(filename, contents)
    387          else:
    388              if jarfilename in self.jarfiles:
    389                  jarfile = self.jarfiles[jarfilename]
    390              else:
    391                  jarstream = self.openoutputstream(None, jarfilename)
    392                  jarfile = ZipFileCatcher(jarstream, "w")
    393                  self.jarfiles[jarfilename] = jarfile
    394                  self.addcatcher(jarstream.slam)
    395              def onclose(contents):
    396                  jarfile.overwritestr(filename, contents)
    397          outputstream = wStringIO.CatchStringOutput(onclose)
    398          outputstream.name = "%s %s" % (jarfilename, filename)
    399          if jarfilename is None:
    400              self.addcatcher(outputstream.slam)
    401          else:
    402              jarfile.addcatcher(outputstream.slam)
    403          return outputstream
    404
    405 -    def close(self):
    406          """Close the file, and for mode "w" and "a" write the ending records."""
    407          for jarfile in self.jarfiles.itervalues():
    408              jarfile.close()
    409          super(XpiFile, self).close()
    410
    411 -    def testzip(self):
    412          """test the xpi zipfile and all enclosed jar files..."""
    413          for jarfile in self.jarfiles.itervalues():
    414              jarfile.testzip()
    415          super(XpiFile, self).testzip()
    416
    417 -    def restructurejar(self, origjarfilename, newjarfilename, otherxpi, newlang, newregion):
    418          """Create a new .jar file with the same contents as the given name, but rename directories, write to outputstream"""
    419          jarfile = self.jarfiles[origjarfilename]
    420          origlang = self.locale[:self.locale.find("-")]
    421          if newregion:
    422              newlocale = "%s-%s" % (newlang, newregion)
    423          else:
    424              newlocale = newlang
    425          for filename in jarfile.namelist():
    426              filenameparts = filename.split("/")
    427              for i in range(len(filenameparts)):
    428                  part = filenameparts[i]
    429                  if part == origlang:
    430                      filenameparts[i] = newlang
    431                  elif part == self.locale:
    432                      filenameparts[i] = newlocale
    433                  elif part == self.region:
    434                      filenameparts[i] = newregion
    435              newfilename = '/'.join(filenameparts)
    436              fileoutputstream = otherxpi.openoutputstream(newjarfilename, newfilename)
    437              fileinputstream = self.openinputstream(origjarfilename, filename)
    438              fileoutputstream.write(fileinputstream.read())
    439              fileinputstream.close()
    440              fileoutputstream.close()
    441
    442 -    def clone(self, newfilename, newmode=None, newlang=None, newregion=None):
    443          """Create a new .xpi file with the same contents as this one..."""
    444          other = XpiFile(newfilename, "w", locale=newlang, region=newregion)
    445          origlang = self.locale[:self.locale.find("-")]
    446          # TODO: check if this language replacement code is still neccessary
    447          if newlang is None:
    448              newlang = origlang
    449          if newregion is None:
    450              newregion = self.region
    451          if newregion:
    452              newlocale = "%s-%s" % (newlang, newregion)
    453          else:
    454              newlocale = newlang
    455          for filename in self.namelist():
    456              filenameparts = filename.split('/')
    457              basename = filenameparts[-1]
    458              if basename.startswith(self.locale):
    459                  newbasename = basename.replace(self.locale, newlocale)
    460              elif basename.startswith(origlang):
    461                  newbasename = basename.replace(origlang, newlang)
    462              elif basename.startswith(self.region):
    463                  newbasename = basename.replace(self.region, newregion)
    464              else:
    465                  newbasename = basename
    466              if newbasename != basename:
    467                  filenameparts[-1] = newbasename
    468                  renamefilename = "/".join(filenameparts)
    469                  print "cloning", filename, "and renaming to", renamefilename
    470              else:
    471                  print "cloning", filename
    472                  renamefilename = filename
    473              if filename.lower().endswith(".jar"):
    474                  self.restructurejar(filename, renamefilename, other, newlang, newregion)
    475              else:
    476                  inputstream = self.openinputstream(None, filename)
    477                  outputstream = other.openoutputstream(None, renamefilename)
    478                  outputstream.write(inputstream.read())
    479                  inputstream.close()
    480                  outputstream.close()
    481          other.close()
    482          if newmode is None: newmode = self.mode
    483          if newmode == "w": newmode = "a"
    484          other = XpiFile(newfilename, newmode)
    485          other.setlangreg(newlocale, newregion)
    486          return other
    487
    488 -    def iterextractnames(self, includenonjars=False, includedirs=False):
    489          """iterates through all the localization files with the common prefix stripped and a jarfile name added if neccessary"""
    490          if includenonjars:
    491              for filename in self.namelist():
    492                  if filename.endswith('/') and not includedirs: continue
    493                  if not self.islocfile(filename) and not self.includenonloc: continue
    494                  if not filename.lower().endswith(".jar"):
    495                      yield self.jartoospath(None, filename)
    496          for jarfilename, jarfile in self.iterjars():
    497              for filename in jarfile.namelist():
    498                  if filename.endswith('/'):
    499                      if not includedirs: continue
    500                  if not self.islocfile(filename) and not self.includenonloc: continue
    501                  yield self.jartoospath(jarfilename, filename)
    502
    503      # the following methods are required by translate.convert.ArchiveConvertOptionParser #
    504 -    def __iter__(self):
    505          """iterates through all the files. this is the method use by the converters"""
    506          for inputpath in self.iterextractnames(includenonjars=True):
    507              yield inputpath
    508
    509 -    def __contains__(self, fullpath):
    510          """returns whether the given pathname exists in the archive"""
    511          try:
    512              jarfilename, filename = self.ostojarpath(fullpath)
    513          except IndexError:
    514              return False
    515          return self.jarfileexists(jarfilename, filename)
    516
    517 -    def openinputfile(self, fullpath):
    518          """opens an input file given the full pathname"""
    519          jarfilename, filename = self.ostojarpath(fullpath)
    520          return self.openinputstream(jarfilename, filename)
    521
    522 -    def openoutputfile(self, fullpath):
    523          """opens an output file given the full pathname"""
    524          try:
    525              jarfilename, filename = self.ostojarpath(fullpath)
    526          except IndexError:
    527              return None
    528          return self.openoutputstream(jarfilename, filename)
    529
    530  if __name__ == '__main__':
    531      import optparse
    532      optparser = optparse.OptionParser(version="%prog "+__version__.sver)
    533      optparser.usage = "%prog [-l|-x] [options] file.xpi"
    534      optparser.add_option("-l", "--list", help="list files", \
    535          action="store_true", dest="listfiles", default=False)
    536      optparser.add_option("-p", "--prefix", help="show common prefix", \
    537          action="store_true", dest="showprefix", default=False)
    538      optparser.add_option("-x", "--extract", help="extract files", \
    539          action="store_true", dest="extractfiles", default=False)
    540      optparser.add_option("-d", "--extractdir", help="extract into EXTRACTDIR", \
    541          default=".", metavar="EXTRACTDIR")
    542      (options, args) = optparser.parse_args()
    543      if len(args) < 1:
    544          optparser.error("need at least one argument")
    545      xpifile = XpiFile(args[0])
    546      if options.showprefix:
    547          for prefix, mapto in xpifile.dirmap.iteritems():
    548              print "/".join(prefix), "->", "/".join(mapto)
    549      if options.listfiles:
    550          for name in xpifile.iterextractnames(includenonjars=True, includedirs=True):
    551              print name #, xpifile.ostojarpath(name)
    552      if options.extractfiles:
    553          if options.extractdir and not os.path.isdir(options.extractdir):
    554              os.mkdir(options.extractdir)
    555          for name in xpifile.iterextractnames(includenonjars=True, includedirs=False):
    556              abspath = os.path.join(options.extractdir, name)
    557              # check neccessary directories exist - this way we don't create empty directories
    558              currentpath = options.extractdir
    559              subparts = os.path.dirname(name).split(os.sep)
    560              for part in subparts:
    561                  currentpath = os.path.join(currentpath, part)
    562                  if not os.path.isdir(currentpath):
    563                      os.mkdir(currentpath)
    564              outputstream = open(abspath, 'w')
    565              jarfilename, filename = xpifile.ostojarpath(name)
    566              inputstream = xpifile.openinputstream(jarfilename, filename)
    567              outputstream.write(inputstream.read())
    568              outputstream.close()
    569