#!/usr/bin/python # -*- coding: utf8 -*- """Vector Linux Package Builder Utility This script serves as a backend to the vpackager utility. You should never have to invoke this script manually. [But you can. Ed.] Builds a Vector Linux package from a provided source tarball. Works pretty much like a slackbuild, only it's much more flexible Credits: Vector Linux Team: M0E-lnx Uelsk8s Easuster Blusk hanumizzle Tukaani Linux Team: Larhzu Original written by M0E-lnx; this version by hanumizzle. I'm not sure who else will want this script. Public domain is probably OK because I don't need a long legalese text. """ import sys import os import shutil import tempfile import re import errno import gzip import time import textwrap import socket from optparse import OptionParser __version__ = '0.8' __newline__ = '\n' # TODO: # # Add slack-build generator (not sure how) # Fix install.sh that produces faulty symlinks (perhaps) # Where to deposit package # Add info pages into install.sh (install-info) class PackageBuilder(object): _build_profiles = { 'configure': { 'standard': '--prefix=/usr \ --sysconfdir=/etc \ --bindir=/usr/bin \ --libdir=/usr/lib \ --localstatedir=/var \ --mandir=/usr/man \ --with-included-gettext' }, 'distutils': { 'standard': 'build' } } _build_identifiers = { 'configure': 'configure', 'distutils': 'setup.py' } _default_config = { 'text': { 'build_profile': 'standard', 'custom_build_options': str(), 'pkg_arch': 'i586', 'pkg_release': '1', 'pkg_type': 'tlz', 'pkgr_id': 'vpackager', 'formatted_desc': False, 'execution_method': 'cli'}, 'methods': ['build_cflags'] } _socket_name = os.path.join(os.sep, 'tmp', 'vlpbuild-remote') _package_dir = '_package' _command_list = ['extract', 'detect', 'build', 'tweak', 'package'] _tweak_list = ( 'slack_desc', 'binaries', # usr_share tweak must come before man_pages 'usr_share', 'man_pages', 'info_pages', 'documentation', 'desktop_file', 'cruft_files') _usr_share_to_usr = ( 'doc', 'man', 'info') _pkg_cruft_files = ( '^perllocal\.pod$', '^ls-R$', '^dir$') # Lame solution _top_level_doc_files = ( 'AUTHORS', 'BUGS', 'COPYING', 'INSTALL', 'NEWS', 'README', 'TODO', 'FAQ', 'ChangeLog') _doc_cruft_files = ['^Makefile'] def __init__(self, **parameters): """Initializes a PackageBuilder object from 'parameters'. This method establishes default configuration as necessary and creates a secure temporary directory for packaging. """ # self._config represents user parameters from the command line, except # with a shorter name self._config = parameters # Set some default options self._set_default_config() # Create a private temporary directory for source compilation and # package building self._temp_dir = tempfile.mkdtemp() def __del__(self): """Deletes the temporary directory used to package the software.""" # Delete temporary directory shutil.rmtree(self._temp_dir) def _ensure_presence_of_directory(self, directory): """Ensures presence of file system directory. Creates directory path, identified by 'directory', with os.makedirs in a try/except block. That mechanism catches OSError, and only propagates the error upwards if its errno is not EEXIST. In other words, the method attempts to create the directories and silences the error that arises when those directories already exist. """ try: os.makedirs(directory) except OSError, e: if e.errno != errno.EEXIST: raise e def _cautious_system(self, command_line): """Cautiously executes a command. Executes 'command_line' with os.system and, in the event of a non-zero return code, raises OSError with the command that failed. """ if os.WEXITSTATUS(os.system(command_line)) != 0: raise OSError, "command '%s' failed" % command_line def _find_files(self, root, criteria): """Look for files matching 'criteria' under 'root'. 'criteria' must be an iterable enumerating regular expressions or callables that match basenames of desired files. If criterion is a callable, _find_files uses it as a predicate, and passes the absolute path to the file (to avoid directory changes for tests). If criterion is a regular expression, _find_files matches the basename of the file against the regular expression. That convenience behavior emulates find path -name pattern, in effect. """ found_files = [] for dir_path, dir_names, file_names in os.walk(root): # Look for files in each directory under the package dir. Use a gay # little trick to avoid clobbering the built-in type. for teh_file in file_names: absolute_path = os.path.join(dir_path, teh_file) for criterion in criteria: if callable(criterion): if criterion(absolute_path): found_files.append(absolute_path) else: if re.search(criterion, teh_file): found_files.append(absolute_path) return found_files # Credit for the name '_compressify' goes to my hero, Jesus Bush. def _compressify(self, original_file): """Compresses 'original_file' using gzip compression. The original file is removed after compression in accordance with the gzip command. """ uncompressed = open(original_file, 'rb') compressed = gzip.GzipFile(original_file + '.gz', 'wb', 9) # Thanks, crappy shutil shutil.copyfileobj(uncompressed, compressed) # Close both buffers and remove original file uncompressed.close() compressed.close() os.remove(original_file) def _set_default_config(self): """Sets default parameters. Uses defaults in _default_config where user has not supplied parameters. These are divided into the categories 'text' and 'methods'. 'text' defaults are simply copied into the _config hash if their key has no value therein. 'methods' defaults are somewhat more complex; the method locates a method '_get_default_%s' in the PackageBuilder class, where '%s' is a unique identifier for the default. Such a method is '_get_default_build_cflags'. """ for k,v in self._default_config['text'].iteritems(): self._config.setdefault(k, v) for i in self._default_config['methods']: self._config.setdefault(i, getattr(self, '_get_default_' + i)()) def _get_default_build_cflags(self): """Uses value of 'pkg_arch' in _config to determine default cflags. Specifically, the default cflags are '-O2 -march=%s -mtune=i686', where '%s' is the package architecture. """ # Assume compilation for default architecture pkg_arch = self._config['pkg_arch'] return '-O2 -march=%s -mtune=i686' % pkg_arch def execute(self): """Executes creation of package. To begin, the current umask and working directory are stored, so as not to interfere with the execution of the program following package compilation. The process occurs within a try/finally block, so that umask and working directory are restored even in the event of an unhandled exception. Packaging occurs in five stages: - extraction of source - detection of build system - building and installation of source - tweaking of installation - compilation of package """ try: old_umask = os.umask(0022) # Used in _package self._old_wd = os.getcwd() # Invoke specified execution method getattr(self, '_execute_in_' + self._config['execution_method'])() finally: # Restore old umask and working directory os.umask(old_umask) os.chdir(self._old_wd) def _execute_in_cli(self): """Executes package creation in command line mode. Very straightforward; building occurs without interruption or conditions. Contrast with vpackager-style package creation. """ for stage in self._command_list: # Invoke stage method appropriately getattr(self, '_' + stage)() def _execute_in_vpackager(self): """Executes package creation for vpackager. vlpbuild opens a socket for vpackager, which then sends commands to it. Such commands are validated against a list (to be safe, though I doubt anyone with privileges to use the socket would send '_del__'), then executed to allow vpackager to update its progress bar. """ try: receiver = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) receiver.bind(self._socket_name) receiver.listen(1) connection, address = receiver.accept() while True: # Maximum of 1024 bytes for command name seems OK...I guess method = connection.recv(1024) if not method: break # Make sure the command issued is on the list if not method in self._command_list: raise ValueError, 'method %s does not exist' % method # Try to invoke the method; send back a message according to the # success of the operation try: getattr(self, '_' + method)() # It passed connection.send('PASS') except Exception, e: # I suck, or some programmer sucks alternatively connection.send('FAIL') connection.close() # Re-raise e raise e connection.close() finally: # Always clean up the socket, and ignore any errors in the event that # vlpbuild could not create the same try: os.unlink(self._socket_name) except OSError: pass def _extract(self): """Extract source archive into temporary directory. Also determines package name and version, and creates temporary packaging root directory within the source directory. """ # Change current directory to teh temp dir created for packager os.chdir(self._temp_dir) # Extract the source archive into the package build directory self._cautious_system('tar xf %(source_archive)s' % self._config) # Assuming the source archive extracts into a single directory, the # element in os.listdir() is it: source_dir = os.listdir(os.getcwd()).pop() self._source_dir = os.path.join(self._temp_dir, source_dir) # Finagle name and version of the applications from source_dir self._finagle_name_and_version(source_dir) # Set up _package directory within the source dir self._package_dir = os.path.join(self._source_dir, '_package') os.mkdir(self._package_dir) def _finagle_name_and_version(self, source_dir): """Determines application name and version from source directory.""" regex = re.compile('^([\w-]+)-([\d.-]+)') self._app_name, self._app_version = regex.match(source_dir).groups() def _detect(self): """Detects build system according to unique files in source archive. The unique files are enumerated in the class variable _build_identifiers.""" for k,v in self._build_identifiers.iteritems(): if os.path.isfile(os.path.join(self._source_dir, v)): self._config['build_system'] = k break def _build(self): """Builds package according to its build system.""" # Change directory to source directory os.chdir(self._source_dir) # Retrieve user or default cflags cflags = self._config['build_cflags'] # The build system: configure, SCons, Makefile.PL, et al. system = self._config['build_system'] # Command line parameters for the build system from build profile profile = self._config['build_profile'] profile = self._build_profiles[system][profile] # Custom build parameters (dependent on system) custom_options = self._config['custom_build_options'] # Internal build method for the system method = getattr(self, '_build_for_' + system) # Run build backend method(cflags, profile, custom_options) def _build_for_configure(self, cflags, profile, custom_options): """Builds source with GNU autoconf.""" # Apply same settings for CFLAGS and CXXFLAGS (C++ compile flags) alike. # (Too bad hardly anyone uses Objective C...) os.putenv('CFLAGS', cflags) os.putenv('CXXFLAGS', cflags) command = './configure %s %s' % (profile, custom_options) # Strip the ugly extraneous spaces out of the command line in the event # of an error command = ' '.join(i for i in command.split(' ') if i != str()) # Store build command for later use self._build_command = command # Run configure self._cautious_system(self._build_command) # Now run make self._cautious_system('make') # Install into the _package subdirectory package_dir = self._package_dir self._cautious_system('make install DESTDIR=%s' % package_dir) def _build_for_distutils(self, cflags, profile, custom_options): """Builds source with distutils.""" # CFLAGS perhaps # Generate command build_command = 'python setup.py %s %s' % (profile, custom_options) # Store command for later use self._build_command = build_command # Run setup.py... self._cautious_system(self._build_command) # Install into the _package subdirectory install_command = 'python setup.py install --root=%s' % self._package_dir self._cautious_system(install_command) # Add a penguinporker easter egg somewhere def _tweak(self): """Executes a battery of 'tweaks' on the installation before packaging. These tweaks modify the binary installation in various ways to improve performance or conform more strongly to Slackware and Vector standards for software installation.""" for tweak in self._tweak_list: getattr(self, '_tweak_' + tweak)() def _tweak_slack_desc(self): """Generate a full slack-desc, with header, description, and data. The slack-desc may come from vpackager, in which case most of it will already exist in good form. Only package statistics remain after that; those are appended. The slack-desc may also be a simple paragraph. In this case, a full slack-desc shall be generated, neatly formatting the paragraph into the whole. """ if not self._config['formatted_desc']: # When desc_file is currently a simple paragraph self._generate_new_slack_desc() else: # desc_file came from vpackager self._modify_old_slack_desc() def _open_slack_desc(self): """Opens install/slack-desc in package dir as file object.""" # Ensure existence of install/ directory in package and open # slack-desc install_dir = os.path.join(self._package_dir, 'install') self._ensure_presence_of_directory(install_dir) slack_desc = open(os.path.join(install_dir, 'slack-desc'), 'w') return slack_desc def _get_slack_desc_margin(self): return self._app_name + ': ' def _generate_new_slack_desc(self): """Format raw paragraph into complete slack_desc.""" # 'contents' holds contents of slack-desc prior to formatting and writing # to disk. Specifically, it is a dictionary consisting of keys 'header', # 'description', and 'data'. 'description' is optional, and is wrapped. contents = {} # Generate a header like 'foobar 5.6' contents['header'] = '%s %s' % (self._app_name, self._app_version) # Try to include user supplied description try: desc_file = open(self._config['desc_file']) contents['description'] = [i.rstrip() for i in desc_file] desc_file.close() except KeyError: pass # Add some vital data to the slack-desc contents['data'] = self._get_slack_desc_data() # Write out the slack-desc self._write_new_slack_desc(contents) def _get_slack_desc_data(self): """Appends automatically determined data to slack-desc.""" data = [] # Build date format = '%a %b %e %H:%M:%S %Z %Y' localtime = time.localtime() data.append('BUILD_DATE: %s' % time.strftime(format, localtime)) # Packager ID data.append('PACKAGER: %s' % self._config['pkgr_id']) # Host (uname fields sysname, release, and machine) data.append('HOST: %s' % ' '.join(os.uname()[::2])) # Distro vector_version = open('/etc/vector-version') version_string = vector_version.readline() vector_version.close() data.append('DISTRO: %s' % version_string) # Compilation flags data.append('CFLAGS: %s' % self._config['build_cflags']) # Build command data.append('BUILD_COMMAND: %s' % self._build_command) return data def _write_new_slack_desc(self, contents): """Writes out completely formatted slack-desc.""" # Ensure existence of install/ directory in package and open # slack-desc slack_desc = self._open_slack_desc() # The margin that precedes every line in a slack-desc, the app name # followed by a colon and space. margin = self._get_slack_desc_margin() # Write teh header slack_desc.write(margin + contents['header'] + __newline__) slack_desc.write(margin + __newline__) # Write out formatted description, if available try: text = __newline__.join(contents['description']) formatted_text = textwrap.wrap(text, width=78 - len(margin)) for line in formatted_text: slack_desc.write(margin + line + __newline__) slack_desc.write(margin + __newline__) except KeyError: pass # Lastly, write out the generated data for datum in contents['data']: slack_desc.write(margin + datum + __newline__) slack_desc.close() def _modify_old_slack_desc(self): """Frobs existing slack-desc a little (adds statistics).""" # Make sure install/ exists and open slack-desc slack_desc = self._open_slack_desc() # Copy original slack-desc to the package slack-desc desc_file = open(self._config['desc_file']) shutil.copyfileobj(desc_file, slack_desc) desc_file.close() # Add statistics to the end of the install/slack-desc data = self._get_slack_desc_data() margin = self._get_slack_desc_margin() slack_desc.write(margin + __newline__) for datum in data: slack_desc.write(margin + datum + __newline__) slack_desc.close() def _tweak_binaries(self): """Strips unnecessary symbols from binaries in the package. Because some people leave -g on for compilation of finished products. grrr... """ root = self._package_dir criteria = [self._is_executable] for binary_file in self._find_files(root, criteria): # Hack introduced for absolute symlink in wxWidgets packaging, which # *appears* broken. if os.path.isfile(binary_file): self._cautious_system('strip --strip-unneeded %s' % binary_file) def _is_executable(self, teh_file): """Determines whether a file is executable.""" # Replace with a file /object/ teh_file = open(teh_file, 'rb') # Read first four bytes magic_string = teh_file.read(4) # Check the magic string against ELF constant if magic_string == '\x7fELF': return True else: return False # Move /usr/share/doc and /usr/share/man contents into /usr/doc and # /usr/man def _tweak_usr_share(self): """Moves certain files installed in /usr/share into /usr. The measure exists to conform with Slackware file system standards. """ for i in self._usr_share_to_usr: original_path = os.path.join(self._package_dir, 'usr', 'share', i) new_path = os.path.join(self._package_dir, 'usr', i) if os.path.isdir(original_path): self._ensure_presence_of_directory(new_path) for i in os.listdir(original_path): original_file = os.path.join(original_path, i) new_file = os.path.join(new_path, i) os.rename(original_file, new_file) # Remove the original path shutil.rmtree(original_path) def _tweak_man_pages(self): """Compresses uncompressed manual pages. Too many packages don't fucking compress their man pages. """ man_dir = os.path.join(self._package_dir, 'usr', 'man') for man_page in self._find_files(man_dir, ['\.\d$']): # Fix the symbolic links that would otherwise point to uncompressed # man pages that will soon cease to be if os.path.islink(man_page): # A potential bottleneck emerges here, but tests will determine # whether it is a real issue. os.chdir(os.path.dirname(man_page)) basename = os.path.basename(man_page) os.symlink(os.readlink(basename) + '.gz', basename + '.gz') else: # It's a a true man page, not a link; compress self._compressify(man_page) def _tweak_info_pages(self): """As with _tweak_man_pages, compress info pages.""" info_dir = os.path.join(self._package_dir, 'usr', 'info') for info_page in self._find_files(info_dir, ['\.info']): self._compressify(info_page) def _tweak_documentation(self): """Weakly tries to ensure some basic documentation for package. Copies top-level files in source directory, such as README and AUTHORS, into usr/doc/app_name-app_version in packaging directory. These files are enumerated in class variable _top_level_doc_files. """ pkg_name = self._app_name + '-' + self._app_version pkg_doc_dir = os.path.join(self._package_dir, 'usr', 'doc', pkg_name) # Ensure presence of documentation directory self._ensure_presence_of_directory(pkg_doc_dir) # Copy certain top-level files into documentation directory for doc_file in self._top_level_doc_files: absolute_path = os.path.join(self._source_dir, doc_file) if os.path.isfile(absolute_path): shutil.copy(absolute_path, pkg_doc_dir) # Try copying doc/ or docs/ from top-level of source directory into # documentation directory of package for i in ('doc', 'docs'): source_doc_dir = os.path.join(self._source_dir, i) if os.path.isdir(source_doc_dir): shutil.copytree(source_doc_dir, os.path.join(pkg_doc_dir, 'docs')) break # Remove some cruft from the copy. for cruft_file in self._find_files(pkg_doc_dir, self._doc_cruft_files): os.remove(cruft_file) def _tweak_desktop_file(self): """Package stray .desktop files in the source directory. Copy .desktop file(s) under the source directory into usr/share/applications under the package directory; create the same directory if necessary. """ func = os.path.join desktop_dir = func(self._package_dir, 'usr', 'share', 'applications') desktop_files = self._find_files(self._source_dir, ['\.desktop$']) for desktop_file in desktop_files: # Ignore .desktop files present in package directory function = os.path.commonprefix common_prefix = function((self._package_dir, desktop_file)) if common_prefix == self._package_dir: continue # Otherwise, make sure /usr/share/applications exists and move the # .desktop file into it. self._ensure_presence_of_directory(desktop_dir) shutil.move(desktop_file, desktop_dir) # To wit, remove them def _tweak_cruft_files(self): """Remove 'cruft' files from the package. The cruft file patterns are listed in _pkg_cruft_file and are automatically generated directories whose presence is undesired in a completed package. """ root = self._package_dir criteria = self._pkg_cruft_files for cruft_file in self._find_files(root, criteria): os.remove(cruft_file) def _package(self): """Compile a usable binary package. Automatically generates a package name from the original source file, as well as user parameters. It uses sane defaults where necessary. In particular, the default compression method is LZMA. """ # Change directory to package dir os.chdir(self._package_dir) # Run makeslapt to produce a tlz (default) or tgz package pkg_type = self._config['pkg_type'] pkg_name = self._get_pkg_name() self._cautious_system('/sbin/makeslapt --%s %s' % (pkg_type, pkg_name)) # Move package to original current directory shutil.move(pkg_name, self._old_wd) def _get_pkg_name(self): """Return a package name. Concatenates application name and version, inferred from source archive name, with package architecture and release number to create the basename, then adds extension according to compression type. The resulting name may resemble: foobar-2.6-i586-1.tlz """ parts = [] parts.append(self._app_name) parts.append(self._app_version) parts.append(self._config['pkg_arch']) parts.append(self._config['pkg_release']) pkg_basename = '-'.join(parts) pkg_type = self._config['pkg_type'] return pkg_basename + '.' + pkg_type def main(args): """Executes PackageBuilder instance via command-line.""" option_template = { 'desc_file': {'short': 'd'}, 'build_system': {'short': 's'}, 'build_profile': {'short': 'p'}, 'custom_build_options': {'short': 'o'}, 'pkg_arch': {'short': 'a'}, 'pkg_release': {'short': 'r'}, 'pkg_type': {'short': 't'}, 'pkgr_id': {'short': 'i'}, 'build_cflags': {'short': 'c'}, 'execution_method': {'short': 'e' }, 'formatted_desc': {'short': 'f', 'action': 'store_true'} } usage = 'usage: %prog [options] source_archive' version = '%%prog %s' % __version__ option_parser = OptionParser(usage=usage, version=version) for k,v in option_template.iteritems(): short_option = '-' + v['short'] # Change hyphens to underscores; I worry not over the expense here long_option = '--' + k.replace('_', '-') # The action is 'store', by default try: action = v['action'] except KeyError: action = 'store' # Add the new option to the parser instance option_parser.add_option(short_option, long_option, action=action, dest=k) # Collect parameters from the command line options, args = option_parser.parse_args(args) # Parameters is a hash that represents both options and positional # parameters parameters = {} # Add all options to the parameters hash that are not None; note the # subtle difference between that test and a simple truth test. for k in option_template.iterkeys(): option = getattr(options, k) if option is not None: parameters[k] = option # The single positional parameter for now is the source archive. try: parameters['source_archive'] = args.pop() except IndexError: raise LookupError, 'source archive not given to script' # Make sure paths to file parameters are absolute try: for k in ['source_archive', 'desc_file']: parameters[k] = os.path.abspath(parameters[k]) except KeyError: pass # Create the package builder and execute package creation package_builder = PackageBuilder(**parameters) package_builder.execute() if __name__ == '__main__': main(sys.argv[1:])