MakeStaticPackage.py Source Code

--D. Thiebaut (talk) 09:26, 23 February 2015 (EST)

---

This Python code is a utility program I wrote to help me generate some particular type of applications, and is reproduced here as an example of the use of functions. In particular, note how some functions are fairly short, while others longer. In general most are less than the equivalent of a screen height, so that you can easily read the whole function at without sliding the code up or down, making it easy to fully understand what the function does. Functions are mini programs, in a way, and have their own logic, their own entry point, and their own exit point(s).

#! /usr/bin/env python
# makeStaticPackage.py
# (C) D. Thiebaut
#
# This program runs under Mac OS X.

# This program takes the executable generated by Qt3 (but would work
# very likely with other compiled output) that relies on Qt3 specific
# libraries that may not be available on the clients of an XGrid system.
#
# The program should be run in an empty directory.  It is important as
# it will copy in this directory the executable and all the libraries it
# depends on and will modify all of them.  So, make sure you start in
# an empty directory.  All you need to know is the path to the executable
# you want to deploy on the XGrid.
#
# This program is given the path of an executable that should be somewhere
# but NOT IN THE CURRENT DIRECTORY:
#
#         makeStaticPackage.py  ~/bin/filterwiki9 
#
# The algorithm is the following:
#  1) copy the executable in the current directory
#  2) extract the list of libraries used by the executable using otool
#  3) keep the names of the libraries that reside in /opt/local/lib
#     as they very likely won't be on the XGrid clients.
#  4) modify the executable and replace the full path reference to these
#     /opt/local/lib libraries and use @executable_path instead.  This
#     way the library can be distributed with the executable and run from
#     the working directory.
#  5) copy the /opt/local/lib libraries to the current directory, and
#     apply steps 1 to 4 above to each one.  Keep on going until the
#     graph of all libraries relating to /opt/local/lib is complete.
#
# Here is an example of the executable filterwiki9 before it is processed
#
#   otool -L filterwiki9
#   filterwiki9:
#	/opt/local/lib/libqt-mt.3.dylib (compatibility version 3.3.0, current version 3.3.8)
#	/opt/local/lib/libXext.6.dylib (compatibility version 11.0.0, current version 11.0.0)
#	/opt/local/lib/libX11.6.dylib (compatibility version 10.0.0, current version 10.0.0)
#	/usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 111.1.5)
#	/usr/lib/libstdc++.6.dylib (compatibility version 7.0.0, current version 7.4.0)
#	/usr/lib/libgcc_s.1.dylib (compatibility version 1.0.0, current version 1.0.0)
#
# and after processing:
#
#	@executable_path/libqt-mt.3.dylib (compatibility version 3.3.0, current version 3.3.8)
#	@executable_path/libXext.6.dylib (compatibility version 11.0.0, current version 11.0.0)
#	@executable_path/libX11.6.dylib (compatibility version 10.0.0, current version 10.0.0)
#	/usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 111.1.5)
#	/usr/lib/libstdc++.6.dylib (compatibility version 7.0.0, current version 7.4.0)
#	/usr/lib/libgcc_s.1.dylib (compatibility version 1.0.0, current version 1.0.0)
#
# Note that only the entries for /opt/local/lib libraries are changed.
# 
# Versions:
# Version 3: 6/10/2010 Can now process several executables entered on the command line
# 
# Version 2: 6/9/2010  Simplified it and remove the processing of data files.
#                      Outputs a list of the files that were brought in.
#
VERSION = "V3 (6/10/2010)"

import sys
import os
import subprocess

copiedLibs = {}


copiedDico = {}    # the dico of all copied files
processedDico = {} # the dico of all processed libraries
                   # a processed library is one that has been
                   # copied and whose library entries have ahd
                   # their path changed to local

def baseNameOf( fullPath ):
    return fullPath.split( '/' )[-1]

def safeCopyHere( fullPath, Debug=False ):
    """copies file from remote library dir to current dir.
    and updates dictionary of copied files"""

    baseName = baseNameOf( fullPath )
    #--- do not copy if we already have it (as it might have been processed ---
    #--- already!)                                                          ---
    if baseName in copiedDico:
        if Debug: print "safeCopyHere: file %s already copied. Skipping..." % baseName
        return

    #--- bring into current directory ---
    try:
        subprocess.check_call( ["cp", fullPath, "." ] )
    except:
        print "could not copy %s to current directory" % fullPath
        sys.exit( 1 )

    #--- make it writable (some are not by default) ---
    try:
        subprocess.check_call( ["chmod", "+w", baseName ] )
    except:
        print "could not chmod +x %s" % baseName
        sys.exit( 1 )

    #--- record it as copied! ---
    copiedDico[ fullPath ] = True
    if Debug: print "safeCopyHere: copied file %s from %s" % ( baseName, fullPath )


def changePathsToLocal( fullPath, Debug=False ):
    """takes the library or executable specified by fullPath and marks them
    as local in the code of fullPath.  Returns the list of libraries found
    as a list of full paths"""

    if Debug:
        print "\n--------------------------------------------------------"
        print "changePathsToLocal( %s )" % fullPath
        print "--------------------------------------------------------"
        
    baseName = baseNameOf( fullPath )

    if not fullPath in copiedDico:
        print "*** Error: changePathsToLocal cannot process %s: hasn't been copied yet ***" % baseName
        sys.exit( 1 )

    if fullPath in processedDico:
        if Debug: print "changePathsToLocal: %s already processed.  Skipping" % baseName
        return []
    
    #--- get libraries ---
    output = subprocess.Popen(["otool", "-L",  baseName ], 
                              stdout=subprocess.PIPE).communicate()[0]

    if Debug:
        print "changePathsToLocal: otool -L", baseName
        print "changePathsToLocal:", output
        
    #--- extract library names ---
    firstLevelLibs  = []
    for i, line in enumerate( output.split( '\n' ) ):
        line = line.strip()
        if line.find( "/opt/local" ) != -1:
            fullPathLib = line.split( ' ' )[0] 
            firstLevelLibs.append( fullPathLib )

    #--- change library path in fullPath to local ---
    for fullPathLib in firstLevelLibs:
        baseNameLib = baseNameOf( fullPathLib )
        if Debug: print "changePathsToLocal: changing path of %s in %s," % (baseNameLib, baseName)

        if baseNameLib == baseName :
            output = subprocess.Popen( [ "install_name_tool", 
                                         "-id",
                                         "@executable_path/" + baseNameLib,
                                         baseName ],
                                       stdout=subprocess.PIPE).communicate()[0] 
            
        else:
            output = subprocess.Popen( [ "install_name_tool", 
                                         "-change",
                                         fullPathLib,
                                         "@executable_path/" + baseNameLib,
                                         baseName ],
                                       stdout=subprocess.PIPE).communicate()[0]
            
    #--- record this entry as processed! --
    processedDico[ fullPath ] = True        
    if Debug:
        output = subprocess.Popen(["otool", "-L",  baseName ], 
                                  stdout=subprocess.PIPE).communicate()[0]
        print "changePathsToLocal: otool -L", baseName
        print "changePathsToLocal:", output
        print "changePathsToLocal finished processing %s" % baseName
        print "--------------------------------------------------------\n"
        
    return firstLevelLibs

def process( executable, Debug=False ):
    """figures out what libraries executable relies on that are not system libraries,
    and are in /opt/local, copies them to the local directory and marks them as
    local in the code of executable."""

    #--- in case the executable comes from far away ---
    safeCopyHere( executable, Debug )
    baseExecutable = baseNameOf( executable )
    
    #--- we assume executable is the code file compiled by Qt and the starting point  ---
    #--- of the process.  It does not need to be copied.                              ---
    fullPathLibs = changePathsToLocal( executable, Debug )

    #--- assumption: fullPathLibs is a list containing libraries that must be brought in ---
    #--- and processed.  If a new library is found and 
    while len( fullPathLibs )>0:
        fullPath = fullPathLibs.pop()
        if Debug: print "<--- Dequeueing ", fullPath
        baseName = baseNameOf( fullPath )
        safeCopyHere( fullPath, Debug )
        newLibsFound = changePathsToLocal( fullPath, Debug )
        for lib in newLibsFound:
            if not lib in processedDico and not lib in fullPathLibs:
                fullPathLibs.insert( 0, lib )
                if Debug: print "---> Enqueueing ", lib


# ---------------------------------------------------------------------------------------------
#                     
#                                __  __      _      ___   _   _ 
#                               |  \/  |    / \    |_ _| | \ | |
#                               | |\/| |   / _ \    | |  |  \| |
#                               | |  | |  / ___ \   | |  | |\  |
#                               |_|  |_| /_/   \_\ |___| |_| \_|
#
# ---------------------------------------------------------------------------------------------
#
def main( Debug=False ):
    global VERSION

    #--- Syntax ---
    args = sys.argv
    if len( args ) < 2:
        print "syntax: %s MacExecutableFile [[MacExecutableFile]...] [-clear]" % args[0]
        print 
        print  "        -clear: if appears in args, then all dylib files are erased first.\n\n"
        print  "        Version: ", VERSION
        return

    #--- get executable ---
    executables = []

    clearDyLib = False
    for i in range( 1, len( args ) ):
        if args[i].lower()=="-clear":
            clearDyLib = True
        else:
            executables.append( args[i] )

    #--- remove all the current *.dylib in directory ---
    if clearDyLib:
        try:
            subprocess.check_call( ["rm", "-f", "*.dylib" ] )
        except:
            print "could not erase all dylib files"
            sys.exit( 1 )

    #--- create a network of libs in /opt/local/libs, bring them to        ---
    #--- the local directory, and mark all their internal paths to other   ---
    #--- /opt/local/lib libraries as local.  Bring all the ones listed and ---
    #--- treat them as well.                                               ---
    for executable in executables:
        if Debug: print "calling process( %s )" % executable
        process( executable, Debug )

    #--- summary info ---
    for k in processedDico.keys():
            print "-->", k.split( '/' )[-1]


main( False ) # True )