build

#!/usr/bin/env python

import platform, sys, os, time, threading, subprocess, copy, codecs, glob, atexit, tempfile


config_name = None
config_file = None


bcolors = {
    "candidate" : '\033[94m',
    "no known conversion" : '\033[94m',
    "expected" : '\033[93m',
    "^" : '\033[91m',
    "static assertion" : '\033[91m',
    "Linking" : '\033[01;32m',
    "In function" : '\033[01;32m',
    "WARNING" : '\033[95m',
    "Warning" : '\033[95m',
    "warning" : '\033[95m',
    "required from" : '\033[94m',
    "In instantiation of" : '\033[01;32m',
    "In member" : '\033[01;32m',
    "ERROR" : '\033[01;95m',
    "error" : '\033[01;31m',
    "failed" : '\033[91m',
    "note" : '\033[94m'}

def colorize(s):
  out = []
  for l in s.splitlines():
    for st in bcolors.keys():
      if st in l:
        l = l.replace (st, bcolors[st] + st) + '\033[0m'
        break
    out.append(l + '\n')
  return out



system = None
dependencies = 0
dep_recursive = False
verbose = False
nowarnings = False
targets = []
use_multiple_cores = True


global todo, headers, object_deps, file_flags, lock, print_lock, stop, main_cindex
global include_paths
todo, headers, object_deps, file_flags = {}, {}, {}, {}
lock = threading.Lock()
print_lock = threading.Lock()
stop = False
error_stream = None
main_cindex = 0

logfile = open ('build.log', 'wb')



def pipe_errors_to_less_handler():
  global error_stream
  if len (error_stream):
    try:
      [ fid, name ] = tempfile.mkstemp()
      try:
        fid = os.fdopen (fid, 'wb')
        [ fid.write (x.encode (errors='ignore')) for x in colorize(error_stream) ]
        fid.close()
        os.system ("less -RX " + name)
      except Exception as e:
        sys.stderr.write (str (e))
      os.unlink (name)
    except Exception as e:
      sys.stderr.write (str (e))
    except:
      raise



if sys.stderr.isatty():
  error_stream = ''
  atexit.register (pipe_errors_to_less_handler)




def disp (msg):
  print_lock.acquire()
  logfile.write (msg.encode (errors='ignore'))
  sys.stdout.write (msg)
  print_lock.release()
  
def log (msg):
  print_lock.acquire()
  logfile.write (msg.encode (errors='ignore'))
  if verbose:
    sys.stderr.write (msg)
  print_lock.release()

def error (msg):
  global error_stream
  print_lock.acquire()
  logfile.write (msg.encode (errors='ignore'))
  if error_stream is not None:
    error_stream += msg
  else:
    sys.stderr.write (msg)
  print_lock.release()
  


def split_path (filename):
  return filename.replace ('\\', '/').split ('/')



def get_git_lib_version (folder):
  log ('''
getting short git version in folder "''' + folder + '"... ')

  try: 
    process = subprocess.Popen ([ 'git', 'describe', '--abbrev=0', '--always' ], cwd=folder, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    ( git_version, stderr ) = process.communicate()
    if process.returncode == 0:
      git_version = git_version.decode('utf-8', 'ignore').strip()
      log (git_version + '\n')
      return git_version
  except:
    pass
  log ('not found\n')
  return None


# common definitions:

bin_dir = 'bin'
cmd_dir = 'cmd'
lib_dir = 'lib'
misc_dir = 'src'
other_dir = '3rd_party'

cpp_suffix = '.cpp'
h_suffix = '.h'
libname = 'mrtrix'






include_paths = [ misc_dir, cmd_dir ]
if os.path.exists (other_dir):
  include_paths += [ other_dir ]

# check if we are compiling a separate project:
mrtrix_dir = [ '.' ]
build_script = sys.argv[0]
separate_project = False

def get_real_name (path):
  if os.path.islink (path): return os.readlink (path)
  else: return (path)

while os.path.abspath (os.path.dirname (get_real_name (build_script))) != os.path.abspath (mrtrix_dir[-1]):
  if not separate_project:
    log ('compiling separate project against:' + os.linesep)
  separate_project = True
  build_script = os.path.normpath (os.path.join (mrtrix_dir[-1], get_real_name (build_script)))
  module_dir = os.path.dirname (build_script) 
  mrtrix_dir += [ module_dir ]
  include_paths += [ os.path.join (module_dir, misc_dir) ]
  if os.path.exists (os.path.join (module_dir, other_dir)):
    include_paths += [ os.path.join (module_dir, other_dir) ]
  log ('    ' + module_dir + os.linesep + os.linesep)





def split_intermediate (name):
  for module_dir in mrtrix_dir:
    relname = os.path.normpath (os.path.relpath (name, module_dir))
    if relname.startswith ('.'):
      continue
    confname = relname.split(os.path.sep)[0]
    is_target = os.path.isfile (os.path.join (mrtrix_dir[-1], confname, 'config'))
    return [ module_dir, confname, os.path.relpath (name, os.path.join (module_dir, confname)) ]


def make_intermediate (name, confname):
  for module_dir in mrtrix_dir:
    relname = os.path.normpath (os.path.relpath (name, module_dir))
    if relname.startswith ('.'):
      continue
    return os.path.normpath (os.path.join (module_dir, confname, relname))






# parse command-line:
command_doc = False
bash_completion = False

for arg in sys.argv[1:]:
  if '-help'.startswith (arg):
    print ('''
usage: ./build [-verbose] [-showdep[=target|all]] [target ...]

This script will compile and link the MRtrix3 source tree. It relies on the
configuration file produced by ./configure - please ensure you have run this
first.

In most cases, a simple invocation is all that is required:

  $ ./build

If no targets are provided, the command will default to building 
all applications by scanning through the cmd/ folder. 

All intermediate files and the target executables will be located in the target
config folder (by default: 'release').

The special target 'clean' is used to remove all compiler-generated 
files, including objects, executables, and shared libraries. 

The special target 'bash' is used to update the bash completion script. Note
that automatic updating of this script can be enabled at the configure stage,
by running './configure -dev' prior to invoking './build'

The special target 'doc' is used to update the command documentation, so that
any modifications to the inline documentation in commands can be propagate
through to the user documentation site.

By default, 'build' will use all available cores to run a parallel build. In some
instances, this can cause problems (notably out of RAM errors). You can control 
how many jobs 'build' will run concurrently using the NUMBER_OF_PROCESSORS 
environment variable. For example:

  $ NUMBER_OF_PROCESSORS=1 ./build


OPTIONS:

  -verbose               print each command as it is being invoked
  -nowarnings            do not print out non-fatal compiler messages (warnings, etc)
  -showdep[=target|all]  print the list of dependencies for every target (with 
                         -showdep=target; the default) or for all files
  -tree                  [only used with -showdep) print full dependency tree for 
                         each file
''')
    exit (0)
  elif '-verbose'.startswith(arg): 
    verbose = True
  elif '-nowarnings'.startswith(arg):
    nowarnings = True
  elif '-showdep'.startswith(arg): 
    dependencies = 1
  elif arg.startswith ('-showdep='):
    if arg[9:] == 'target': dependencies = 1
    elif arg[9:] == 'all': dependencies = 2
    else: 
      sys.stderr.write ('invalid specified for option "-showdep" (expected target, all)\n')
      sys.exit (1)
  elif '-tree'.startswith(arg):
    dep_recursive = True
  elif arg[0] == '-':
    sys.stderr.write ('unknown command-line option "' + arg + '"\n')
    sys.exit (1)
  elif arg == 'bash':
    bash_completion = True
  elif arg == 'doc':
    command_doc = True
  elif arg == 'clean':
    targets = [ 'clean' ]
  elif os.path.isfile (os.path.join (mrtrix_dir[-1], arg, "config")):
    if config_name is not None and config_name is not arg:
      sys.stderr.write ('conflicting configurations requested: "' + config_name + '" and "' + arg + '"\n')
      sys.exit (1)
    config_name = arg
  else: 
    ret = split_intermediate (arg)
    if arg is None:
      sys.stderr.write ('unable to resolve target "' + arg + '"\n')
      sys.exit (1)
    targets.append(arg)
    if config_name is not None and config_name != ret[1]:
      sys.stderr.write ('conflicting configurations requested: "' + config_name + '" and "' + ret[1] + '"\n')
      sys.exit (1)
    config_name = ret[1]



# clean-up:
if 'clean' in targets:
  if config_name:
    folders = [ config_name ]
  else:
    folders = [ f for f in os.listdir('.') if os.path.exists (os.path.join (f, 'config')) ]

  folders += [ 'dev' ]

  if '.git' in folders: folders.remove ('.git')
  for f in folders:
    for root, dirs, files in os.walk(f, topdown=False):
      for name in files:
        fname = os.path.join(root, name)
        if name != 'config' and root != f:
          print ('delete file:', fname)
          os.remove (fname)
      for name in dirs:
        os.rmdir (os.path.join(root, name))


  sys.exit (0)




# load configuration file:
if config_name is None: 
  config_name = 'release'
if config_file is None:
  config_file = os.path.normpath (os.path.join (mrtrix_dir[-1], config_name, 'config'))

try:
  log ('reading configuration from "' + config_file + '"...' + os.linesep)
  exec (codecs.open (config_file, mode='r', encoding='utf-8').read())
except IOError:
  sys.stderr.write ('no configuration file found for target build "' + config_name + '''"!
please run "./configure" prior to invoking this script

''')
  sys.exit (1)


environ = os.environ.copy()
environ.update ({ 'PATH': PATH })

target_bin_dir = os.path.join (mrtrix_dir[0], config_name, bin_dir);

system = platform.system().lower()
if system.startswith('mingw') or system.startswith('msys'):
  target_lib_dir = os.path.join (mrtrix_dir[-1], config_name, bin_dir)
else:
  target_lib_dir = os.path.join (mrtrix_dir[-1], config_name, lib_dir)
lib_dir = os.path.join (mrtrix_dir[-1], lib_dir)

if ld_enabled and len(runpath):
  ld_flags += [ runpath+os.path.relpath (target_lib_dir,target_bin_dir) ]


# get version info:

lib_version = get_git_lib_version (mrtrix_dir[-1])
if lib_version is not None:
  if lib_version.find('-') > 0:
    lib_version = lib_version[0:lib_version.find('-')]
  libname += '-' + lib_version

if ld_enabled:
  ld_flags.insert(0, '-l' + libname)
  libname = lib_prefix + libname + lib_suffix


# other settings:
include_paths += [ lib_dir, cmd_dir ]
cpp_flags += [ '-I' + entry for entry in include_paths ]
ld_flags += [ '-L' + target_lib_dir ]

moc_cpp_suffix = '_moc' + cpp_suffix
moc_obj_suffix = '_moc' + obj_suffix









###########################################################################
#                           TODO list Entry
###########################################################################

class Entry:
  def __init__ (self, name):
    global todo
    name = os.path.normpath (name)
    if name in todo.keys(): return
    todo[name] = self


    self.name = name
    self.cmd = []
    self.deps = set()
    self.action = '--'
    self.timestamp = mtime (self.name)
    self.dep_timestamp = self.timestamp
    self.currently_being_processed = False

    if is_executable (self.name): self.set_executable()
    elif is_icon (self.name): self.set_icon()
    elif is_object (self.name): self.set_object()
    elif is_library (self.name): self.set_library()
    elif is_moc (self.name): self.set_moc()
    elif not os.path.exists (self.name): 
      raise Exception ('unknown target "' + self.name + '"')


    [ Entry(item) for item in self.deps ]
    dep_timestamp = [ todo[item].timestamp for item in todo.keys() if item in self.deps and not is_library(item) ]
    dep_timestamp += [ todo[item].dep_timestamp for item in todo.keys() if item in self.deps and not is_library(item) ]
    if len(dep_timestamp): 
      self.dep_timestamp = max(dep_timestamp)

    

  def execute (self, cindex, formatstr):
    folder = os.path.dirname (self.name)
    try:
      os.makedirs (folder)
    except OSError as e:
      if not os.path.isdir (folder):
        error ('ERROR: can''t create target folder "' + folder + '": ' + os.strerror (e.errno))
        exit (1)
        
    if self.action == 'RCC':
      with codecs.open (self.cmd[1], mode='w', encoding='utf-8') as fd:
        fd.write ('<!DOCTYPE RCC><RCC version="1.0">\n<qresource>\n')
        for entry in self.deps:
          entry = os.path.basename (entry)
          if not entry.startswith ('config'):
            fd.write ('<file>' + entry + '</file>\n')
        fd.write ('</qresource>\n</RCC>\n')
      
    if len(self.cmd) > 0: 
      return execute (formatstr.format (cindex, self.action, self.name), self.cmd)
    else: 
      return None


  def set_executable (self):
    self.action = 'LB'
    if len(exe_suffix) > 0 and self.name.endswith(exe_suffix): cc_file = self.name[:-len(exe_suffix)]
    else: cc_file = self.name
    cc_file = split_intermediate (cc_file)
    cc_file = os.path.normpath (os.path.join (cc_file[0], cmd_dir, os.sep.join (split_path(cc_file[2])[1:]) + cpp_suffix))
    self.deps = list_cmd_deps(cc_file)

    skip = False
    flags = []
    if 'Q' in file_flags[cc_file]: flags += qt_ldflags

    if not skip: 
      if not ld_enabled: 
        self.deps = self.deps.union (list_lib_deps())

      self.cmd = fillin (ld, { 
        'LDFLAGS': [ s.replace ('LIBNAME', os.path.basename (self.name)) for s in ld_flags ] + flags,
        'OBJECTS': self.deps,
        'EXECUTABLE': [ self.name ] })

      try:
        if ld_use_shell: self.cmd = [ 'sh', '-c', ' '.join(self.cmd) ]
      except NameError: pass

      if ld_enabled:
        self.deps.add (os.path.normpath (os.path.join (target_lib_dir, libname)))



  def set_object (self):
    self.action = 'CC'
    cc_file = self.name[:-len(obj_suffix)] + cpp_suffix

    flags = copy.copy (eigen_cflags)

    if is_moc (cc_file):
      cc_file = split_intermediate (cc_file)
      src_header = os.path.normpath (os.path.join (cc_file[0], cc_file[2][:-len(moc_cpp_suffix)] + h_suffix))
      cc_file = os.path.normpath (os.path.join (cc_file[0], cc_file[1], cc_file[2]))
      list_headers (src_header)
      file_flags[cc_file] = file_flags[src_header]
    else:
      cc_file = split_intermediate (cc_file)
      cc_file = os.path.normpath (os.path.join (cc_file[0], cc_file[2]))
      self.deps = self.deps.union (list_headers (cc_file))
    self.deps.add (config_file)

    self.deps.add( cc_file )

    skip = False
    if 'Q' in file_flags[cc_file]: flags += qt_cflags

    if not skip: 
      self.cmd = fillin (cpp, { 
        'CFLAGS': cpp_flags + flags,
        'OBJECT': [ self.name ],
        'SRC': [ cc_file ] })


  def set_moc (self):
    self.action = 'MOC'
    src_file = split_intermediate (self.name[:-len(moc_cpp_suffix)] + h_suffix)
    src_file = os.path.normpath (os.path.join (src_file[0], src_file[2]))
    self.deps = set([ src_file ])
    self.deps = self.deps.union (list_headers (src_file))
    self.deps.add (config_file)
    self.cmd = [ moc ]
    self.cmd += [ src_file, '-o', self.name ]

    
  def set_library (self):
    if not ld_enabled:
      error ('ERROR: shared library generation is disabled in this configuration')
      exit (1)

    self.action = 'LD'
    self.deps = list_lib_deps()

    self.cmd = fillin (ld_lib, {
      'LDLIB_FLAGS': [ s.replace ('LIBNAME', os.path.basename (self.name)) for s in ld_lib_flags ],
      'OBJECTS': [ item for item in self.deps ],
      'LIB': [ self.name ] })

    try:
      if ld_use_shell: self.cmd = [ 'sh', '-c', ' '.join(self.cmd) ]
    except NameError: pass


  def set_icon (self):
    self.action = 'RCC'
    with codecs.open (self.name[:-len(cpp_suffix)]+h_suffix, mode='r', encoding='utf-8') as fd:
      for line in fd:
        if line.startswith ('//RCC:'):
          for entry in line[6:].strip().split():
            self.deps = self.deps.union (glob.glob (os.path.normpath (os.path.join (mrtrix_dir[-1], 'icons', entry))))
    self.deps.add (config_file)
    qrc_file = os.path.normpath (os.path.join (mrtrix_dir[-1], 'icons', os.path.basename (os.path.dirname(self.name)) + '.qrc'))
    self.cmd = [ rcc, qrc_file, '-o', self.name ]


  def need_rebuild (self):
    return self.timestamp == float("inf") or self.timestamp < self.dep_timestamp

  def display (self, indent=''):
    show_rebuild = lambda x: x+' [REBUILD]' if todo[x].need_rebuild() else x 
    sys.stdout.write (indent + '[' + self.action + '] ' + show_rebuild (self.name) + ':\n')
    sys.stdout.write (indent + '  timestamp: ' + str(self.timestamp))
    if len(self.deps):
      sys.stdout.write (', dep timestamp: ' + str(self.dep_timestamp) + ', diff: ' + str(self.timestamp-self.dep_timestamp))
    sys.stdout.write ('\n')
    if len(self.cmd):
      sys.stdout.write (indent + '  command:   ' + ' '.join(self.cmd) + '\n')
    if len(self.deps):
      sys.stdout.write (indent + '  deps:      ')
      if dep_recursive:
        sys.stdout.write ('\n')
        for x in self.deps:
          todo[x].display (indent + '    ')
      else:
        sys.stdout.write ((indent+'\n             ').join([ show_rebuild(x) for x in self.deps ]) + '\n')






###########################################################################
#                         FUNCTION DEFINITIONS
###########################################################################


def default_targets():
  if not os.path.isdir (cmd_dir): 
    sys.stderr.write ('ERROR: no "cmd" folder - unable to determine default targets' + os.linesep)
    sys.exit (1)
  for entry in os.listdir (cmd_dir):
    if entry.endswith(cpp_suffix):
      targets.append (os.path.normpath (os.path.join (target_bin_dir, entry[:-len(cpp_suffix)] + exe_suffix)))
  return targets

def is_executable (target):
  return os.path.normpath (split_path (split_intermediate (target)[2])[0]) == os.path.normpath (bin_dir) and not is_moc (target) and not is_library (target)

def is_library (target):
  return target.endswith (lib_suffix) and split_path(target)[-1].startswith (lib_prefix)

def is_object (target):
  return target.endswith (obj_suffix)

def is_moc (target):
  return target.endswith (moc_cpp_suffix)

def is_icon (target):
  return os.path.basename (target) == 'icons'+cpp_suffix

def mtime (target):
  if not os.path.exists (target): return float('inf')
  return os.stat(target).st_mtime


def fillin (template, keyvalue):
  cmd = []
  for item in template:
    if item in keyvalue: 
      cmd += keyvalue[item]
    else: 
      cmd += [ item ]
  return cmd




def execute (message, cmd, working_dir=None):
  disp (message + os.linesep)
  log (' '.join(cmd) + os.linesep)

  try:
    process = subprocess.Popen (cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environ, cwd=working_dir)
    ( stdout, stderr ) = process.communicate()
    if process.returncode != 0:
      if error_stream is not None:
        disp ('ERROR: ' + message + os.linesep)
      error ('\nERROR: ' + message + '\n\n' + ' '.join(cmd) + '\n\nfailed with output\n\n' + stderr.decode (errors='ignore'))
      return 1

    errstr = None
    if len(stdout) or len(stderr):
      errstr = '\n' + ' '.join(cmd) + ':\n\n'

    if len(stdout):
      errstr += stdout.decode (errors='ignore') + '\n\n'
    if len(stderr):
      errstr += stderr.decode (errors='ignore') + '\n\n'

    if errstr and not nowarnings:
      disp (errstr);

  except OSError:
    error (cmd[0] + ': command not found')
    return 1
  except:
    error ('unexpected exception: ' + str(sys.exc_info()))
    raise


def print_deps (current_file, indent=''):
  current_file = os.path.normpath (current_file)
  sys.stdout.write (indent + current_file)
  if current_file in file_flags: 
    if len(file_flags[current_file]): 
      sys.stdout.write (' [' + file_flags[current_file] + ']')
  sys.stdout.write (os.linesep)
  if len(todo[current_file].deps):
    for entry in todo[current_file].deps:
      print_deps (entry, indent + '    ')




def is_GUI_target (current_file):
  if 'gui' in split_path (current_file):
    return True
  if current_file in file_flags: 
    if 'Q' in file_flags[current_file]: 
      return True
  if len(todo[current_file].deps):
    for entry in todo[current_file].deps:
      if is_GUI_target (entry):
        return True
  return False



def list_headers (current_file):
  global headers, file_flags
  current_file = os.path.normpath (current_file)

  if current_file not in headers.keys(): 
    headers[current_file] = set()

    if current_file not in file_flags: 
      file_flags[current_file] = ''

    if 'gui' in split_path (current_file):
      if 'Q' not in file_flags[current_file]:
        file_flags[current_file] += 'Q'
    if not os.path.exists (current_file):
      if os.path.basename(current_file) == 'icons'+cpp_suffix: 
        return headers[current_file]
      sys.stderr.write ('ERROR: cannot find file "' + current_file + '"' + os.linesep)
      sys.exit(1)
    with codecs.open (current_file, mode='r', encoding='utf-8') as fd:
      for line in fd:
        line = line.strip()
        if line.startswith('#include'):
          line = line[8:].split ('//')[0].split ('/*')[0].strip()
          if line[0] == '"':
            line = line[1:].rstrip('"')
            for path in include_paths:
              if os.path.exists (os.path.join (path, line)):
                line = os.path.normpath (os.path.join (path, line))
                headers[current_file].add (line)
                [ headers[current_file].add(entry) for entry in list_headers(line) ]
                break
            else:
              sys.stderr.write ('ERROR: cannot find header file \"' + line + '\" (from file \"' + current_file + '\")' + os.linesep)
              sys.exit(1)
        elif line == 'Q_OBJECT':
          if 'M' not in file_flags[current_file]:
            file_flags[current_file] += 'M'

    for entry in headers[current_file]:
      for c in file_flags[entry]:
        if c != 'M' and c not in file_flags[current_file]:
          file_flags[current_file] += c

  return headers[current_file]






def list_cmd_deps (file_cc):
  global object_deps, file_flags
  file_cc = os.path.normpath (file_cc)

  if file_cc not in object_deps.keys():
    object_deps[file_cc] = set([ make_intermediate (file_cc[:-len(cpp_suffix)] + obj_suffix, config_name) ])
    for entry in list_headers (file_cc):
      if os.path.abspath(entry).startswith(os.path.abspath(lib_dir)): continue
      if 'M' in file_flags[entry]: 
        object_deps[file_cc] = object_deps[file_cc].union ([ make_intermediate (entry[:-len(h_suffix)] + moc_obj_suffix, config_name) ])
      entry_cc = entry[:-len(h_suffix)] + cpp_suffix
      if os.path.exists (entry_cc):
        object_deps[file_cc] = object_deps[file_cc].union (list_cmd_deps(entry_cc))
      if os.path.basename (entry) == 'icons'+h_suffix:
        object_deps[file_cc].add (make_intermediate (entry[:-len(h_suffix)] + obj_suffix, config_name))
    if file_cc not in file_flags: file_flags[file_cc] = ''
    for entry in headers[file_cc]:
      for c in file_flags[entry]:
        if c != 'M' and c not in file_flags[file_cc]:
          file_flags[file_cc] += c

  return object_deps[file_cc]



def list_lib_deps ():
  deps = set()
  for root, dirs, files in os.walk (lib_dir):
    for current_file in files:
      if current_file[0] == '.': continue
      if current_file.endswith (cpp_suffix):
        deps.add (make_intermediate (os.path.join (root, current_file[:-len(cpp_suffix)] + obj_suffix), config_name))

  return (deps)



def build_next (id):
  global todo, lock, stop, main_cindex
  total_count = len(todo)
  cindex = 0
  formatstr = '({:>'+str(len(str(total_count)))+'}/'+str(total_count)+') [{}] {}'

  try:
    while not stop:
      current = None
      lock.acquire()
      if len(todo):
        for item in todo.keys():
          if todo[item].currently_being_processed: continue
          unsatisfied_deps = set(todo[item].deps).intersection (todo.keys())
          if not len(unsatisfied_deps):
            todo[item].currently_being_processed = True
            current = item
            main_cindex+=1 
            cindex = main_cindex
            break
      else: stop = max (stop, 1)
      lock.release()
  
      if stop: return
      if current == None: 
        time.sleep (0.01)
        continue
  
      target = todo[current]
      if target.execute(cindex, formatstr):
        stop = 2
        return

      lock.acquire()
      del todo[current]
      lock.release()

  except:
    stop = 2
    return
    
  stop = max(stop, 1)



def start_html (fid, title, left, up, home, right):
  fid.write ('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">\n<html>\n<head>\n<meta http-equiv="Content-Type" content="text/html;charset=iso-8859-1">\n')
  fid.write ('<title>MRtrix documentation</title>\n<link rel="stylesheet" href="../stylesheet.css" type="text/css" media=screen>\n</head>\n<body>\n\n')
  fid.write ('<table class=nav>\n<tr>\n<td><a href="' + left + '.html"><img src="../left.png"></a></td>\n')
  fid.write ('<td><a href="' + up + '.html"><img src="../up.png"></a></td>\n')
  fid.write ('<td><a href="' + home + '.html"><img src="../home.png"></a></td>\n')
  fid.write ('<th>' + title + '</th>\n')
  fid.write ('<td><a href="' + right + '.html"><img src="../right.png"></a></td>\n</tr>\n</table>\n')





def get_git_version (folder):
  log ('''
getting git version in folder "''' + folder + '"... ')

  try: 
    process = subprocess.Popen ([ 'git', 'describe', '--abbrev=8', '--dirty', '--always' ], cwd=folder, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    ( git_version, stderr ) = process.communicate()
    if process.returncode == 0:
      git_version = git_version.decode(errors='ignore').strip()
      log (git_version + '\n')
      return git_version
  except:
    pass


  log ('not found\n')
  return 'unknown'


def update_git_version (folder, git_version_file, contents):
  git_version = get_git_version (folder)
  version_file_contents = contents.replace ('%%%', git_version)
  current_version_file_contents = ''
  try:
    with open (git_version_file) as fd:
      current_version_file_contents = fd.read()
  except:
    pass
    
  if not current_version_file_contents or (version_file_contents != current_version_file_contents and git_version != 'unknown'):
    log ('version file "' + git_version_file + '" is out of date - updating\n')
    with open (git_version_file, 'w') as fd:
      fd.write (version_file_contents)


def update_bash_completion ():
  # Only attempt to generate completion file if POSIX compliant
  if os.name != 'posix': return
   
  # Check whether completion generation script exists
  scripts_dir = os.path.join (mrtrix_dir[-1], 'scripts' )
  script_path = os.path.join (scripts_dir, 'generate_mrtrix_cmd_completion.py') 
  completion_path = os.path.join (scripts_dir, 'mrtrix_bash_completion')
  if not os.path.isfile (script_path):
    disp (colorize('WARNING: Skipping bash completion generation. Could not find script at ' + script_path + '\n')[0])
    return
  
  import re

  # Check whether both command files and completion file exist and completion file is newer than commands
  if not os.path.isdir (target_bin_dir):
    return
  else:
    # Only look at relevant executables in bin file
    commands = [comm for comm in os.listdir (target_bin_dir) if os.access( os.path.join( target_bin_dir, comm ), os.X_OK) and not re.match (r'^\w+__debug', comm)]
    if not commands: 
      return
    else:
      command_freshness_time = max (commands, key = lambda x: time.ctime ( os.path.getmtime( os.path.join( target_bin_dir, x ) ) ) )
      if os.path.isfile (completion_path) and time.ctime ( os.path.getmtime(completion_path) ) >= command_freshness_time:
        return

  execute ('[SH] ' + completion_path, [ script_path, "-c", completion_path, "-m", target_bin_dir ] )

def update_user_docs ():
  # Only attempt to update docs if POSIX compliant
  if os.name != 'posix': return

  scripts_dir = os.path.abspath(os.path.join (mrtrix_dir[-1], 'docs' ))
  script_path = os.path.join (scripts_dir, 'generate_user_docs.sh')  

  # Check whether generate docs script exists
  if not os.path.isfile (script_path):
    disp (colorize('WARNING: Skipping user documentation generation. Could not find script at ' + script_path + '\n')[0])
    return

  # Check whether commands dir exists
  if not os.path.isdir (target_bin_dir):
    return
  
  import re

  # Fetch relevant executables in bin file
  commands = [comm for comm in os.listdir (target_bin_dir) if os.access( os.path.join( target_bin_dir, comm ), os.X_OK) and not re.match (r'^\w+__debug', comm)]
  
  # Fetch relevant user docs
  rst_files = []
  
  for root, subfolders, files in os.walk (scripts_dir):
    for rst_file in files:
      if rst_file.endswith('.rst'):
        rst_files.append(os.path.join(root, rst_file))

  if not commands:
    return

  if rst_files:
    command_freshness_time = max (commands, key = lambda x: time.ctime ( os.path.getmtime( os.path.join( target_bin_dir, x ) ) ) )
    docs_freshness_time = min (rst_files, key = lambda x: time.ctime ( os.path.getmtime( x ) ) )
    if docs_freshness_time >= command_freshness_time:
      return

  if execute ('[DOC] generating user documentation (./docs)', [ script_path ], scripts_dir):
    sys.exit (1)
  



###########################################################################
#                            START OF PROGRAM
###########################################################################


if len(targets) == 0: 
  targets = default_targets()



# get git version info:
update_git_version (mrtrix_dir[-1], os.path.join (mrtrix_dir[-1], 'lib/version.cpp'), '''
namespace MR { 
  namespace App { 
    const char* mrtrix_version = "%%%";
  } 
}
''')

if separate_project:
  if not os.path.exists (misc_dir):
    os.mkdir (misc_dir)
  git_version_file = os.path.join (misc_dir, 'project_version.h')
  update_git_version ('.', git_version_file, '#define MRTRIX_PROJECT_VERSION "%%%"\n');
  if not os.path.exists (git_version_file):
    with open (git_version_file, 'w'):
      pass


    





log ('''
compiling TODO list...
''')

[ Entry(item) for item in targets ]



# for nogui config, remove GUI elements from targets and todo list:
if nogui:
  nogui_targets = []
  for entry in targets:
    if not is_GUI_target (entry):
      nogui_targets.append (entry)
  targets = nogui_targets
  
  nogui_todo = {}
  for item in todo.keys():
    if not is_GUI_target (todo[item].name):
       nogui_todo[item] = todo[item]
  todo = nogui_todo





log ('building targets: ' + ' '.join (targets) + os.linesep)


if dependencies==1:
  sys.stderr.write ('''
Printing dependencies for targets:

''')
  for entry in targets:
    todo[entry].display()
  sys.exit (0)
elif dependencies==2:
  sys.stderr.write ('''
Printing dependencies for all files:

''')
  for entry in todo:
    todo[entry].display()
  sys.exit (0)

todo_tmp = {}
for item in todo.keys():
  if todo[item].action != '--' and todo[item].need_rebuild():
    todo_tmp[item] = todo[item]
todo = todo_tmp

log ('TODO list contains ' + str(len(todo)) + ''' items

''')


#for entry in todo.values(): entry.display()

if len(todo): 
  
  try: num_processors = int(os.environ['NUMBER_OF_PROCESSORS'])
  except:
    try: num_processors = os.sysconf('SC_NPROCESSORS_ONLN')
    except: num_processors = 1

  log ('''
  
  launching ''' + str(num_processors) + ''' threads
  
  ''')
  
  threads = []
  for i in range (1, num_processors):
    t = threading.Thread (target=build_next, args=(i,));
    t.start()
    threads.append (t)
  
  build_next(0)
  
  for t in threads: t.join()


# generate development-specific files (if needed)
# i.e. bash completion and user documentation
if not separate_project and stop <= 1:
  if bash_completion:
    update_bash_completion()
  if command_doc:
    update_user_docs()

sys.exit (stop > 1)