#!/bin/sh
''':'
# This code will be treated as a comment in Python, but will be executed
# by the shell, allowing us to do some special handling before the python
# interpreter starts.
PYTHONPATH="" exec python3 "$0" "$@"
' '''

#
# 2023-11-13  Now ignores PYTHONPATH
# 2021-07-01  Support for expanding includes in .ppf files.
#             Fixed some issues when run with python3
# 2019-09-09  Fixed a crash if an unknown file type was encountered.
# 2019-08-20  Improved performance.
#             Added -t option.
# 2019-05-20  Ignore files not starting with the take name followed by a
#             dot.
# 2017-01-25  Fixed missing .cal file
# 2017-01-24  Initial implementation
#

import sys
import os
import subprocess
import re
stdout=sys.stdout
stderr=sys.stderr
running_unit_tests=False

class SystemRunner():
  def runAndReturnOutput(self,args):
    assert not running_unit_tests
    return runAndReturnOutput(args)
  def expandVars(self,path):
    return os.path.expandvars(path)
  def readFile(self,path):
    return open(path,'r').read()
  def filesInDirectory(self,dir_path):
    return filesInDirectory(dir_path)
  def fileExists(self,path):
    return os.path.exists(path) and os.path.isfile(path)
  def directoryExists(self,path):
    return os.path.exists(path) and os.path.isdir(path)
  def getpid(self):
    return os.getpid()
  def deleteFile(self,path):
    return os.remove(path)
  def log(self,message):
    stderr.write(message)

def uncompressedPath(path):
  if not path.endswith('.gz'):
    return path
  return path[:-3]

def getTagFiles(tag,path,runner):
  output = runner.runAndReturnOutput(['tagfile','-getfile',tag,path])
  split_output = output.split('\n')
  split_output = [x for x in split_output if x != '']
  return split_output


def existingExtension(full_path,runner):
  if runner.fileExists(full_path):
    return ''
  if runner.fileExists(full_path + '.gz'):
    return '.gz'
  return None

def getIncludes(base_file,runner):
  uncompressed_base_file = uncompressedPath(base_file)

  if uncompressed_base_file != base_file:
    tmp_uncompressed_base_file = \
      uncompressed_base_file + '.tartake_' + str(runner.getpid())
    runner.runAndReturnOutput(
      ['sh','-c','gunzip -c '+base_file+' >'+tmp_uncompressed_base_file]
    )
    tagfile_file = tmp_uncompressed_base_file
  else:
    tagfile_file = base_file

  include_file_names = getTagFiles('INCLUDE.FILE_NAME',tagfile_file,runner)

  if uncompressed_base_file != base_file:
    runner.deleteFile(tmp_uncompressed_base_file)
    #rm -f tmp_uncompressed_base_file

  for path in include_file_names:
    checkFullPath(path)

  return include_file_names


def debug(s):
  stderr.write(s)

system_runner = SystemRunner()

def throwException(e):
  raise e

class UserException(Exception):
  def __init__(self,message):
    self.message = message

def makeTarFilePathFrom(take_dir):
  return os.path.basename(take_dir)+'.tar.gz'

def filesInDirectory(dir_path):
  walk_result = os.walk(dir_path,onerror=throwException)
  try:
    for dirpath,dirnames,filenames in walk_result:
      return filenames
  except OSError as e:
    message = e.strerror + ': ' + e.filename
    raise UserException(message)

def workspaceRelativePath(path,data_dir):
  if path.startswith(data_dir+'/'):
    return path[len(data_dir)+1:]
  return path

def isPSFFile(path):
  return path.endswith('.psf')

def isCPJFile(path):
  return path.endswith('.cpj')

def isANMFile(path):
  return path.endswith('.anm')

def isRTParamsFile(path):
  return path.endswith('.rtparams')

def runAndReturnOutput(args):
  # Using this instead of check_output for compatibility with python 2.6
  popen = subprocess.Popen(['/usr/bin/env']+args, stdout=subprocess.PIPE)
  return str(popen.communicate()[0].decode('ascii'))

def makeSpec(line):
  m = re.match('([^:]+): *(.*)',line)
  if m is None:
    debug('line: %s\n'%str(line))
    return None
  type_name = m.groups()[0]
  relative_path = m.groups()[1]
  return (type_name,relative_path)

def makeSpecs(output):
  result = output.split('\n')
  result = [makeSpec(x) for x in result if x !='']
  result = [x for x in result if x is not None]
  return result

def getTopIncludes(input_path,type_name,data_dir,runner):
  full_include_paths = getIncludes(input_path,runner)
  relative_include_paths = \
    [workspaceRelativePath(path,data_dir) for path in full_include_paths]
  return [(type_name,path) for path in relative_include_paths]

def getTopFilesWithCommand(command,path,runner):
  checkFullPath(path)
  args=[command,'-t',path]
  return makeSpecs(runner.runAndReturnOutput(args))

def getTopPSFFiles(path,runner):
  return getTopFilesWithCommand('tarpsf',path,runner)

def getTopCPJFiles(path,runner):
  return getTopFilesWithCommand('tarcpj',path,runner)

def getTopMDLFiles(path,runner):
  return getTopFilesWithCommand('tarmdl',path,runner)

def getTopANMFiles(path,runner):
  return getTopFilesWithCommand('taranim',path,runner)

def getTopPRJFiles(path,runner):
  return getTopFilesWithCommand('tarprj',path,runner)

def getTopRTParamsFiles(path,runner,data_dir):
  lines = runner.readFile(path).split('\n')
  for line in lines:
    m = re.match('CAMERA_CALIBRATION: *(.*)',line)
    if m is not None:
      expanded_path = runner.expandVars(m.groups()[0])
      relative_path = workspaceRelativePath(expanded_path,data_dir)
      return [('FILE',relative_path)]
  return []


def addPathTo(files,new_file):
  assert new_file!=''
  if new_file in files:
    return
  files.append(new_file)

def addPathsTo(files,new_files):
  assert type(new_files)==list
  for path in new_files:
    addPathTo(files,path)

def fullPathsOf(filenames,base_dir):
  result = list(map(lambda filename: os.path.join(base_dir,filename),filenames))
  for path in result:
    checkFullPath(path)
  return list(result)

def noTakeDirectoryGivenErrorMessage():
  return 'No take directory given'

def noTarFileOptionErrorMessage():
  return '-c must be specified if multiple files are given.'

def parseArgs(argv):
  class ParseResult:
    pass

  result = ParseResult()
  result.error = None
  result.list_only = False
  result.top_only = False
  result.output_tar_path = None

  if argv[1]=='-l':
    if len(argv)<3:
      result.error = noTakeDirectoryGivenErrorMessage()
      return result
    result.list_only = True
    result.dir_paths = argv[2:]
  elif argv[1] == '-t':
    if len(argv)<3:
      assert False
    result.top_only = True
    result.dir_paths = argv[2:]
  elif argv[1] == '-c':
    if len(argv) < 3:
      assert False
    result.output_tar_path = argv[2]
    result.dir_paths = argv[3:]
  elif len(argv[1:]) != 1:
    result.error = noTarFileOptionErrorMessage()
  else:
    result.dir_paths = [argv[1]]

  return result


def isBasicTakePath(path,take_dir):
  take_name = os.path.basename(take_dir)
  filename = os.path.basename(path)
  if not filename.startswith(take_name + '.'):
    return False
  extension = filename[len(take_name) + 1:]
  if '.' in extension:
    return False
  generated_extensions = [
    'bdf','pdf','mdl','scl','lmt','dof','ppf','bmo','raw','gid'
  ]
  if extension in generated_extensions:
    # Files with these extensions are generated as part of processing
    # a project, so they are not considered part of the take itself.
    return False
  return True


def isTakePath(path,take_dir):
  take_name = os.path.basename(take_dir)
  filename = os.path.basename(path)
  if not filename.startswith(take_name + '.'):
    return False
  return True


def isRelativePath(path,data_dir):
  if path.startswith('..'):
    return False
  return not path.startswith(data_dir)


def checkFullPath(path):
  assert path != ''
  assert path != '/'
  assert '/..' not in path
  assert 'data/data' not in path
  if not path.startswith('/'):
    debug('path: %s\n'%str(path))
    assert False

class Generator:
  def __init__(self,data_dir,runner):
    self.relative_files = []
    self.data_dir = data_dir
    self.unexpanded_specs = []
    self.expanded_specs = []
    self.runner = runner
    self.take_files = []
      # These are all the files that are in the take directory that
      # need to be included.
    self.basic_paths = set()
      # These paths are part of the main take data and should always be
      # in the final result.
  def addTake(self,take_dir,paths_of_files_in_take_dir):
    for path in paths_of_files_in_take_dir:
      checkFullPath(path)
      if isTakePath(path,take_dir):
        self.take_files.append(path)
        if isBasicTakePath(path,take_dir):
          self.basic_paths.add(path)
  def addTakeFiles(self):
    for full_path in self.take_files:
      relative_path = workspaceRelativePath(full_path,self.data_dir)
      self.addFiles([relative_path])
    self.take_files = []
  def addUnexpandedBasicTakeFiles(self):
    new_take_files = []
    for full_path in self.take_files:
      if full_path in self.basic_paths:
        relative_path = workspaceRelativePath(full_path,self.data_dir)
        self.addUnexpandedSpec(('FILE',relative_path))
      else:
        new_take_files.append(full_path)
    self.take_files = new_take_files
  def addUnexpandedPSFFiles(self):
    new_take_files = []
    for full_path in self.take_files:
      if isPSFFile(full_path):
        relative_path = workspaceRelativePath(full_path,self.data_dir)
        assert isRelativePath(relative_path,self.data_dir)
        self.addUnexpandedSpec(('PSF',relative_path))
      else:
        new_take_files.append(full_path)
    self.take_files = new_take_files
  def addUnexpandedSpecs(self,specs):
    for spec in specs:
      self.addUnexpandedSpec(spec)
  def addUnexpandedSpec(self,spec):
    type_name, relative_path = spec
    if spec not in self.unexpanded_specs and spec not in self.expanded_specs:
      self.unexpanded_specs.append((type_name,relative_path))
  def addUnexpandedCPJFiles(self):
    new_take_files = []
    for full_path in self.take_files:
      checkFullPath(full_path)
      if isCPJFile(full_path):
        relative_path = workspaceRelativePath(full_path,self.data_dir)
        self.addUnexpandedSpec(('CPJ',relative_path))
      else:
        new_take_files.append(full_path)
    self.take_files = new_take_files
  def addUnexpandedANMFiles(self):
    new_take_files = []
    for full_path in self.take_files:
      if isANMFile(full_path):
        relative_path = workspaceRelativePath(full_path,self.data_dir)
        self.addUnexpandedSpec(('ANIMATION',relative_path))
      else:
        new_take_files.append(full_path)
    self.take_files = new_take_files
  def addUnexpandedRTParamsFiles(self):
    new_take_files = []
    for full_path in self.take_files:
      if isRTParamsFile(full_path):
        relative_path = workspaceRelativePath(full_path,self.data_dir)
        self.addUnexpandedSpec(('RTPARAMS',relative_path))
      else:
        new_take_files.append(full_path)
    self.take_files = new_take_files
  def addUnexpandedTakePRJ(self,take_dir):
    prj_path = take_dir+'.prj'
    if self.runner.fileExists(prj_path):
      relative_prj_path = workspaceRelativePath(prj_path,self.data_dir)
      self.addUnexpandedSpec(('FILE',relative_prj_path))
  def addFiles(self,new_files):
    addPathsTo(self.relative_files,new_files)
  def log(self,message):
    self.runner.log(message)
  def isLeafType(self,file_type):
    # Leaf types do not reference other files.
    leaf_types = [ 'MOTION', 'FILE', 'DIRECTORY' ]
    return file_type in leaf_types
  def isSimpleType(self,file_type):
    # Simple types only have includes of the same type.
    simple_types = [
      'PROBLEM_DEF', 'BODY_DEF', 'MAPPING', 'SCALING', 'LIMITS', 'PATTERN'
    ]
    return file_type in simple_types
  def topSpecs(self, file_type, full_file_path):
    if self.isLeafType(file_type):
      return []
    if self.isSimpleType(file_type):
      include_specs = \
        getTopIncludes(full_file_path,file_type,self.data_dir,self.runner)
      return include_specs
    if file_type == 'PSF':
      return getTopPSFFiles(full_file_path,self.runner)
    if file_type == 'ANIMATION':
      return getTopANMFiles(full_file_path,self.runner)
    if file_type == 'PROJECT':
      return getTopPRJFiles(full_file_path,self.runner)
    if file_type == 'CPJ':
      return getTopCPJFiles(full_file_path,self.runner)
    if file_type == 'MODEL':
      return getTopMDLFiles(full_file_path,self.runner)
    if file_type == 'RTPARAMS':
      return getTopRTParamsFiles(full_file_path,self.runner,self.data_dir)
    self.log('Unknown file type: %s\n'%str(file_type))
    return []
  def addUnexpandedTopFiles(self,file_type,full_file_path):
    top_specs = self.topSpecs(file_type,full_file_path)
    self.addUnexpandedSpecs(top_specs)
  def expand(self, file_type, full_file_path, relative_file_path):
    checkFullPath(full_file_path)
    self.addFiles([relative_file_path])
    self.addUnexpandedTopFiles(file_type,full_file_path)
  def expandFiles(self):
    while len(self.unexpanded_specs) > 0:
      file_type,relative_path = self.unexpanded_specs[0]
      del self.unexpanded_specs[0]
      if (file_type,relative_path) in self.expanded_specs:
        debug('%s has already been expanded.\n'%str(relative_path))
        assert False
      self.expanded_specs.append((file_type,relative_path))
      if relative_path not in self.relative_files:
        # We want existing_file_path to end up being an existing file,
        # but specified relative to the workspace.
        # However, it's not good enough to use workspaceRelativePath()
        # because the relative path may actually be an absolute path,
        # and if it is, we want the result to also be an absolute path.
        file_path = os.path.join(self.data_dir,relative_path)

        if file_type == 'DIRECTORY':
          if self.runner.directoryExists(file_path):
            existing_relative_path = relative_path
            existing_file_path = file_path
          else:
            existing_relative_path = None
            existing_file_path = None
        else:
          extension = existingExtension(file_path,self.runner)

          if extension is None:
            existing_file_path = None
            existing_relative_path = None
          else:
            existing_file_path = file_path + extension
            existing_relative_path = relative_path + extension

        if existing_relative_path is not None:
          self.expand(file_type,existing_file_path,existing_relative_path)
        else:
          self.log('Missing file %s\n'%str(file_path))


def addTopFilesTo(generator,dir_paths,runner):
  for take_dir in dir_paths:
    files = runner.filesInDirectory(take_dir)
    paths_of_files_in_take_dir = fullPathsOf(files,take_dir)
    generator.addTake(take_dir,paths_of_files_in_take_dir)
    generator.addUnexpandedTakePRJ(take_dir)

  generator.addUnexpandedPSFFiles()
  generator.addUnexpandedCPJFiles()
  generator.addUnexpandedANMFiles()
  generator.addUnexpandedRTParamsFiles()
  generator.addUnexpandedBasicTakeFiles()


def generateRelativeFiles(dir_paths,data_dir,runner):
  generator = Generator(data_dir,runner)
  addTopFilesTo(generator, dir_paths, runner)
  generator.expandFiles()
  generator.addTakeFiles()
  return generator.relative_files


def generateTopFiles(dir_paths,data_dir,runner):
  generator = Generator(data_dir,runner)
  addTopFilesTo(generator,dir_paths,runner)
  return generator.unexpanded_specs


def removePrefix(text,prefix):
  if text.startswith(prefix):
    return text[len(prefix):]
  return text

class FakeRunner:
  def __init__(self):
    self.executed_commands = []
    self.table = self.makeTable()
    self.leaf_files = self.makeLeafFiles()
    self.log_messages = []

  def getpid(self):
    return 1234

  def makeTable(self):
    return {
      ('tarpsf','-t','/my/workspace/capture/takes/test/test.psf'): (
        'ANIMATION: anim/char1.anm\n'
        'FILE: charmap1.cm\n'
        'FILE: subject/joe/joe.ppf\n'
        'PROJECT: capture/subject/joe.scale.prj\n'
      ),
      ('tarpsf','-t','/my/workspace/capture/takes/test1/test1.psf'): (
        'ANIMATION: anim/char1.anm\n'
        'FILE: charmap1.cm\n'
        'FILE: subject/joe/joe.ppf\n'
        'PROJECT: capture/subject/joe.scale.prj\n'
        'PROJECT: capture/subject/bob.scale.prj\n'
        'PROJECT: capture/subject/tim.rom.prj\n'
      ),
      ('tarpsf','-t','/my/workspace/capture/takes/test2/test2.psf'): (
        'ANIMATION: anim/char1.anm\n'
        'FILE: charmap1.cm\n'
        'FILE: subject/joe/joe.ppf\n'
        'PROJECT: capture/subject/joe.scale.prj\n'
      ),
      ('tarpsf', '-t', '/my/workspace/capture/takes/test3/test3.psf'): (
        'PROJECT: capture/subject/bob.scale.prj\n'
      ),
      ('tarpsf', '-t', '/my/workspace/capture/takes/test4/test4.psf'): (
        'PROJECT: capture/subject/albert.scale.prj\n'
        'IMUCAL: calib/test.imucal\n'
      ),
      ('tarprj', '-t', '/my/workspace/capture/subject/joe.scale.prj'): (
        'FILE: capture/subject/joe.scale/joe.scale.raw\n'
        'FILE: capture/subject/joe.scale/joe.scale.gid\n'
        'DIRECTORY: capture/subject/joe.scale/images/\n'
        'PROBLEM_DEF: probspec/human/human.pdf\n'
        'LIMITS: probspec/human/human.lmt\n'
        'FILE: probspec/human/human.sdf\n'
        'FILE: probspec/human/human.sch\n'
        'BODY_DEF: subject/joe/joe.bdf\n'
        'FILE: subject/joe/joe.ppf\n'
        'FILE: calib/test.cal\n'
      ),
      ('tarprj', '-t', '/my/workspace/capture/subject/tim.rom.prj'): (
        'FILE: capture/subject/tim.rom/tim.rom.rtparams\n'
      ),
      ('tarprj', '-t', '/my/workspace/capture/subject/bob.scale.prj'): (
        'BODY_DEF: subject/bob/bob.bdf\n'
        'FILE: subject/bob/bob.ppf\n'
      ),
      ('tarprj', '-t', '/my/workspace/capture/subject/albert.scale.prj'): (
        'BODY_DEF: /my/workspace/subject/albert/albert.bdf\n'
      ),
      (
        'taranim',
        '-t',
        '/my/workspace/capture/takes/test1/test1.secondary.anm'
      ): '',
      ('taranim','-t','/my/workspace/anim/char1.anm'): (
        'MODEL: model/char1.mdl\n'
        'MAPPING: dof/char1.dof\n'
      ),
      ( 'sh', '-c',
        ('gunzip '
         '-c '
         '/my/workspace/scaling/tim.scl.gz '
         '>/my/workspace/scaling/tim.scl.tartake_1234'
        )
      ): '',
      (
        'tagfile',
        '-getfile',
        'INCLUDE.FILE_NAME',
        '/my/workspace/scaling/tim.scl.tartake_1234'
      ) : '',
      (
        'tagfile',
        '-getfile',
        'INCLUDE.FILE_NAME',
        '/my/workspace/probspec/human/human.pdf'
      ) : '',
      (
        'tagfile',
        '-getfile',
        'INCLUDE.FILE_NAME',
        '/my/workspace/probspec/human/human.lmt'
      ) : '',
      ('tarcpj', '-t', '/my/workspace/capture/takes/test1/test1.cpj'): (
        'MODEL: capture/takes/test1/test1.target.mdl\n'
        'MAPPING: capture/takes/test1/test1.target.dof\n'
        'MODEL: capture/takes/test1/test1.source.mdl\n'
        'MAPPING: capture/takes/test1/test1.source.dof\n'
        'MOTION: capture/takes/test1/test1.bmo\n'
        'SCALING: capture/takes/test1/test1.source.scl\n'
      ),
      ('tarcpj', '-t', '/my/workspace/capture/takes/test3/test3.cpj'): (
        'SCALING: scaling/tim.scl'
      ),
      (
        'tarmdl',
        '-t',
        '/my/workspace/capture/takes/test1/test1.target.mdl'
      ) : (
        'MODEL: model/char1.mdl\n'
        'MODEL: model/char2.mdl\n'
      ),
      (
        'tarmdl',
        '-t',
        '/my/workspace/capture/takes/test1/test1.source.mdl'
      ) : (
        'MODEL: model/bob.mdl\n'
        'MODEL: model/joe.mdl\n'
      ),
      ('tarmdl', '-t', '/my/workspace/model/char1.mdl') : (
        'FILE: model/texture/char1.tif\n'
      ),
      ('tarmdl', '-t', '/my/workspace/model/char2.mdl') : '',
      ('tarmdl', '-t', '/my/workspace/model/bob.mdl') : '',
      ('tarmdl', '-t', '/my/workspace/model/joe.mdl') : '',
      ('tarmdl', '-t', '/my/workspace/test.mdl') : '',
      (
        'tagfile',
        '-getfile',
        'INCLUDE.FILE_NAME',
        '/my/workspace/capture/takes/test1/test1.target.dof'
      ) : (
        '/my/workspace/dof/char1.dof\n'
        '/my/workspace/dof/char2.dof\n'
      ),
      (
        'tagfile',
        '-getfile',
        'INCLUDE.FILE_NAME',
        '/my/workspace/capture/takes/test1/test1.source.dof'
      ) : (
        '/my/workspace/dof/joe.dof\n'
        '/my/workspace/dof/bob.dof\n'
      ),
      (
        'tagfile',
        '-getfile',
        'INCLUDE.FILE_NAME',
        '/my/workspace/test.bdf'
      ) : '',
      (
        'tagfile',
        '-getfile',
        'INCLUDE.FILE_NAME',
        '/my/workspace/capture/takes/test1/test1.source.scl'
      ) : (
        '/my/workspace/scaling/joe.scl\n'
        '/my/workspace/scaling/bob.scl\n'
        '/my/workspace/scaling/missing.scl\n'
      ),
      (
        'tagfile',
        '-getfile',
        'INCLUDE.FILE_NAME',
        '/my/workspace/subject/joe/joe.bdf'
      ): '',
      (
        'tagfile',
        '-getfile',
        'INCLUDE.FILE_NAME',
        '/my/workspace/subject/bob/bob.bdf'
      ): '',
      (
        'tagfile',
        '-getfile',
        'INCLUDE.FILE_NAME',
        '/my/workspace/test.dof'
      ): '',
      (
        'tagfile',
        '-getfile',
        'INCLUDE.FILE_NAME',
        '/my/workspace/test.scl'
      ): '',
    }

  def expandVars(self,path):
    if path == '$GIANT_DATA_DIR/calib/test.cal':
      return '/my/workspace/calib/test.cal'
    debug('path: ' + str(path) + '\n')
    assert False

  def runAndReturnOutput(self,args):
    self.executed_commands.append(args)
    if args[0:3] == ['tagfile','-getfile','INCLUDE.FILE_NAME']:
      path = args[3]
      if path in self.leaf_files:
        return ''
    if tuple(args) in self.table:
      return self.table[tuple(args)]
    stderr.write('args: %s\n'%str(tuple(args)))
    assert False

  def readFile(self,path):
    if path == '/my/workspace/capture/takes/test1/test1.rtparams':
      return (
        'PROBLEM_NAME: hchxd\n'
        'CAMERA_CALIBRATION: $GIANT_DATA_DIR/calib/test.cal\n'
        'CALIBRATION_ERROR: 0.56\n'
        'PATTERN_TOLERANCE: 20\n'
        'GRAPHICMODEL: /usr/tmp/rttrack/rttrack_model.mdl\n'
        'REFERENCE_OFFSET: 0 0 0\n'
        'ACQUISITION_RATE: 60\n'
        'FILTER_LEVEL: 0.46\n'
        'MIN_VISIBILITY: 2\n'
        'AUTO_SCALE: 0\n'
        'TIMECODE_24FPS: 0\n'
        'RECORD_AUDIO: 0\n'
        'DRIFT_COMPENSATION: 0\n'
      )

      assert False
    debug('path: '+str(path)+'\n')
    assert False

  def existingFiles(self):
    return [
      '/my/workspace/anim/char1.anm',
      '/my/workspace/calib/test.cal',
      '/my/workspace/calib/test.imucal',
      '/my/workspace/capture/subject/bob.scale.prj',
      '/my/workspace/capture/subject/joe.scale.prj',
      '/my/workspace/capture/subject/joe.scale/joe.scale.raw',
      '/my/workspace/capture/subject/joe.scale/joe.scale.gid',
      '/my/workspace/capture/subject/tim.rom.prj',
      '/my/workspace/capture/subject/tim.rom/tim.rom.rtparams',
      '/my/workspace/capture/subject/albert.scale.prj',
      '/my/workspace/capture/takes/test1.prj',
      '/my/workspace/capture/takes/test1/test1.raw',
      '/my/workspace/capture/takes/test1/test1.globs',
      '/my/workspace/capture/takes/test1/test1.psf',
      '/my/workspace/capture/takes/test1/test1.secondary.anm',
      '/my/workspace/capture/takes/test1/test1.cpj',
      '/my/workspace/capture/takes/test1/test1.rtparams',
      '/my/workspace/capture/takes/test1/test1.target.dof',
      '/my/workspace/capture/takes/test1/test1.bmo',
      '/my/workspace/capture/takes/test1/test1.bdf',
      '/my/workspace/capture/takes/test1/test1.pdf',
      '/my/workspace/capture/takes/test1/test1.mdl',
      '/my/workspace/capture/takes/test1/test1.scl',
      '/my/workspace/capture/takes/test1/test1.lmt',
      '/my/workspace/capture/takes/test1/test1.dof',
      '/my/workspace/capture/takes/test1/test1.ppf',
      '/my/workspace/capture/takes/test1/test1.globs',
      '/my/workspace/capture/takes/test1/test1.cpj',
      '/my/workspace/capture/takes/test1/test1.raw',
      '/my/workspace/capture/takes/test1/test1.rtparams',
      '/my/workspace/capture/takes/test1/test1.target.dof',
      '/my/workspace/capture/takes/test1/test1.target.mdl',
      '/my/workspace/capture/takes/test1/test1.secondary.anm',
      '/my/workspace/capture/takes/test1/test1.source.dof',
      '/my/workspace/capture/takes/test1/test1.source.mdl',
      '/my/workspace/capture/takes/test1/test1.source.scl',
      '/my/workspace/capture/takes/test1/test1.psf',
      '/my/workspace/capture/takes/test2.prj',
      '/my/workspace/capture/takes/test2/test2.globs',
      '/my/workspace/capture/takes/test2/test2.psf',
      '/my/workspace/capture/takes/test2/test2.raw',
      '/my/workspace/capture/takes/test3/test3.cpj',
      '/my/workspace/capture/takes/test3/test3.psf',
      '/my/workspace/capture/takes/test4/test4.psf',
      '/my/workspace/capture/test2.prj',
      '/my/workspace/capture/test.prj',
      '/my/workspace/char1.anm',
      '/my/workspace/charmap1.cm',
      '/my/workspace/model/bob.mdl',
      '/my/workspace/model/char1.mdl',
      '/my/workspace/model/char2.mdl',
      '/my/workspace/model/joe.mdl',
      '/my/workspace/model/texture/char1.tif',
      '/my/workspace/probspec/human/human.lmt',
      '/my/workspace/probspec/human/human.pdf',
      '/my/workspace/probspec/human/human.sch',
      '/my/workspace/probspec/human/human.sdf',
      '/my/workspace/scaling/tim.scl.gz',
      '/my/workspace/subject/albert/albert.bdf',
      '/my/workspace/subject/bob/bob.bdf',
      '/my/workspace/subject/bob/bob.ppf',
      '/my/workspace/subject/joe/joe.bdf',
      '/my/workspace/subject/joe/joe.ppf',
      '/my/workspace/test.anm',
      '/my/workspace/test.cm',
      '/my/workspace/test.bdf',
      '/my/workspace/test.bmo',
      '/my/workspace/test.dof',
      '/my/workspace/test.mdl',
      '/my/workspace/test.ppf',
      '/my/workspace/test.prj',
      '/my/workspace/test.scl',
      '/blah/blah.psf',
    ]

  def filesInDirectory(self,dir_path):
    result = []

    for path in self.existingFiles():
      if path.startswith(dir_path + '/'):
        filename = removePrefix(path,dir_path + '/')
        if '/' not in filename:
          result.append(filename)

    if len(result) != 0:
      for filename in result:
        assert '/' not in filename
      return result

    stderr.write('filesInDirectory: dir_path: %s\n'%str(dir_path))
    assert False

  def makeLeafFiles(self):
    return [
      '/my/workspace/dof/char1.dof',
      '/my/workspace/dof/char2.dof',
      '/my/workspace/dof/joe.dof',
      '/my/workspace/dof/bob.dof',
      '/my/workspace/scaling/joe.scl',
      '/my/workspace/scaling/bob.scl',
      '/my/workspace/subject/albert/albert.bdf'
    ]

  def nonExistingFiles(self):
    non_existing_files = [
      '/my/workspace/capture/takes/test3.prj',
      '/my/workspace/capture/takes/test4.prj',
      '/my/workspace/scaling/tim.scl',
      '/my/workspace/scaling/missing.scl',
      '/my/workspace/scaling/missing.scl.gz'
    ]
    return non_existing_files
  def fileExists(self,path):
    if path in self.leaf_files:
      return True
    if path in self.existingFiles():
      return True
    if path in self.nonExistingFiles():
      return False
    debug('path=%s\n'%str(path))
    assert False

  def directoryExists(self,path):
    if path == '/my/workspace/capture/subject/joe.scale/images/':
      return True
    debug('path: %s\n'%str(path))
    assert False

  def deleteFile(self,path):
    if path == '/my/workspace/scaling/tim.scl.tartake_1234':
      return
    debug('path: %s\n'%str(path))
    assert False

  def log(self,message):
    self.log_messages.append(message)

def hasNoDuplicates(items):
  return len(set(map(tuple,items))) == len(items)

def checkPaths(relative_files,expected_result):
  if sorted(relative_files) != sorted(expected_result):
    debug('relative_files:\n%s\n'%str('\n'.join(map(str,sorted(relative_files)))))
  any_missing = False
  for path in expected_result:
    if path not in relative_files:
      if not any_missing:
        debug('Missing Files:\n')
        any_missing = True
      debug('  %s\n'%path)

  for path in relative_files:
    if path not in expected_result:
      debug('extra [%s]\n'%path)

  assert sorted(relative_files) == sorted(expected_result)

def test1Files():
  return [
    'anim/char1.anm',
    'calib/test.cal',
    'capture/subject/bob.scale.prj',
    'capture/subject/joe.scale.prj',
    'capture/subject/joe.scale/joe.scale.raw',
    'capture/subject/joe.scale/joe.scale.gid',
    'capture/subject/joe.scale/images/',
    'capture/subject/tim.rom.prj',
    'capture/subject/tim.rom/tim.rom.rtparams',
    'capture/takes/test1.prj',
    'capture/takes/test1/test1.bmo',
    'capture/takes/test1/test1.cpj',
    'capture/takes/test1/test1.globs',
    'capture/takes/test1/test1.psf',
    'capture/takes/test1/test1.raw',
    'capture/takes/test1/test1.rtparams',
    'capture/takes/test1/test1.secondary.anm',
    'capture/takes/test1/test1.source.dof',
    'capture/takes/test1/test1.source.mdl',
    'capture/takes/test1/test1.source.scl',
    'capture/takes/test1/test1.target.dof',
    'capture/takes/test1/test1.target.mdl',
    'capture/takes/test1/test1.bdf',
    'capture/takes/test1/test1.pdf',
    'capture/takes/test1/test1.mdl',
    'capture/takes/test1/test1.scl',
    'capture/takes/test1/test1.lmt',
    'capture/takes/test1/test1.dof',
    'capture/takes/test1/test1.ppf',
    'charmap1.cm',
    'dof/bob.dof',
    'dof/char1.dof',
    'dof/char2.dof',
    'dof/joe.dof',
    'model/bob.mdl',
    'model/char1.mdl',
    'model/char2.mdl',
    'model/joe.mdl',
    'model/texture/char1.tif',
    'probspec/human/human.lmt',
    'probspec/human/human.pdf',
    'probspec/human/human.sch',
    'probspec/human/human.sdf',
    'scaling/bob.scl',
    'scaling/joe.scl',
    'subject/bob/bob.bdf',
    'subject/bob/bob.ppf',
    'subject/joe/joe.bdf',
    'subject/joe/joe.ppf'
  ]


def testGenerateRelativeFiles1():
  runner = FakeRunner()
  take_dir1 = '/my/workspace/capture/takes/test1'
  take_dir2 = '/my/workspace/capture/takes/test2'
  take_dirs = [take_dir1,take_dir2]
  data_dir = '/my/workspace'
  relative_files = generateRelativeFiles(take_dirs,data_dir,runner)

  extra_files = [
    'capture/takes/test2/test2.psf',
    'capture/takes/test2.prj',
    'capture/takes/test2/test2.raw',
    'capture/takes/test2/test2.globs'
  ]

  expected_result = test1Files() + extra_files
  assert hasNoDuplicates(runner.executed_commands)
  checkPaths(relative_files,expected_result)


def testGenerateRelativeFiles2():
  runner = FakeRunner()
  take_dir1 = '/my/workspace/capture/takes/test1'
  take_dir2 = '/my/workspace/capture/takes/test1'
  take_dirs = [take_dir1,take_dir2]
  data_dir = '/my/workspace'
  relative_files = generateRelativeFiles(take_dirs,data_dir,runner)
  expected_result = test1Files()
  assert hasNoDuplicates(runner.executed_commands)
  checkPaths(relative_files,expected_result)


def testGenerateRelativeFiles3():
  runner = FakeRunner()
  take_dir = '/my/workspace/capture/takes/test3'
  take_dirs = [take_dir]
  data_dir = '/my/workspace'
  relative_files = generateRelativeFiles(take_dirs,data_dir,runner)
  expected_result = [
    'capture/subject/bob.scale.prj',
    'capture/takes/test3/test3.cpj',
    'capture/takes/test3/test3.psf',
    'subject/bob/bob.bdf',
    'subject/bob/bob.ppf',
    'scaling/tim.scl.gz'
  ]
  assert hasNoDuplicates(runner.executed_commands)
  checkPaths(relative_files,expected_result)


def testGenerateRelativeFiles4():
  # Test the case where we have a subject prj that uses absolute paths.
  runner = FakeRunner()
  take_dir = '/my/workspace/capture/takes/test4'
  take_dirs = [take_dir]
  data_dir = '/my/workspace'
  relative_files = generateRelativeFiles(take_dirs,data_dir,runner)

  expected_result = [
    'capture/takes/test4/test4.psf',
    'capture/subject/albert.scale.prj',
    '/my/workspace/subject/albert/albert.bdf',
    'calib/test.imucal'
  ]

  expected_log = [
    'Unknown file type: IMUCAL\n'
  ]

  assert hasNoDuplicates(runner.executed_commands)
  checkPaths(relative_files,expected_result)
  assert runner.log_messages == expected_log


def formatSpecs(specs):
  lines = []
  for type_name,relative_path in specs:
    lines.append('%s: %s\n'%(type_name,relative_path))
  return ''.join(lines);


def topFilesText(take_dirs,data_dir,runner):
  return formatSpecs(generateTopFiles(take_dirs,data_dir,runner))


def testGenerateTopFiles():
  runner = FakeRunner()
  take_dir = '/my/workspace/capture/takes/test1'
  take_dirs = [take_dir]
  data_dir = '/my/workspace'
  output = topFilesText(take_dirs,data_dir,runner)
  expected_output = (
    'FILE: capture/takes/test1.prj\n'
    'PSF: capture/takes/test1/test1.psf\n'
    'CPJ: capture/takes/test1/test1.cpj\n'
    'ANIMATION: capture/takes/test1/test1.secondary.anm\n'
    'RTPARAMS: capture/takes/test1/test1.rtparams\n'
    'FILE: capture/takes/test1/test1.globs\n'
  )

  if output != expected_output:
    debug('output:\n' + str(output) + '\n')

  assert(output == expected_output)


def runUnitTests():
  def testAddTo():
    l = []
    addPathsTo(l,['a'])
    assert l==['a']
  def testParseArgsWithListOptionAndNoDirectories():
    args = ['tartake','-l']
    result = parseArgs(args)
    assert result.error==noTakeDirectoryGivenErrorMessage()
  def testParseArgsWithTopOption():
    args = ['tartake','-t','take1','take2']
    result = parseArgs(args)
    assert result.dir_paths == ['take1','take2']
    assert result.top_only
  def testParseArgsWithNoOptions():
    args = ['tartake','some/dir']
    result = parseArgs(args)
    assert result.error==None
    assert result.dir_paths==['some/dir']
  def testParseArgsWithTwoDirectoriesButNoTarFileOption():
    args = ['tartake','some/dir1','some/dir2']
    result = parseArgs(args)
    assert result.error==noTarFileOptionErrorMessage()
  def testParseArgsWithTwoDirectories():
    args = ['tartake','-c','blah.tar.gz','some/dir1','some/dir2']
    result = parseArgs(args)
    assert result.error==None
    assert result.dir_paths==['some/dir1','some/dir2']
    assert result.output_tar_path == 'blah.tar.gz'
  def testParseArgsWithListOption():
    args = ['tartake','-l','some/dir']
    result = parseArgs(args)
    assert result.error==None
    assert result.list_only==True
    assert result.dir_paths==['some/dir']
  def testParseArgsWithListOptionAndTwoDirectories():
    args = ['tartake','-l','some/dir1','some/dir2']
    result = parseArgs(args)
    assert result.error is None
    assert result.list_only == True
    assert result.dir_paths == ['some/dir1','some/dir2']
  def testMakeTarFilePathFrom():
    result = makeTarFilePathFrom('some_dir')
    assert result=='some_dir.tar.gz'
  def testMakeTarFilePathFrom2():
    result = makeTarFilePathFrom('/a/b/c/some_dir')
    assert result=='some_dir.tar.gz'

  global running_unit_tests
  running_unit_tests = True
  testAddTo()
  testParseArgsWithListOptionAndNoDirectories()
  testParseArgsWithTopOption()
  testParseArgsWithNoOptions()
  testParseArgsWithTwoDirectories()
  testParseArgsWithTwoDirectoriesButNoTarFileOption()
  testParseArgsWithListOption()
  testParseArgsWithListOptionAndTwoDirectories()
  testMakeTarFilePathFrom()
  testMakeTarFilePathFrom2()
  testGenerateRelativeFiles1()
  testGenerateRelativeFiles2()
  testGenerateRelativeFiles3()
  testGenerateRelativeFiles4()
  testGenerateTopFiles()
  running_unit_tests = False


def usageText():
  return (
    '\n'
    'Usage: tartake <take directory>\n'
    '  Creates a .tar.gz file with all files related to the given take.\n'
    '\n'
    'Usage: tartake -c <tar file> <take directory> [<take directory> ...]\n'
    '  Creates a .tar.gz file with all files related to the given takes.\n'
    '\n'
    'Usage: tartake -l <take directory> [<take directory> ...]\n'
    '  Lists all files that would be included in the tar file.\n'
    '\n'
    'Usage: tartake -t <take directory> [<take directory> ...]\n'
    '  Lists only the files that are directly part of the take.\n'
    '\n'
  )

def determineDataDir():
  if 'GIANT_DATA_DIR' not in os.environ:
    stderr.write('\n')
    stderr.write('GIANT_DATA_DIR is not set.\n')
    stderr.write('\n')
    return None

  data_dir = os.environ['GIANT_DATA_DIR']

  return data_dir

def printErrorTo(f,message):
  f.write('\n')
  f.write(message+'\n')
  f.write('\n')


def main(argv):
  EXIT_SUCCESS=0
  EXIT_FAILURE=1

  if len(argv)<2:
    stderr.write(usageText())
    return EXIT_FAILURE

  data_dir = determineDataDir()

  if data_dir is None:
    return EXIT_FAILURE

  params = parseArgs(argv)

  if params.error is not None:
    printErrorTo(stderr,params.error);
    return EXIT_FAILURE

  dir_paths = [os.path.abspath(path) for path in params.dir_paths]

  if params.top_only:
    try:
      stdout.write(topFilesText(dir_paths,data_dir,system_runner))
    except UserException as e:
      printErrorTo(stderr,e.message)
      return EXIT_FAILURE
    return EXIT_SUCCESS

  try:
    relative_files = generateRelativeFiles(dir_paths,data_dir,system_runner)
  except OSError as e:
    printErrorTo(stderr,str(e))
    return EXIT_FAILURE
  except UserException as e:
    printErrorTo(stderr,e.message)
    return EXIT_FAILURE

  if len(relative_files)==0:
    printErrorTo(stderr,'The take contains no files.')
    return EXIT_FAILURE

  if params.list_only:
    stdout.write('\n'.join(relative_files)+'\n')
  else:
    if params.output_tar_path:
      tarfile_path = params.output_tar_path
    else:
      assert(len(dir_paths)==1)
      tarfile_path = makeTarFilePathFrom(dir_paths[0])
    exit_code =\
      subprocess.call(
        ('tar','zcvf',tarfile_path,'-C',data_dir)+tuple(relative_files)
      )
    if exit_code!=0:
      return exit_code
    stderr.write('\n')
    stderr.write('Created %s\n'%(os.path.realpath(tarfile_path),))
    stderr.write('\n')

  return EXIT_SUCCESS

if __name__ == "__main__":
  exit_code = main(sys.argv)
  sys.exit(exit_code)
