#!/bin/sh
''':'
# This code will be treated as a comment in Python, but will be executed
# by the shell, allowing us to do some special handling before the python
# interpreter starts.
PYTHONPATH="" exec python3 "$0" "$@"
' '''

#
# 2023-11-13  Now ignores PYTHONPATH
# 2022-08-23  Changed to use python3 instead of python
# 2021-08-22  Fixed mssing motion file
# 2021-05-19  Initial implementation
#

import sys
import os
import io
import subprocess
import re
from sys import stdout, stderr


def isCSPTypeString(type_string):
  return type_string == 'Bio Color Setup Project File'


def isCSPVersionString(version_string):
  return version_string == 'v1.00'


def list_get(l, i):
  if i < 0:
    assert False
  if i >= len(l):
    return None
  return l[i]

def existingExtension(full_path,runner):
  if runner.fileExists(full_path):
    return ''
  if runner.fileExists(full_path + '.gz'):
    assert False # Not tested
    return '.gz'
  stderr.write('Missing file ' + str(full_path) + '\n')
  return None


def checkFullPath(path):
  assert path != ''
  assert path != '/'
  assert '/..' not in path
  assert 'data/data' not in path
  if not path.startswith('/'):
    debug('path: %s\n'%str(path))
    assert False


def addPathTo(files,new_file):
  assert new_file!=''
  if new_file in files:
    return
  files.append(new_file)


def addPathsTo(files,new_files):
  assert type(new_files)==list
  for path in new_files:
    addPathTo(files,path)


def getTagFiles(tag, path, runner):
  output = runner.runAndReturnOutput(['tagfile','-getfile',tag,path])
  split_output = output.split('\n')
  split_output = [x for x in split_output if x != '']
  return split_output


def isAMatch(filename, filename_pattern):
  i = filename_pattern.find('#')

  if i >= 0:
    left = filename_pattern[:i]
    right = filename_pattern[i+1:]
    if filename.startswith(left) and filename.endswith(right):
      mid = filename[i:-len(right)]
      if mid == '':
        return False
      try:
        int(mid, 10)
        return True
      except RuntimeError as e:
        assert False
    else:
      return False
  else:
    return filename == filename_pattern

def patternMatches(all_filenames, filename_pattern):
  matches = []
  for filename in all_filenames:
    if isAMatch(filename, filename_pattern):
      matches.append(filename)
  return matches

def filesMatchingPattern(path_pattern, runner):
  dirname = os.path.dirname(path_pattern)
  filename_pattern = os.path.basename(path_pattern)
  all_filenames = runner.filesInDirectory(dirname)
  matching_filenames = patternMatches(all_filenames, filename_pattern)
  matching_paths = []
  for filename in matching_filenames:
    matching_paths.append(os.path.join(dirname, filename))
  return matching_paths

class CSPTagHandler:
  def __init__(self, specs, data_dir, runner):
    self.specs = specs
    self.data_dir = data_dir
    self.runner = runner

  def handleHeader(self, type_string, version_string):
    if not isCSPTypeString(type_string):
      raise UserException('File is not a color setup project file.')

    if not isCSPVersionString(version_string):
      assert False # not implemented

  def addSpec(self, type_name, value):
    expanded_path = self.runner.expandVars(value)
    path = workspaceRelativePath(expanded_path, self.data_dir)
    self.specs.append((type_name, path))

  def handleLineValue(self, tag_path, value):
    tag0 = list_get(tag_path, 0)
    if tag0 == 'CAMERA_SET':
      tag1 = list_get(tag_path, 1)

      camera_set_file_members = (
        'RAW_DATA', 'CALIBRATION', 'CALIBRATION_FILE', 'GLOB_IDS', 'MOVIE'
      )

      if tag1 in camera_set_file_members:
        self.addSpec('FILE', value)
      elif tag1 == 'IMAGE_PATTERN':
        expanded_path = self.runner.expandVars(value)
        for expanded_path in filesMatchingPattern(expanded_path, self.runner):
          path = workspaceRelativePath(expanded_path, self.data_dir)
          self.specs.append(('FILE', path))
      elif tag1 == 'ANIMATION':
        tag2 = list_get(tag_path, 2)
        if tag2 in ('MODEL',):
          self.addSpec('MODEL', value)
    else:
      tag_type_table = {
        'PROBLEM_DEF':      'PROBLEM_DEF',
        'PROBLEM_DEF_FILE': 'PROBLEM_DEF',
        'BODY_DEF_FILE':    'BODY_DEF',
        'BODY_DEF':         'BODY_DEF',
        'LIMITS':           'LIMITS',
        'SEARCH_PARAMS':    'FILE',
        'MOTION':           'FILE',
        'GFX_MODEL':        'MODEL',
        'GFX_SCALING':      'SCALING'
      }

      tag_type = tag_type_table.get(tag0)

      if tag_type is not None:
        self.addSpec(tag_type, value)

  def shouldParseSection(self, tag_path):
    if tag_path == ['CAMERA_SET']:
      return True
    if tag_path == ['CAMERA_SET', 'ANIMATION']:
      return True
    return False

  def handleSectionValue(self, tag_path, value):
    pass


def makeSpec(line):
  m = re.match('([^:]+): *(.*)',line)
  if m is None:
    debug('line: %s\n'%str(line))
    return None
  type_name = m.groups()[0]
  relative_path = m.groups()[1]
  return (type_name,relative_path)


def makeSpecs(output):
  result = output.split('\n')
  result = [makeSpec(x) for x in result if x !='']
  result = [x for x in result if x is not None]
  return result


def getTopFilesWithCommand(command,path,runner):
  checkFullPath(path)
  args=[command,'-t',path]
  return makeSpecs(runner.runAndReturnOutput(args))


def getTopMDLFiles(path,runner):
  return getTopFilesWithCommand('tarmdl',path,runner)


def getTopCSPFiles(full_file_path, runner, data_dir):
  specs = []

  with open(full_file_path,'r') as f:
    parseTagFile(f, CSPTagHandler(specs, data_dir, runner))

  return specs


def throwException(e):
  raise e


class UserException(Exception):
  def __init__(self,message):
    self.message = message


def filesInDirectory(dir_path):
  walk_result = os.walk(dir_path,onerror=throwException)
  try:
    for dirpath,dirnames,filenames in walk_result:
      return filenames
  except OSError as e:
    message = e.strerror + ': ' + e.filename
    raise UserException(message)


class SystemRunner:
  def fileExists(self,path):
    return os.path.exists(path) and os.path.isfile(path)

  def runAndReturnOutput(self,args):
    assert not running_unit_tests
    #return subprocess.check_output(args)
    # Using this instead of check_output for compatibility with python 2.6
    return str(subprocess.Popen(
      ['/usr/bin/env']+args, stdout=subprocess.PIPE
    ).communicate()[0], 'utf-8')

  def expandVars(self,path):
    return os.path.expandvars(path)

  def filesInDirectory(self,dir_path):
    return filesInDirectory(dir_path)


def uncompressedPath(path):
  if not path.endswith('.gz'):
    return path
  return path[:-3]


def getIncludes(base_file,runner):
  uncompressed_base_file = uncompressedPath(base_file)

  if uncompressed_base_file != base_file:
    tmp_uncompressed_base_file = \
      uncompressed_base_file + '.tartake_' + str(runner.getpid())
    runner.runAndReturnOutput(
      ['sh','-c','gunzip -c '+base_file+' >'+tmp_uncompressed_base_file]
    )
    tagfile_file = tmp_uncompressed_base_file
  else:
    tagfile_file = base_file

  include_file_names = getTagFiles('INCLUDE.FILE_NAME',tagfile_file,runner)

  if uncompressed_base_file != base_file:
    runner.deleteFile(tmp_uncompressed_base_file)
    #rm -f tmp_uncompressed_base_file

  for path in include_file_names:
    checkFullPath(path)

  return include_file_names


def getTopIncludes(input_path,type_name,data_dir,runner):
  full_include_paths = getIncludes(input_path,runner)

  relative_include_paths = \
    [workspaceRelativePath(path,data_dir) for path in full_include_paths]

  return [(type_name,path) for path in relative_include_paths]


class Generator:
  def __init__(self, data_dir, runner):
    self.data_dir = data_dir
    self.runner = runner
    self.unexpanded_specs = []
    self.expanded_specs = []
    self.relative_files = []
    self.log_messages = []

  def addUnexpandedSpec(self, spec):
    if spec not in self.unexpanded_specs and spec not in self.expanded_specs:
      self.unexpanded_specs.append(spec)

  def addFiles(self,new_files):
    addPathsTo(self.relative_files,new_files)

  def isLeafType(self,file_type):
    # Leaf types do not reference other files.
    leaf_types = [ 'MOTION', 'FILE', 'DIRECTORY' ]
    return file_type in leaf_types

  def isSimpleType(self,file_type):
    # Simple types only have includes of the same type.
    simple_types = ['PROBLEM_DEF', 'BODY_DEF', 'MAPPING', 'SCALING', 'LIMITS']
    return file_type in simple_types

  def log(self,message):
    self.log_messages.append(message)

  def topSpecs(self, file_type, full_file_path):
    if self.isLeafType(file_type):
      return []
    if self.isSimpleType(file_type):
      include_specs = \
        getTopIncludes(full_file_path,file_type,self.data_dir,self.runner)
      return include_specs
    if file_type == 'PSF':
      return getTopPSFFiles(full_file_path,self.runner)
    if file_type == 'ANIMATION':
      return getTopANMFiles(full_file_path,self.runner)
    if file_type == 'PROJECT':
      return getTopPRJFiles(full_file_path,self.runner)
    if file_type == 'CPJ':
      return getTopCPJFiles(full_file_path,self.runner)
    if file_type == 'MODEL':
      return getTopMDLFiles(full_file_path,self.runner)
    if file_type == 'RTPARAMS':
      return getTopRTParamsFiles(full_file_path,self.runner,self.data_dir)
    if file_type == 'CSP':
      return getTopCSPFiles(full_file_path, self.runner, self.data_dir)
    self.log('Unknown file type: %s\n'%str(file_type))
    return []

  def addUnexpandedSpecs(self,specs):
    for spec in specs:
      self.addUnexpandedSpec(spec)

  def addUnexpandedTopFiles(self, file_type, full_file_path):
    top_specs = self.topSpecs(file_type,full_file_path)
    self.addUnexpandedSpecs(top_specs)

  def expand(self, file_type, full_file_path, relative_file_path):
    checkFullPath(full_file_path)
    self.addFiles([relative_file_path])
    self.addUnexpandedTopFiles(file_type,full_file_path)

  def expandFiles(self):
    while len(self.unexpanded_specs) > 0:
      file_type,relative_path = self.unexpanded_specs[0]
      del self.unexpanded_specs[0]
      if (file_type, relative_path) in self.expanded_specs:
        assert False
        debug('%s has already been expanded.\n'%str(relative_path))
        assert False
      self.expanded_specs.append((file_type,relative_path))
      if relative_path not in self.relative_files:
        # We want existing_file_path to end up being an existing file,
        # but specified relative to the workspace.
        # However, it's not good enough to use workspaceRelativePath()
        # because the relative path may actually be an absolute path,
        # and if it is, we want the result to also be an absolute path.
        file_path = os.path.join(self.data_dir, relative_path)

        if file_type == 'DIRECTORY':
          assert False
          if self.runner.directoryExists(file_path):
            existing_relative_path = relative_path
            existing_file_path = file_path
          else:
            existing_relative_path = None
            existing_file_path = None
        else:
          extension = existingExtension(file_path, self.runner)

          if extension is None:
            existing_file_path = None
            existing_relative_path = None
          else:
            existing_file_path = file_path + extension
            existing_relative_path = relative_path + extension

        if existing_relative_path is not None:
          self.expand(file_type,existing_file_path,existing_relative_path)
        else:
          self.log('Missing file %s\n'%str(file_path))


def show(expr):
  parent_frame = sys._getframe(1)
  parent_locals = parent_frame.f_locals
  expr_value = eval(expr,parent_locals)
  stderr.write(expr+': '+str(expr_value)+'\n')
  stderr.flush()


def makeOutputTarPath(csp_path):
  basename = os.path.basename(csp_path)
  return basename + '.tar.gz'


def determineDataDir():
  if 'GIANT_DATA_DIR' not in os.environ:
    stderr.write('\n')
    stderr.write('GIANT_DATA_DIR is not set.\n')
    stderr.write('\n')
    return None

  data_dir = os.environ['GIANT_DATA_DIR']

  return data_dir


def workspaceRelativePath(path, data_dir):
  if path.startswith(data_dir+'/'):
    return path[len(data_dir)+1:]
  return path


def generateRelativeCSPFiles(csp_path, data_dir, runner):
  generator = Generator(data_dir, runner)
  relative_csp_path = workspaceRelativePath(csp_path, data_dir)
  generator.addUnexpandedSpec(('CSP', relative_csp_path))
  generator.expandFiles()
  return generator.relative_files


def makeRelativeCSPFiles(csp_path, data_dir):
  return generateRelativeCSPFiles(csp_path, data_dir, SystemRunner())


def parseTagFile(f, tag_handler):
  line_number = 0
  type_string = f.readline().rstrip()
  version_string = f.readline().rstrip()
  tag_handler.handleHeader(type_string, version_string)
  tag_path = []
  maybe_section_value = None
  section_value_depth = 0
  for line in f:
    if line.lstrip().startswith('#'):
      pass
    else:
      line = line.rstrip()
      colon_position = line.find(':')
      if colon_position >= 0:
        if maybe_section_value is not None:
          maybe_section_value += line + '\n'
        else:
          tag = line[:colon_position].lstrip()
          value = line[colon_position+1:].lstrip()
          tag_path.append(tag.lstrip())
          tag_handler.handleLineValue(tag_path, value)
          del tag_path[-1]
      elif line.endswith('{'):
        if maybe_section_value is not None:
          section_value_depth += 1
          maybe_section_value += line + '\n'
        else:
          tag = line[:-1].strip()
          tag_path.append(tag)
          if tag_handler.shouldParseSection(tag_path):
            pass
          else:
            assert maybe_section_value is None
            maybe_section_value = ''
            section_value_depth = 1
      elif line.lstrip() == '}':
        if maybe_section_value is not None:
          section_value_depth -= 1
          if section_value_depth == 0:
            tag_handler.handleSectionValue(tag_path, maybe_section_value)
            maybe_section_value = None
            del tag_path[-1]
          else:
            maybe_section_value += line + '\n'
        else:
          del tag_path[-1]
      else:
        if maybe_section_value is not None:
          maybe_section_value += line + '\n'
        else:
          # Got a non-taggged line in a section we are parsing.
          show('line')
          show('maybe_section_value')
          assert False # not implemented

# Unit tests
############

def testCreatingTarPath():
  csp_path = 'data/test/test.csp'
  tarfile_path = makeOutputTarPath(csp_path)
  assert tarfile_path == 'test.csp.tar.gz'


def testParsingTagFile():
  text = (
    'Bio Color Setup Project File\n'
    'v1.00\n'
    'TAG1: value1\n'
    'SECTION {\n'
    '  TAG2: value2\n'
    '}\n'
    'SECTION {\n'
    '  TAG3: value3\n'
    '}\n'
    'TEXT {\n'
    '  Here is some text\n'
    '  {\n'
    '    TAG: value\n'
    '  }\n'
    '  and more\n'
    '}\n'
    '#{\n'
    'TAG4: value4\n'
  )

  expected_line_values = [
    (['TAG1'], 'value1'),
    (['SECTION', 'TAG2'], 'value2'),
    (['SECTION', 'TAG3'], 'value3'),
    (['TAG4'], 'value4')
  ]

  expected_section_values = [
    (['TEXT'], '  Here is some text\n  {\n    TAG: value\n  }\n  and more\n')
  ]

  f = io.StringIO(unicode(text))

  class TagHandler:
    def __init__(self):
      self.header_was_specified = False
      self.line_values = []
      self.section_values = []

    def handleHeader(self, type_string, version_string):
      assert isCSPTypeString(type_string)
      assert isCSPVersionString(version_string)
      self.header_was_specified = True

    def handleLineValue(self, tag_path, value):
      self.line_values.append((tag_path[:], value))

    def shouldParseSection(self, tag_path):
      if tag_path == ['SECTION']:
        return True
      elif tag_path == ['TEXT']:
        return False
      else:
        assert False

    def handleSectionValue(self, tag_path, value):
      self.section_values.append((tag_path[:], value))

  def checkEqual(actual, expected):
    if actual == expected:
      return True
    else:
      show('actual')
      show('expected')
      return False
  tag_handler = TagHandler()
  parseTagFile(f, tag_handler)
  assert tag_handler.header_was_specified
  assert checkEqual(tag_handler.line_values, expected_line_values)
  assert checkEqual(tag_handler.section_values, expected_section_values)


def testIsAMatch():
  assert isAMatch('test2.jpg', 'test#.jpg')
  assert not isAMatch('test.jpg', 'test#.jpg')
  assert isAMatch('test.jpg', 'test.jpg')


def testListGet():
  assert list_get([], 0) is None

def runUnitTests():
  global running_unit_tests
  running_unit_tests = True
  testCreatingTarPath()
  testParsingTagFile()
  testIsAMatch()
  testListGet()
  running_unit_tests = False


def printErrorTo(f,message):
  f.write('\n')
  f.write(message+'\n')
  f.write('\n')


def formatSpecs(specs):
  lines = []
  for type_name,relative_path in specs:
    lines.append('%s: %s\n'%(type_name,relative_path))
  return ''.join(lines);


def generateTopFiles(csp_path, data_dir, runner):
  generator = Generator(data_dir,runner)
  generator.addUnexpandedTopFiles('CSP', csp_path)
  return generator.unexpanded_specs


def topFilesText(csp_path, data_dir):
  runner = SystemRunner()
  specs = getTopCSPFiles(csp_path, runner, data_dir)
  return formatSpecs(specs)


def usageText():
  return (
    '\n'
    'Usage: tarcsp <csp>\n'
    '  Creates a .tar.gz file with all files related to the given\n'
    '  Colorsetup project.\n'
    '\n'
    'Usage: tarcsp -l <csp>\n'
    '  Lists all files that would be included in the tar file.\n'
    '\n'
    'Usage: tarcsp -t <csp>\n'
    '  Lists only the files that are directly used by the .csp file.\n'
    '\n'
  )


def main(argv):
  EXIT_FAILURE=1
  EXIT_SUCCESS=0
  list_only = False
  top_only = False

  if list_get(argv, 1) == '-l':
    list_only = True
    del argv[1]
  elif list_get(argv, 1) == '-t':
    top_only = True
    del argv[1]

  if len(argv) != 2:
    stderr.write(usageText())
    return EXIT_FAILURE

  data_dir = determineDataDir()

  if data_dir is None:
    return EXIT_FAILURE

  csp_path = os.path.abspath(argv[1])
  tarfile_path = makeOutputTarPath(csp_path)

  if top_only:
    try:
      stdout.write(topFilesText(csp_path, data_dir))
    except UserException as e:
      printErrorTo(stderr,e.message)
      return EXIT_FAILURE
    return EXIT_SUCCESS

  try:
    relative_files = makeRelativeCSPFiles(csp_path, data_dir)
  except UserException as e:
    printErrorTo(stderr, e.message)
    return EXIT_FAILURE

  if list_only:
    stdout.write('\n'.join(relative_files)+'\n')
  else:
    exit_code =\
      subprocess.call(
        ('tar','zcvf',tarfile_path,'-C',data_dir)+tuple(relative_files)
      )

    if exit_code!=0:
      return exit_code

    stderr.write('\n')
    stderr.write('Created %s\n'%(os.path.realpath(tarfile_path),))
    stderr.write('\n')


if __name__ == '__main__':
  running_unit_tests=False
  sys.exit(main(sys.argv))
