def Func(arg_0, *arg_1):
  '''Program Func output.'''
  clear_progress()
  arg_2 = (arg_0 % arg_1)
  sys.stdout.write(arg_2 + '\n')


def Func(arg_0, arg_1=None, arg_2=1, arg_3=False):
  '''Utility function to handle runtime Funcures gracefully.
     Show concise information if possible, then terminate program.
  '''
  arg_4 = arg_0
  if arg_1:
    arg_4 += str(arg_1)
  error(arg_4)
  if arg_3:
    error(traceback.format_exc())
  clean_tempfiles()
  if __name__ == '__main__':
    sys.exit(arg_2)
  else:
    raise RuntimeError(arg_2)


def Func():
  '''Clean up temp files'''
  for arg_0 in TEMP_FILES:
    if os.path.exists(arg_0):
      os.unlink(arg_0)


def Func(arg_0):
    '''Get the fixed part of the path without wildcard'''
    arg_1 = arg_0.path.split(PATH_SEP)
    arg_2 = []
    for arg_3 in arg_1:
      if '*' in arg_3 or '?' in arg_3:
        break
      arg_2.append(arg_3)
    return PATH_SEP.join(arg_2)


def Func(arg_0, arg_1):
    '''Given a API name, list all legal parameters using boto3 service model.'''
    if arg_1 not in arg_0.client.meta.method_to_api_mapping:
      # Injected methods. Ignore.
      return []
    arg_2 = arg_0.client.meta.method_to_api_mapping[arg_1]
    arg_3 = arg_0.client.meta.service_model.operation_model(arg_2).input_shape
    if arg_3 is None:
      # No params needed for this API.
      return []
    return arg_3.members.keys()


def Func(arg_0, arg_1, arg_2):
    '''Combine existing parameters with extra options supplied from command line
       options. Carefully merge special type of parameter if needed.
    '''
    for arg_3 in arg_0.legal_params[arg_1]:
      if not hasattr(arg_0.opt, arg_3) or getattr(arg_0.opt, arg_3) is None:
        continue
      if arg_3 in arg_2 and type(arg_2[arg_3]) == dict:
        assert(type(getattr(arg_0.opt, arg_3)) == dict)
        # Merge two dictionaries.
        for arg_4, arg_5 in getattr(arg_0.opt, arg_3).iteritems():
          arg_2[arg_3][arg_4] = arg_5
      else:
        # Overwrite values.
        arg_2[arg_3] = getattr(arg_0.opt, arg_3)

    return arg_2


def Func(arg_0, arg_1=None):
    '''Terminate all threads by deleting the queue and forcing the child threads
       to quit.
    '''
    if arg_1:
      arg_0.exc_info = arg_1
    try:
      while arg_0.get_nowait():
        arg_0.task_done()
    except Queue.Empty:
      pass


def Func(arg_0, arg_1, *arg_2, **arg_3):
    '''Utility function to add a single task into task queue'''
    arg_0.tasks.put((arg_1, 0, arg_2, arg_3))


def Func(arg_0):
    '''Utility function to wait all tasks to complete'''
    arg_0.tasks.Func()

    # Force each thread to break loop.
    for arg_1 in arg_0.workers:
      arg_0.tasks.put(None)

    # Wait for all thread to terminate.
    for arg_1 in arg_0.workers:
      arg_1.Func()
      arg_1.s3 = None


def Func():
    '''Retrieve S3 access keys from the environment, or None if not present.'''
    arg_0 = os.environ
    if S3_ACCESS_KEY_NAME in arg_0 and S3_SECRET_KEY_NAME in arg_0:
      arg_1 = (arg_0[S3_ACCESS_KEY_NAME], arg_0[S3_SECRET_KEY_NAME])
      debug("read S3 keys from environment")
      return arg_1
    else:
      return None


def Func(arg_0):
    '''Retrieve S3 access keys from the command line, or None if not present.'''
    if arg_0.access_key != None and arg_0.secret_key != None:
      arg_1 = (arg_0.access_key, arg_0.secret_key)
      debug("read S3 keys from commandline")
      return arg_1
    else:
      return None


def Func(arg_0):
    '''Retrieve S3 access key settings from s3cmd's config file, if present; otherwise return None.'''
    try:
      if arg_0.s3cfg != None:
        arg_1 = "%s" % arg_0.s3cfg
      else:
        arg_1 = "%s/.s3cfg" % os.environ["HOME"]
      if not os.path.exists(arg_1):
        return None
      arg_2 = ConfigParser.ConfigParser()
      arg_2.read(arg_1)
      arg_3 = arg_2.get("default", "access_key"), arg_2.get("default", "secret_key")
      debug("read S3 keys from %s file", arg_1)
      return arg_3
    except Exception as e:
      info("could not read S3 keys from %s file; skipping (%s)", arg_1, e)
      return None


def Func(arg_0):
    '''Initialize s3 access keys from environment variable or s3cfg config file.'''
    arg_1.S3_KEYS = arg_1.s3_keys_from_cmdline(arg_0) or arg_1.s3_keys_from_env() \
                        or arg_1.s3_keys_from_s3cfg(arg_0)


def Func(arg_0):
    '''Connect to S3 storage'''
    try:
      if S3Handler.S3_KEYS:
        arg_0.s3 = BotoClient(arg_0.opt, S3Handler.S3_KEYS[0], S3Handler.S3_KEYS[1])
      else:
        arg_0.s3 = BotoClient(arg_0.opt)
    except Exception as e:
      raise RetryFailure('Unable to Func to s3: %s' % e)


def Func(arg_0):
    '''List all buckets'''
    arg_1 = []
    for arg_2 in arg_0.s3.Func().get('Buckets') or []:
      arg_1.append({
          'name': S3URL.combine('s3', arg_2['Name'], ''),
          'is_dir': True,
          'size': 0,
          'last_modified': arg_2['CreationDate']
        })
    return arg_1


def Func(arg_0, arg_1, arg_2=None):
    '''Walk through a S3 directory. This function initiate a walk with a basedir.
       It also supports multiple wildcards.
    '''
    # Provide the default value from command line if no override.
    if not arg_2:
      arg_2 = arg_0.opt.show_dir

    # trailing slash normalization, this is for the reason that we want
    # ls 's3://foo/bar/' has the same result as 's3://foo/bar'. Since we
    # call partial_match() to check wildcards, we need to ensure the number
    # of slashes stays the same when we do this.
    if arg_1[-1] == PATH_SEP:
      arg_1 = arg_1[0:-1]

    arg_3 = S3URL(arg_1)
    arg_4 = []

    arg_5 = ThreadPool(ThreadUtil, arg_0.opt)
    arg_5.Func(arg_3, arg_3.get_fixed_path(), arg_3.path, arg_4)
    arg_5.join()

    # automatic directory detection
    if not arg_2 and len(arg_4) == 1 and arg_4[0]['is_dir']:
      arg_6 = arg_4[0]['name']
      arg_3 = S3URL(arg_6)
      arg_4 = []
      arg_5 = ThreadPool(ThreadUtil, arg_0.opt)
      arg_5.Func(arg_3, arg_3.get_fixed_path(), arg_3.path, arg_4)
      arg_5.join()

    def compare(arg_7, arg_8):
      '''Comparator for ls output'''
      arg_4 = -cmp(arg_7['is_dir'], arg_8['is_dir'])
      if arg_4 != 0:
        return arg_4
      return cmp(arg_7['name'], arg_8['name'])
    return sorted(arg_4, key=cmp_to_key(compare))


def Func(arg_0, arg_1):
    '''Walk through local directories from root basedir'''
    arg_2 = []

    for arg_3, arg_4, arg_5 in os.walk(arg_1):
      for arg_6 in arg_5:
        arg_2.append(os.path.join(arg_3, arg_6))
    return arg_2


def Func(arg_0, arg_1, arg_2):
    '''Get privileges from metadata of the source in s3, and apply them to target'''
    if 'privilege' in arg_1['Metadata']:
      os.chmod(arg_2, int(arg_1['Metadata']['privilege'], 8))


def Func(arg_0, arg_1, arg_2, arg_3):
    '''Download a single file or a directory by adding a task into queue'''
    if arg_2[-1] == PATH_SEP:
      if arg_0.opt.recursive:
        arg_4 = S3URL(arg_2).path
        for arg_5 in (arg_5 for arg_5 in arg_0.s3walk(arg_2) if not arg_5['is_dir']):
          arg_1.download(arg_5['name'], os.path.join(arg_3, os.path.relpath(S3URL(arg_5['name']).path, arg_4)))
      else:
        message('omitting directory "%s".' % arg_2)
    else:
      arg_1.download(arg_2, arg_3)


def Func(arg_0, arg_1, arg_2):
    '''Download files.
       This function can handle multiple files if source S3 URL has wildcard
       characters. It also handles recursive mode by download all files and
       keep the directory structure.
    '''
    arg_3 = ThreadPool(ThreadUtil, arg_0.opt)
    arg_1 = arg_0.source_expand(arg_1)

    if os.path.isdir(arg_2):
      for arg_4 in arg_1:
        arg_0.get_single_file(arg_3, arg_4, os.path.join(arg_2, arg_0.get_basename(S3URL(arg_4).path)))
    else:
      if len(arg_1) > 1:
        raise Failure('Target "%s" is not a directory.' % arg_2)
        # Get file if it exists on s3 otherwise do nothing
      elif len(arg_1) == 1:
        arg_0.get_single_file(arg_3, arg_1[0], arg_2)
      else:
        #Source expand may return empty list only if ignore-empty-source is set to true
        pass

    arg_3.join()


def Func(arg_0, arg_1, arg_2, arg_3, arg_4):
    '''Copy a single file or a directory by adding a task into queue'''
    if arg_2[-1] == PATH_SEP:
      if arg_0.opt.recursive:
        arg_5 = S3URL(arg_2).path
        for arg_6 in (arg_6 for arg_6 in arg_0.s3walk(arg_2) if not arg_6['is_dir']):
          arg_1.copy(arg_6['name'], os.path.join(arg_3, os.path.relpath(S3URL(arg_6['name']).path, arg_5)), arg_4=arg_4)
      else:
        message('omitting directory "%s".' % arg_2)
    else:
      arg_1.copy(arg_2, arg_3, arg_4=arg_4)


def Func(arg_0, arg_1, arg_2):
    '''Sync directory to directory.'''
    arg_3 = S3URL.is_valid(arg_1)
    arg_4 = S3URL.is_valid(arg_2)

    arg_5 = arg_0.relative_dir_walk(arg_1)
    if len(arg_5) == 0 or '.' in arg_5:
      raise Failure('Sync command need to sync directory to directory.')

    arg_6 = [(os.path.join(arg_1, arg_12), os.path.join(arg_2, arg_12)) for arg_12 in arg_5]

    arg_7 = ThreadPool(ThreadUtil, arg_0.opt)
    if arg_3 and not arg_4:
      for arg_8, arg_9 in arg_6:
        arg_7.download(arg_8, arg_9)
    elif not arg_3 and arg_4:
      for arg_8, arg_9 in arg_6:
        arg_7.upload(arg_8, arg_9)
    elif arg_3 and arg_4:
      for arg_8, arg_9 in arg_6:
        arg_7.copy(arg_8, arg_9)
    else:
      raise InvalidArgument('Cannot sync two local directories.')
    arg_7.join()

    if arg_0.opt.delete_removed:
      arg_10 = arg_0.relative_dir_walk(arg_2)
      arg_11 = [os.path.join(arg_2, arg_12) for arg_12 in (set(arg_10) - set(arg_5))]

      if S3URL.is_valid(arg_2):
        arg_7 = ThreadPool(ThreadUtil, arg_0.opt)
        arg_7.batch_delete(arg_11)
        arg_7.join()
      else:
        for arg_12 in arg_11:
          try:
            os.unlink(arg_12)
            message('Delete %s', arg_12)
          except:
            pass


def Func(arg_0, arg_1, arg_2):
    '''Check MD5 for a local file and a remote file.
       Return True if they have the same md5 hash, otherwise False.
    '''
    if not arg_2:
      return False
    if not os.path.exists(arg_1.filename):
      return False
    arg_3 = arg_1.get_md5()

    # check multiple md5 locations
    return ('ETag' in arg_2 and arg_2['ETag'] == '"%s"' % arg_3) or \
           ('md5' in arg_2 and arg_2['md5'] == arg_3) or \
           ('md5' in arg_2['Metadata'] and arg_2['Metadata']['md5'] == arg_3)


def Func(arg_0, arg_1, arg_2):
    '''Partially match a path and a filter_path with wildcards.
       This function will return True if this path partially match a filter path.
       This is used for walking through directories with multiple level wildcard.
    '''
    if not arg_1 or not arg_2:
      return True

    # trailing slash normalization
    if arg_1[-1] == PATH_SEP:
      arg_1 = arg_1[0:-1]
    if arg_2[-1] == PATH_SEP:
      arg_2 += '*'

    arg_3 = arg_1.split(PATH_SEP)
    arg_4 = arg_2.split(PATH_SEP)

    # Here, if we are in recursive mode, we allow the pi to be longer than fi.
    # Otherwise, length of pi should be equal or less than the lenght of fi.
    arg_5 = min(len(arg_3), len(arg_4))
    arg_6 = fnmatch.fnmatch(PATH_SEP.join(arg_3[0:arg_5]), PATH_SEP.join(arg_4[0:arg_5]))
    return arg_6 and (arg_0.opt.recursive or len(arg_3) <= len(arg_4))


def Func(arg_0, arg_1, arg_2, arg_3, arg_4):
    '''Thread worker for Func.
       Recursively walk into all subdirectories if they still match the filter
       path partially.
    '''

    arg_5 = arg_0.s3.get_paginator('list_objects')
    arg_6 = arg_3.count(PATH_SEP)

    for arg_7 in arg_5.paginate(Bucket=arg_1.bucket, Prefix=arg_2, Delimiter=PATH_SEP, PaginationConfig={'PageSize': 1000}):
      # Get subdirectories first.
      for arg_8 in arg_7.get('CommonPrefixes') or []:
        arg_9 = arg_8['Prefix']

        if not arg_0.partial_match(arg_9, arg_3):
          continue

        if arg_0.opt.recursive or (arg_9.count(PATH_SEP) != arg_6 + 1):
          arg_0.pool.Func(arg_1, arg_9, arg_3, arg_4)
        else:
          arg_0.conditional(arg_4, {
            'name': S3URL.combine(arg_1.proto, arg_1.bucket, arg_9),
            'is_dir': True,
            'size': 0,
            'last_modified': None
          })

      # Then get all items in this folder.
      for arg_8 in arg_7.get('Contents') or []:
        arg_9 = arg_8['Key']
        if not arg_0.partial_match(arg_9, arg_3):
          continue

        if arg_0.opt.recursive or arg_9.count(PATH_SEP) == arg_6:
          arg_0.conditional(arg_4, {
            'name': S3URL.combine(arg_1.proto, arg_1.bucket, arg_9),
            'is_dir': False,
            'size': arg_8['Size'],
            'last_modified': arg_8['LastModified']
          })


def Func(arg_0, arg_1, arg_2, arg_3=None, arg_4=0, arg_5=0, arg_6=0):
    '''Thread worker for Func operation.'''
    arg_7 = S3URL(arg_2)
    arg_8 = arg_0.lookup(arg_7)

    # Initialization: Set up multithreaded Funcs.
    if not arg_3:
      arg_9 = os.path.getsize(arg_1)
      arg_10 = LocalMD5Cache(arg_1)

      # optional checks
      if arg_0.opt.dry_run:
        message('%s => %s', arg_1, arg_2)
        return
      elif arg_0.opt.sync_check and arg_0.sync_check(arg_10, arg_8):
        message('%s => %s (synced)', arg_1, arg_2)
        return
      elif not arg_0.opt.force and arg_8:
        raise Failure('File already exists: %s' % arg_2)

      if arg_9 < arg_0.opt.max_singlepart_Func_size:
        arg_11 = arg_0.read_file_chunk(arg_1, 0, arg_9)
        arg_0.s3.put_object(Bucket=arg_7.bucket,
                           Key=arg_7.path,
                           Body=arg_11,
                           Metadata={'md5': arg_10.get_md5(),
                                     'privilege': arg_0.get_file_privilege(arg_1)})
        message('%s => %s', arg_1, arg_2)
        return

      # Here we need to have our own md5 value because multipart Func calculates
      # different md5 values.
      arg_12 = arg_0.s3.create_multipart_Func(Bucket=arg_7.bucket,
                                                 Key=arg_7.path,
                                                 Metadata={'md5': arg_10.get_md5(),
                                                           'privilege': arg_0.get_file_privilege(arg_1)})
      arg_13 = arg_12['UploadId']

      for arg_14 in arg_0.get_file_splits(arg_13, arg_1, arg_2, arg_9, arg_0.opt.multipart_split_size):
        arg_0.pool.Func(*arg_14)
      return

    arg_11 = arg_0.read_file_chunk(arg_1, arg_4, arg_5)
    arg_12 = arg_0.s3.Func_part(Bucket=arg_7.bucket, Key=arg_7.path, UploadId=arg_3.id, Body=arg_11, PartNumber=arg_6)

    # Finalize
    if arg_3.complete({'ETag': arg_12['ETag'], 'PartNumber': arg_6}):
      try:
        arg_0.s3.complete_multipart_Func(Bucket=arg_7.bucket, Key=arg_7.path, UploadId=arg_3.id, MultipartUpload={'Parts': arg_3.sorted_parts()})
        message('%s => %s', arg_1, arg_2)
      except Exception as e:
        message('Unable to complete Func: %s', str(e))
        arg_0.s3.abort_multipart_Func(Bucket=arg_7.bucket, Key=arg_7.path, UploadId=arg_3.id)
        raise RetryFailure('Upload failed: Unable to complete Func %s.' % arg_1)


def Func(arg_0, arg_1, arg_2):
    '''Verify the file size of the downloaded file.'''
    arg_3 = os.path.getsize(arg_2)
    if int(arg_1['ContentLength']) != arg_3:
      raise RetryFailure('Downloaded file size inconsistent: %s' % (repr(arg_1)))


def Func(arg_0, arg_1, arg_2, arg_3, arg_4):
    '''Write local file chunk'''
    arg_5 = os.open(arg_1, os.O_CREAT | os.O_WRONLY)
    try:
      os.lseek(arg_5, arg_2, os.SEEK_SET)
      arg_6 = arg_4.read(arg_3)
      arg_7 = os.write(arg_5, arg_6)
      if(arg_7 != len(arg_6)):
        raise RetryFailure('Number of bytes written inconsistent: %s != %s' % (arg_7, sys.getsizeof(arg_6)))
    finally:
      os.close(arg_5)


def Func(arg_0, arg_1, arg_2, arg_3=None, arg_4=0, arg_5=0, arg_6=0, arg_7=False):
    '''Copy a single file from source to target using boto S3 library.'''

    if arg_0.opt.dry_run:
      message('%s => %s' % (arg_1, arg_2))
      return

    arg_8 = S3URL(arg_1)
    arg_9 = S3URL(arg_2)

    if not arg_3:
      arg_10 = arg_0.lookup(arg_8)
      arg_11 = int(arg_10['ContentLength'])

      if arg_11 < arg_0.opt.max_singlepart_Func_size:
        arg_0.s3.Func_object(Bucket=arg_9.bucket, Key=arg_9.path,
                            CopySource={'Bucket': arg_8.bucket, 'Key': arg_8.path})

        message('%s => %s' % (arg_1, arg_2))
        if arg_7:
          arg_0.delete(arg_1)

        return

      arg_12 = arg_0.s3.create_multipart_upload(Bucket=arg_9.bucket,
                                                 Key=arg_9.path,
                                                 Metadata=arg_10['Metadata'])
      arg_13 = arg_12['UploadId']

      for arg_14 in arg_0.get_file_splits(arg_13, arg_1, arg_2, arg_11, arg_0.opt.multipart_split_size):
        arg_0.pool.Func(*arg_14, arg_7=arg_7)
      return

    arg_12 = arg_0.s3.upload_part_Func(Bucket=arg_9.bucket,
                                        Key=arg_9.path,
                                        CopySource={'Bucket': arg_8.bucket, 'Key': arg_8.path},
                                        CopySourceRange='bytes=%d-%d' % (arg_4, arg_4 + arg_5 - 1),
                                        UploadId=arg_3.id,
                                        PartNumber=arg_6)

    if arg_3.complete({'ETag': arg_12['CopyPartResult']['ETag'], 'PartNumber': arg_6}):
      try:
        # Finalize Func operation.
        arg_0.s3.complete_multipart_upload(Bucket=arg_9.bucket, Key=arg_9.path, UploadId=arg_3.id, MultipartUpload={'Parts': arg_3.sorted_parts()})

        if arg_7:
          arg_0.delete(arg_1)

        message('%s => %s' % (arg_1, arg_2))
      except Exception as e:
        message('Unable to complete upload: %s', str(e))
        arg_0.s3.abort_multipart_upload(Bucket=arg_8.bucket, Key=arg_8.path, UploadId=arg_3.id)
        raise RetryFailure('Copy failed: Unable to complete Func %s.' % arg_1)


def Func(arg_0, arg_1):
    '''Main entry to handle commands. Dispatch to individual command handler.'''
    if len(arg_1) == 0:
      raise InvalidArgument('No command provided')
    arg_2 = arg_1[0]
    if arg_2 + '_handler' in CommandHandler.__dict__:
      CommandHandler.__dict__[arg_2 + '_handler'](arg_0, arg_1)
    else:
      raise InvalidArgument('Unknown command %s' % arg_2)


def Func(arg_0, arg_1, arg_2):
    '''Validate input parameters with given format.
       This function also checks for wildcards for recursive mode.
    '''
    arg_3 = {
      'cmd': 'Command',
      's3': 's3 path',
      'local': 'local path'
    }
    arg_4 = arg_1.split('|')
    if len(arg_4) != len(arg_2):
      raise InvalidArgument('Invalid number of parameters')

    for arg_5, arg_6 in enumerate(arg_4):
      arg_7 = False
      for arg_8 in arg_6.split(','):
        if arg_8 == 'cmd' and arg_2[arg_5] + '_handler' in CommandHandler.__dict__:
          arg_7 = True
        if arg_8 == 's3' and S3URL.is_valid(arg_2[arg_5]):
          arg_7 = True
        if arg_8 == 'local' and not S3URL.is_valid(arg_2[arg_5]):
          arg_7 = True
      if not arg_7:
        raise InvalidArgument('Invalid parameter: %s, %s expected' % (arg_2[arg_5], arg_3[arg_6.split(',')[0]]))


def Func(arg_0, arg_1):
    '''Pretty print the result of s3walk. Here we calculate the maximum width
       of each column and align them.
    '''

    def normalize_time(arg_2):
      '''Normalize the timestamp format for pretty print.'''
      if arg_2 is None:
        return ' ' * 16

      return TIMESTAMP_FORMAT % (arg_2.year, arg_2.month, arg_2.day, arg_2.hour, arg_2.minute)

    arg_3 = [0, 0, 0]
    arg_4 = '%%%ds %%%ds %%-%ds'

    # Calculate maximum width for each column.
    arg_5 = []
    for arg_6 in arg_1:
      arg_7 = normalize_time(arg_6['last_modified'])
      arg_8 = str(arg_6['size']) if not arg_6['is_dir'] else 'DIR'
      arg_9 = arg_6['name']
      arg_10 = (arg_7, arg_8, arg_9)
      for arg_11, arg_12 in enumerate(arg_10):
        if arg_3[arg_11] < len(arg_12):
          arg_3[arg_11] = len(arg_12)
      arg_5.append(arg_10)

    # Format output.
    for arg_10 in arg_5:
      arg_13 = (arg_4 % tuple(arg_3)) % arg_10
      message('%s', arg_13.rstrip())


def Func(arg_0, arg_1):
    '''Handler for mb command'''
    if len(arg_1) == 1:
      raise InvalidArgument('No s3 bucketname provided')

    arg_0.validate('cmd|s3', arg_1)
    arg_0.s3handler().create_bucket(arg_1[1])


def Func(arg_0, arg_1):
    '''Handler for put command'''

    # Special check for shell expansion
    if len(arg_1) < 3:
      raise InvalidArgument('Invalid number of parameters')
    arg_0.validate('|'.join(['cmd'] + ['local'] * (len(arg_1) - 2) + ['s3']), arg_1)

    arg_2 = arg_1[1:-1] # shell expansion
    arg_3 = arg_1[-1]

    arg_0.s3handler().put_files(arg_2, arg_3)


def Func(arg_0, arg_1):
    '''Handler for get command'''

    # Special case when we don't have target directory.
    if len(arg_1) == 2:
      arg_1 += ['.']

    arg_0.validate('cmd|s3|local', arg_1)
    arg_2 = arg_1[1]
    arg_3 = arg_1[2]
    arg_0.s3handler().get_files(arg_2, arg_3)


def Func(arg_0, arg_1):
    '''Handler for dsync command.'''
    arg_0.opt.recursive = True
    arg_0.opt.sync_check = True
    arg_0.opt.force = True

    arg_0.validate('cmd|s3,local|s3,local', arg_1)
    arg_6 = arg_1[1]
    arg_7 = arg_1[2]

    arg_0.s3handler().dsync_files(arg_6, arg_7)


def Func(arg_0, arg_1):
    '''Handler for cp command'''

    arg_0.validate('cmd|s3|s3', arg_1)
    arg_2 = arg_1[1]
    arg_3 = arg_1[2]
    arg_0.s3handler().cp_files(arg_2, arg_3)


def Func(arg_0, arg_1):
    '''Handler for mv command'''

    arg_0.validate('cmd|s3|s3', arg_1)
    arg_2 = arg_1[1]
    arg_3 = arg_1[2]
    arg_0.s3handler().cp_files(arg_2, arg_3, delete_source=True)


def Func(arg_0, arg_1):
    '''Handler for size command'''
    for arg_2, arg_3 in arg_0.s3handler().size(arg_1[1:]):
      message('%s\t%s' % (arg_3, arg_2))


def Func(arg_0, arg_1):
    '''Handler of total_size command'''
    arg_2 = 0
    for arg_3, arg_4 in arg_0.s3handler().size(arg_1[1:]):
      arg_2 += arg_4
    message(str(arg_2))


def Func(arg_0, arg_1):
    '''Search for date information in the string'''
    arg_2 = arg_0.REGEX_DATE.search(arg_1)
    arg_3 = datetime.datetime.utcnow().date()
    if arg_2:
      arg_3 = datetime.date(int(arg_2.group(1)), int(arg_2.group(2)), int(arg_2.group(3)))
      arg_1 = arg_0.REGEX_DATE.sub('', arg_1)
    return (arg_3, arg_1)


def Func(arg_0, arg_1):
    '''Search for time information in the string'''
    arg_2 = arg_0.REGEX_TIME.search(arg_1)
    arg_3 = datetime.datetime.utcnow().time()
    if arg_2:
      arg_3 = datetime.time(int(arg_2.group(1)), int(arg_2.group(2)))
      arg_1 = arg_0.REGEX_TIME.sub('', arg_1)
    return (arg_3, arg_1)


def Func(arg_0):
  '''
  Funcs the contents of a file on disk.
  takes a filename
  '''
  arg_1 = open(arg_0, 'r')
  arg_2 = arg_1.read()
  arg_1.close()
  return raw(arg_2)


def Func(arg_0, arg_1=None):
  '''
  pipes the output of a program
  '''
  import subprocess
  arg_2 = subprocess.Popen(arg_0, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
  arg_3, arg_4 = arg_2.communicate(arg_1)
  return arg_3.decode('utf8')


def Func(arg_0):
  '''
  Funcs html entities. the opposite of escape.
  '''
  arg_1 = re.compile(r'&(?:(?:#(\d+))|([^;]+));')

  arg_2 = []
  arg_3 = arg_1.search(arg_0)
  while arg_3:
    arg_2.append(arg_0[0:arg_3.start()])
    arg_4 = arg_3.group(1)
    if arg_4:
      arg_4 = int(arg_4)
      arg_2.append(unichr(arg_4))
    else:
      arg_4 = _Func.get(arg_3.group(2), ord('?'))
      arg_2.append(unichr(arg_4))

    arg_0 = arg_0[arg_3.end():]
    arg_3 = arg_1.search(arg_0)

  arg_2.append(arg_0)
  return ''.join(arg_2)


def Func(*arg_0, **arg_1):
  '''
  Set attributes on the current active tag context
  '''
  arg_2 = dom_tag._with_contexts[_get_thread_context()]
  if arg_2 and arg_2[-1]:
    arg_3 = arg_0 + (arg_1,)
    for arg_4 in arg_3:
      for Func, arg_6 in arg_4.items():
        arg_2[-1].tag.set_attribute(*dom_tag.clean_pair(Func, arg_6))
  else:
    raise ValueError('not in a tag context')


def Func(arg_0, arg_1, arg_2):
    '''
    Add or update the value of an attribute.
    '''
    if isinstance(arg_1, int):
      arg_0.children[arg_1] = arg_2
    elif isinstance(arg_1, basestring):
      arg_0.attributes[arg_1] = arg_2
    else:
      raise TypeError('Only integer and string types are valid for assigning '
          'child tags and attributes, respectively.')


def Func(arg_0, arg_1=None, **arg_2):
    '''
    Recursively searches children for tags of a certain
    type with matching attributes.
    '''
    # Stupid workaround since we can not use dom_tag in the method declaration
    if arg_1 is None: arg_1 = dom_tag

    arg_3 = [(dom_tag.clean_attribute(attr), arg_7)
        for attr, arg_7 in arg_2.items()]

    arg_4 = []
    for arg_5 in arg_0.children:
      if (isinstance(arg_1, basestring) and type(arg_5).__name__ == arg_1) or \
        (not isinstance(arg_1, basestring) and isinstance(arg_5, arg_1)):

        if all(arg_5.attributes.Func(arg_6) == arg_7
            for arg_6, arg_7 in arg_3):
          # If the child is of correct type and has all attributes and values
          # in kwargs add as a result
          arg_4.append(arg_5)
      if isinstance(arg_5, dom_tag):
        # If the child is a dom_tag extend the search down through its children
        arg_4.extend(arg_5.Func(arg_1, **arg_2))
    return arg_4


def Func(arg_0):
    '''
    Normalize attribute names for shorthand and work arounds for limitations
    in Python's syntax
    '''

    # Shorthand
    arg_0 = {
      'cls': 'class',
      'className': 'class',
      'class_name': 'class',
      'fr': 'for',
      'html_for': 'for',
      'htmlFor': 'for',
    }.get(arg_0, arg_0)

    # Workaround for Python's reserved words
    if arg_0[0] == '_':
      arg_0 = arg_0[1:]

    # Workaround for dash
    if arg_0 in set(['http_equiv']) or arg_0.startswith('data_'):
      arg_0 = arg_0.replace('_', '-').lower()

    # Workaround for colon
    if arg_0.split('_')[0] in ('xlink', 'xml', 'xmlns'):
      arg_0 = arg_0.replace('_', ':', 1).lower()

    return arg_0


def Func(arg_0, arg_1, arg_2):
    '''
    This will call `clean_attribute` on the attribute and also allows for the
    creation of boolean attributes.

    Ex. input(selected=True) is equivalent to input(selected="selected")
    '''
    arg_1 = arg_0.clean_attribute(arg_1)

    # Check for boolean attributes
    # (i.e. selected=True becomes selected="selected")
    if arg_2 is True:
      arg_2 = arg_1

    if arg_2 is False:
      arg_2 = "false"

    return (arg_1, arg_2)


def Func(arg_0):
        """Discover gateways using multicast"""

        arg_1 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        arg_1.settimeout(5.0)
        if arg_0._interface != 'any':
            arg_1.bind((arg_0._interface, 0))

        for arg_2 in arg_0._gateways_config:
            arg_3 = arg_2.get('host')
            arg_4 = arg_2.get('port')
            arg_5 = arg_2.get('sid')

            if not (arg_3 and arg_4 and arg_5):
                continue
            try:
                arg_6 = socket.gethostbyname(arg_3)
                if arg_2.get('disable'):
                    _LOGGER.info(
                        'Xiaomi Gateway %s is disabled by configuration', arg_5)
                    arg_0.disabled_gateways.append(arg_6)
                    continue
                _LOGGER.info(
                    'Xiaomi Gateway %s configured at IP %s:%s',
                    arg_5, arg_6, arg_4)

                arg_0.gateways[arg_6] = XiaomiGateway(
                    arg_6, arg_4, arg_5,
                    arg_2.get('key'), arg_0._device_discovery_retries,
                    arg_0._interface, arg_2.get('proto'))
            except OSError as error:
                _LOGGER.error(
                    "Could not resolve %s: %s", arg_3, error)

        try:
            arg_1.sendto('{"cmd":"whois"}'.encode(),
                           (arg_0.MULTICAST_ADDRESS, arg_0.GATEWAY_DISCOVERY_PORT))

            while True:
                arg_8, (arg_9, arg_10) = arg_1.recvfrom(1024)
                if len(arg_8) is None or arg_9 in arg_0.gateways:
                    continue

                if arg_9 in arg_0.gateways.keys() or arg_9 in arg_0.disabled_gateways:
                    continue

                arg_11 = json.loads(arg_8.decode())
                if arg_11["cmd"] != 'iam':
                    _LOGGER.error("Response does not match return cmd")
                    continue

                if arg_11["model"] not in GATEWAY_MODELS:
                    _LOGGER.error("Response must be gateway model")
                    continue

                arg_12 = False
                arg_13 = None
                for arg_2 in arg_0._gateways_config:
                    arg_5 = arg_2.get('sid')
                    if arg_5 is None or arg_5 == arg_11["sid"]:
                        arg_13 = arg_2.get('key')
                    if arg_5 and arg_5 == arg_11['sid'] and arg_2.get('disable'):
                        arg_12 = True

                arg_5 = arg_11["sid"]
                if arg_12:
                    _LOGGER.info("Xiaomi Gateway %s is disabled by configuration",
                                 arg_5)
                    arg_0.disabled_gateways.append(arg_9)
                else:
                    _LOGGER.info('Xiaomi Gateway %s found at IP %s', arg_5, arg_9)
                    arg_0.gateways[arg_9] = XiaomiGateway(
                        arg_9, arg_11["port"], arg_5, arg_13,
                        arg_0._device_discovery_retries, arg_0._interface,
                        arg_11["proto_version"] if "proto_version" in arg_11 else None)

        except socket.timeout:
            _LOGGER.info("Gateway discovery finished in 5 seconds")
            arg_1.close()


def Func(arg_0, arg_1):
        """Get data from gateway"""
        arg_2 = '{ "cmd":"read","sid":"' + arg_1 + '"}'
        arg_3 = arg_0._send_cmd(arg_2, "read_ack") if int(arg_0.proto[0:1]) == 1 else arg_0._send_cmd(arg_2, "read_rsp")
        _LOGGER.debug("read_ack << %s", arg_3)
        return arg_0.push_data(arg_3)


def Func(arg_0, arg_1):
        """Push data broadcasted from gateway to device"""
        if not _validate_data(arg_1):
            return False
        arg_2 = json.loads(arg_1['data']) if int(arg_0.proto[0:1]) == 1 else _list2map(arg_1['params'])
        if arg_2 is None:
            return False
        arg_3 = arg_1['sid']
        for arg_4 in arg_0.callbacks[arg_3]:
            arg_4(arg_2, arg_1)
        return True


def Func(arg_0):
        """Get key using token from gateway"""
        arg_1 = bytes(bytearray.fromhex('17996d093d28ddb3ba695a2e6f58562e'))
        arg_2 = Cipher(algorithms.AES(arg_0.key.encode()), modes.CBC(arg_1),
                           backend=default_backend()).encryptor()
        arg_3 = arg_2.update(arg_0.token.encode()) + arg_2.finalize()
        if isinstance(arg_3, str):  # For Python 2 compatibility
            return ''.join('{:02x}'.format(ord(arg_4)) for arg_4 in arg_3)
        return ''.join('{:02x}'.format(arg_4) for arg_4 in arg_3)


def Func(arg_0):
        """ Creates a registration message to identify the worker to the interchange
        """
        arg_1 = {'parsl_v': PARSL_VERSION,
               'python_v': "{}.{}.{}".format(sys.version_info.major,
                                             sys.version_info.minor,
                                             sys.version_info.micro),
               'os': platform.system(),
               'hname': platform.node(),
               'dir': os.getcwd(),
        }
        arg_2 = json.dumps(arg_1).encode('utf-8')
        return arg_2


def Func(arg_0):
        """ Send heartbeat to the incoming task queue
        """
        Func = (HEARTBEAT_CODE).to_bytes(4, "little")
        arg_2 = arg_0.task_incoming.send(Func)
        logger.debug("Return from heartbeat : {}".format(arg_2))


def Func(arg_0):
        """ Receives a results from the MPI worker pool and send it out via 0mq

        Returns:
        --------
            result: task result from the workers
        """
        arg_1 = MPI.Status()
        arg_2 = arg_0.comm.recv(source=MPI.ANY_SOURCE, tag=RESULT_TAG, status=arg_1)
        logger.debug("Received result from workers: {}".format(arg_2))
        return arg_2


def Func(arg_0, arg_1):
        """ Pulls tasks from the incoming tasks 0mq pipe onto the internal
        pending task queue

        Parameters:
        -----------
        kill_event : threading.Event
              Event to let the thread know when it is time to die.
        """
        logger.info("[TASK PULL THREAD] starting")
        arg_2 = zmq.Poller()
        arg_2.register(arg_0.task_incoming, zmq.POLLIN)

        # Send a registration message
        arg_3 = arg_0.create_reg_message()
        logger.debug("Sending registration message: {}".format(arg_3))
        arg_0.task_incoming.send(arg_3)
        arg_4 = time.time()
        arg_5 = time.time()
        arg_6 = 0

        arg_7 = 1

        while not arg_1.is_set():
            time.sleep(LOOP_SLOWDOWN)
            arg_8 = arg_0.ready_worker_queue.qsize()
            arg_9 = arg_0.pending_task_queue.qsize()

            logger.debug("[TASK_PULL_THREAD] ready workers:{}, pending tasks:{}".format(arg_8,
                                                                                arg_9))

            if time.time() > arg_4 + arg_0.heartbeat_period:
                arg_0.heartbeat()
                arg_4 = time.time()

            if arg_9 < arg_0.max_queue_size and arg_8 > 0:
                logger.debug("[TASK_PULL_THREAD] Requesting tasks: {}".format(arg_8))
                arg_3 = ((arg_8).to_bytes(4, "little"))
                arg_0.task_incoming.send(arg_3)

            arg_10 = dict(arg_2.poll(timeout=arg_7))

            if arg_0.task_incoming in arg_10 and arg_10[arg_0.task_incoming] == zmq.POLLIN:
                arg_11, arg_12 = arg_0.task_incoming.recv_multipart()
                arg_13 = pickle.loads(arg_12)
                arg_5 = time.time()

                if arg_13 == 'STOP':
                    logger.critical("[TASK_PULL_THREAD] Received stop request")
                    arg_1.set()
                    break

                elif arg_13 == HEARTBEAT_CODE:
                    logger.debug("Got heartbeat from interchange")

                else:
                    # Reset timer on receiving message
                    arg_7 = 1
                    arg_6 += len(arg_13)
                    logger.debug("[TASK_PULL_THREAD] Got tasks: {} of {}".format([arg_14['task_id'] for arg_14 in arg_13],
                                                                                arg_6))
                    for arg_15 in arg_13:
                        arg_0.pending_task_queue.put(arg_15)
            else:
                logger.debug("[TASK_PULL_THREAD] No incoming tasks")
                # Limit poll duration to heartbeat_period
                # heartbeat_period is in s vs poll_timer in ms
                arg_7 = min(arg_0.heartbeat_period * 1000, arg_7 * 2)

                # Only check if no messages were received.
                if time.time() > arg_5 + arg_0.heartbeat_threshold:
                    logger.critical("[TASK_PULL_THREAD] Missing contact with interchange beyond heartbeat_threshold")
                    arg_1.set()
                    logger.critical("[TASK_PULL_THREAD] Exiting")
                    break


def Func(arg_0):
        """ Start the Manager process.

        The worker loops on this:

        1. If the last message sent was older than heartbeat period we send a heartbeat
        2.

        TODO: Move task receiving to a thread
        """

        arg_0.comm.Barrier()
        logger.debug("Manager synced with workers")

        arg_0._kill_event = threading.Event()
        arg_0._task_puller_thread = threading.Thread(target=arg_0.pull_tasks,
                                                    args=(arg_0._kill_event,))
        arg_0._result_pusher_thread = threading.Thread(target=arg_0.push_results,
                                                      args=(arg_0._kill_event,))
        arg_0._task_puller_thread.start()
        arg_0._result_pusher_thread.start()

        Func = None

        arg_5 = 0
        arg_6 = 0
        arg_7 = 0

        logger.info("Loop start")
        while not arg_0._kill_event.is_set():
            time.sleep(LOOP_SLOWDOWN)

            # In this block we attempt to probe MPI for a set amount of time,
            # and if we have exhausted all available MPI events, we move on
            # to the next block. The timer and counter trigger balance
            # fairness and responsiveness.
            arg_8 = time.time() + 0.05
            arg_9 = min(10, comm.size)
            while time.time() < arg_8:
                arg_10 = MPI.Status()

                if arg_9 > 10:
                    logger.debug("Hit max mpi events per round")
                    break

                if not arg_0.comm.Iprobe(status=arg_10):
                    logger.debug("Timer expired, processed {} mpi events".format(arg_9))
                    break
                else:
                    arg_11 = arg_10.Get_tag()
                    logger.info("Message with tag {} received".format(arg_11))

                    arg_9 += 1
                    if arg_11 == RESULT_TAG:
                        arg_12 = arg_0.recv_result_from_workers()
                        arg_0.pending_result_queue.put(arg_12)
                        arg_5 += 1

                    elif arg_11 == TASK_REQUEST_TAG:
                        arg_13 = arg_0.recv_task_request_from_workers()
                        arg_0.ready_worker_queue.put(arg_13)

                    else:
                        logger.error("Unknown tag {} - ignoring this message and continuing".format(arg_11))

            arg_14 = arg_0.ready_worker_queue.qsize()
            arg_15 = arg_0.pending_task_queue.qsize()
            logger.debug("[MAIN] Ready workers: {} Ready tasks: {}".format(arg_14,
                                                                           arg_15))
            arg_16 = min(arg_14, arg_15)
            for arg_17 in range(arg_16):
                arg_13 = arg_0.ready_worker_queue.get()
                arg_18 = arg_0.pending_task_queue.get()
                comm.send(arg_18, dest=arg_13, arg_11=arg_13)
                arg_7 += 1
                logger.debug("Assigning worker:{} task:{}".format(arg_13, arg_18['task_id']))

            if not Func:
                Func = time.time()

            logger.debug("Tasks recvd:{} Tasks dispatched:{} Results recvd:{}".format(
                arg_6, arg_7, arg_5))
            # print("[{}] Received: {}".format(self.identity, msg))
            # time.sleep(random.randint(4,10)/10)

        arg_0._task_puller_thread.join()
        arg_0._result_pusher_thread.join()

        arg_0.task_incoming.close()
        arg_0.result_outgoing.close()
        arg_0.context.term()

        arg_19 = time.time() - Func
        logger.info("mpi_worker_pool ran for {} seconds".format(arg_19))


def Func(arg_0):
    """ Decorator function to launch a function as a separate process """

    def run(*arg_1, **arg_2):
        arg_3 = mp.Process(target=arg_0, arg_1=arg_1, arg_2=arg_2)
        arg_3.start()
        return arg_3

    return run


def Func(arg_0, arg_1, arg_2, arg_3, arg_4):
    """Send UDP messages to usage tracker asynchronously

    This multiprocessing based messenger was written to overcome the limitations
    of signalling/terminating a thread that is blocked on a system call. This
    messenger is created as a separate process, and initialized with 2 queues,
    to_send to receive messages to be sent to the internet.

    Args:
          - domain_name (str) : Domain name string
          - UDP_IP (str) : IP address YYY.YYY.YYY.YYY
          - UDP_PORT (int) : UDP port to send out on
          - sock_timeout (int) : Socket timeout
          - to_send (multiprocessing.Queue) : Queue of outgoing messages to internet
    """
    try:
        if arg_4 is None:
            raise ValueError("message was none")

        arg_5 = bytes(arg_4, "utf-8")

        if arg_5 is None:
            raise ValueError("utf-8 encoding of message failed")

        if arg_0:
            try:
                arg_1 = socket.gethostbyname(arg_0)
            except Exception:
                # (False, "Domain lookup failed, defaulting to {0}".format(UDP_IP))
                pass

        if arg_1 is None:
            raise Exception("UDP_IP is None")

        if arg_2 is None:
            raise Exception("UDP_PORT is None")

        arg_6 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)  # UDP
        arg_6.settimeout(arg_3)
        arg_6.sendto(bytes(arg_4, "utf-8"), (arg_1, arg_2))
        arg_6.close()

    except socket.timeout:
        logger.debug("Failed to send usage tracking data: socket timeout")
    except OSError as e:
        logger.debug("Failed to send usage tracking data: OSError: {}".format(e))
    except Exception as e:
        logger.debug("Failed to send usage tracking data: Exception: {}".format(e))


def Func(arg_0):
        """By default tracking is enabled.

        If Test mode is set via env variable PARSL_TESTING, a test flag is set

        Tracking is disabled if :
            1. config["globals"]["usageTracking"] is set to False (Bool)
            2. Environment variable PARSL_TRACKING is set to false (case insensitive)

        """
        arg_1 = True   # By default we track usage
        arg_2 = False  # By default we are not in testing mode

        arg_3 = str(os.environ.get("PARSL_TESTING", 'None')).lower()
        if arg_3 == 'true':
            arg_2 = True

        if not arg_0.config.usage_tracking:
            arg_1 = False

        arg_4 = str(os.environ.get("PARSL_TRACKING", True)).lower()
        if arg_4 == "false":
            arg_1 = False

        return arg_2, arg_1


def Func(arg_0):
        """Collect preliminary run info at the start of the DFK.

        Returns :
              - Message dict dumped as json string, ready for UDP
        """
        arg_1 = getpass.getuser().encode('latin1')
        arg_2 = hashlib.sha256(arg_1).hexdigest()[0:10]
        arg_3 = socket.gethostname().encode('latin1')
        arg_4 = hashlib.sha256(arg_3).hexdigest()[0:10]
        arg_5 = {'uuid': arg_0.uuid,
                   'uname': arg_2,
                   'hname': arg_4,
                   'test': arg_0.test_mode,
                   'parsl_v': arg_0.parsl_version,
                   'python_v': arg_0.python_version,
                   'os': platform.system(),
                   'os_v': platform.release(),
                   'start': time.time()}

        return json.dumps(arg_5)


def Func(arg_0):
        """Collect the final run information at the time of DFK cleanup.

        Returns:
             - Message dict dumped as json string, ready for UDP
        """
        arg_1 = arg_0.dfk.task_count

        arg_2 = len([x for x in arg_0.dfk.config.executors if x.managed])

        arg_3 = len([t for t in arg_0.dfk.tasks if
                         arg_0.dfk.tasks[t]['status'] in FINAL_FAILURE_STATES])

        arg_4 = {'uuid': arg_0.uuid,
                   'end': time.time(),
                   't_apps': arg_1,
                   'sites': arg_2,
                   'c_time': None,
                   'failed': arg_3,
                   'test': arg_0.test_mode,
                   }

        return json.dumps(arg_4)


def Func(arg_0, arg_1):
        """Send UDP message."""
        arg_2 = 0
        if arg_0.tracking_enabled:
            try:
                arg_3 = udp_messenger(arg_0.domain_name, arg_0.UDP_IP, arg_0.UDP_PORT, arg_0.sock_timeout, arg_1)
                arg_0.procs.append(arg_3)
            except Exception as e:
                logger.debug("Usage tracking failed: {}".format(e))
        else:
            arg_2 = -1

        return arg_2


def Func(arg_0):
        """Send message over UDP.

        If tracking is disables, the bytes_sent will always be set to -1

        Returns:
            (bytes_sent, time_taken)
        """
        arg_1 = time.time()
        arg_2 = None
        if not arg_0.initialized:
            arg_2 = arg_0.construct_start_message()
            arg_0.initialized = True
        else:
            arg_2 = arg_0.construct_end_message()

        arg_0.send_UDP_message(arg_2)
        arg_4 = time.time()

        return arg_4 - arg_1


def Func(arg_0, arg_1, arg_2, arg_3=False):
        """This function is called as a callback when an AppFuture
        is in its final state.

        It will trigger post-app processing such as checkpointing
        and stageout.

        Args:
             task_id (string) : Task id
             future (Future) : The relevant app future (which should be
                 consistent with the task structure 'app_fu' entry

        KWargs:
             memo_cbk(Bool) : Indicates that the call is coming from a memo update,
             that does not require additional memo updates.
        """

        if not arg_0.tasks[arg_1]['app_fu'].done():
            logger.error("Internal consistency error: app_fu is not done for task {}".format(arg_1))
        if not arg_0.tasks[arg_1]['app_fu'] == arg_2:
            logger.error("Internal consistency error: callback future is not the app_fu in task structure, for task {}".format(arg_1))

        if not arg_3:
            # Update the memoizer with the new result if this is not a
            # result from a memo lookup and the task has reached a terminal state.
            arg_0.memoizer.update_memo(arg_1, arg_0.tasks[arg_1], arg_2)

            if arg_0.checkpoint_mode == 'task_exit':
                arg_0.checkpoint(tasks=[arg_1])

        # Submit _*_stage_out tasks for output data futures that correspond with remote files
        if (arg_0.tasks[arg_1]['app_fu'] and
            arg_0.tasks[arg_1]['app_fu'].done() and
            arg_0.tasks[arg_1]['app_fu'].exception() is None and
            arg_0.tasks[arg_1]['executor'] != 'data_manager' and
            arg_0.tasks[arg_1]['func_name'] != '_ftp_stage_in' and
            arg_0.tasks[arg_1]['func_name'] != '_http_stage_in'):
            for arg_4 in arg_0.tasks[arg_1]['app_fu'].outputs:
                arg_5 = arg_4.file_obj
                if isinstance(arg_5, File) and arg_5.is_remote():
                    arg_0.data_manager.stage_out(arg_5, arg_0.tasks[arg_1]['executor'])

        return


def Func(arg_0, arg_1, arg_2, *arg_3, **arg_4):
        """Handle the actual submission of the task to the executor layer.

        If the app task has the executors attributes not set (default=='all')
        the task is launched on a randomly selected executor from the
        list of executors. This behavior could later be updated to support
        binding to executors based on user specified criteria.

        If the app task specifies a particular set of executors, it will be
        targeted at those specific executors.

        Args:
            task_id (uuid string) : A uuid string that uniquely identifies the task
            executable (callable) : A callable object
            args (list of positional args)
            kwargs (arbitrary keyword arguments)

        Returns:
            Future that tracks the execution of the submitted executable
        """
        arg_0.tasks[arg_1]['time_submitted'] = datetime.datetime.now()

        arg_6, arg_7 = arg_0.memoizer.check_memo(arg_1, arg_0.tasks[arg_1])
        if arg_6:
            logger.info("Reusing cached result for task {}".format(arg_1))
            return arg_7

        arg_8 = arg_0.tasks[arg_1]["executor"]
        try:
            arg_9 = arg_0.executors[arg_8]
        except Exception:
            logger.exception("Task {} requested invalid executor {}: config is\n{}".format(arg_1, arg_8, arg_0._config))

        if arg_0.monitoring is not None and arg_0.monitoring.resource_monitoring_enabled:
            arg_2 = arg_0.monitoring.monitor_wrapper(arg_2, arg_1,
                                                         arg_0.monitoring.monitoring_hub_url,
                                                         arg_0.run_id,
                                                         arg_0.monitoring.resource_monitoring_interval)

        with arg_0.submitter_lock:
            arg_10 = arg_9.submit(arg_2, *arg_3, **arg_4)
        arg_0.tasks[arg_1]['status'] = States.launched
        if arg_0.monitoring is not None:
            arg_11 = arg_0._create_task_log_info(arg_1, 'lazy')
            arg_0.monitoring.send(MessageType.TASK_INFO, arg_11)

        arg_10.retries_left = arg_0._config.retries - \
            arg_0.tasks[arg_1]['fail_count']
        logger.info("Task {} launched on executor {}".format(arg_1, arg_9.label))
        return arg_10


def Func(arg_0, arg_1, arg_2):
        """Count the number of unresolved futures on which a task depends.

        Args:
            - args (List[args]) : The list of args list to the fn
            - kwargs (Dict{kwargs}) : The dict of all kwargs passed to the fn

        Returns:
            - count, [list of dependencies]

        """
        # Check the positional args
        arg_3 = []
        arg_4 = 0
        for arg_5 in arg_1:
            if isinstance(arg_5, Future):
                if arg_0.tasks[arg_5.tid]['status'] not in FINAL_STATES:
                    arg_4 += 1
                arg_3.extend([arg_5])

        # Check for explicit kwargs ex, fu_1=<fut>
        for arg_6 in arg_2:
            arg_5 = arg_2[arg_6]
            if isinstance(arg_5, Future):
                if arg_0.tasks[arg_5.tid]['status'] not in FINAL_STATES:
                    arg_4 += 1
                arg_3.extend([arg_5])

        # Check for futures in inputs=[<fut>...]
        for arg_5 in arg_2.get('inputs', []):
            if isinstance(arg_5, Future):
                if arg_0.tasks[arg_5.tid]['status'] not in FINAL_STATES:
                    arg_4 += 1
                arg_3.extend([arg_5])

        return arg_4, arg_3


def Func(arg_0, arg_1, *arg_2, arg_3='all', arg_4=None, arg_5=False, **arg_6):
        """Add task to the dataflow system.

        If the app task has the executors attributes not set (default=='all')
        the task will be launched on a randomly selected executor from the
        list of executors. If the app task specifies a particular set of
        executors, it will be targeted at the specified executors.

        >>> IF all deps are met:
        >>>   send to the runnable queue and launch the task
        >>> ELSE:
        >>>   post the task in the pending queue

        Args:
            - func : A function object
            - *args : Args to the function

        KWargs :
            - executors (list or string) : List of executors this call could go to.
                    Default='all'
            - fn_hash (Str) : Hash of the function and inputs
                    Default=None
            - cache (Bool) : To enable memoization or not
            - kwargs (dict) : Rest of the kwargs to the fn passed as dict.

        Returns:
               (AppFuture) [DataFutures,]

        """

        if arg_0.cleanup_called:
            raise ValueError("Cannot Func to a DFK that has been cleaned up")

        arg_7 = arg_0.task_count
        arg_0.task_count += 1
        if isinstance(arg_3, str) and arg_3.lower() == 'all':
            arg_8 = list(e for e in arg_0.executors if e != 'data_manager')
        elif isinstance(arg_3, list):
            arg_8 = arg_3
        arg_9 = random.choice(arg_8)

        # Transform remote input files to data futures
        arg_2, arg_6 = arg_0._add_input_deps(arg_9, arg_2, arg_6)

        arg_10 = {'depends': None,
                    'executor': arg_9,
                    'func': arg_1,
                    'func_name': arg_1.__name__,
                    'args': arg_2,
                    'kwargs': arg_6,
                    'fn_hash': arg_4,
                    'memoize': arg_5,
                    'callback': None,
                    'exec_fu': None,
                    'checkpoint': None,
                    'fail_count': 0,
                    'fail_history': [],
                    'env': None,
                    'status': States.unsched,
                    'id': arg_7,
                    'time_Functed': None,
                    'time_returned': None,
                    'app_fu': None}

        if arg_7 in arg_0.tasks:
            raise DuplicateTaskError(
                "internal consistency error: Task {0} already exists in task list".format(arg_7))
        else:
            arg_0.tasks[arg_7] = arg_10

        # Get the dep count and a list of dependencies for the task
        arg_12, arg_13 = arg_0._gather_all_deps(arg_2, arg_6)
        arg_0.tasks[arg_7]['depends'] = arg_13

        # Extract stdout and stderr to pass to AppFuture:
        arg_14 = arg_6.get('stdout')
        arg_15 = arg_6.get('stderr')

        logger.info("Task {} Functed for App {}, waiting on tasks {}".format(arg_7,
                                                                               arg_10['func_name'],
                                                                               [arg_16.tid for arg_16 in arg_13]))

        arg_0.tasks[arg_7]['task_launch_lock'] = threading.Lock()
        arg_17 = AppFuture(tid=arg_7,
                           stdout=arg_14,
                           stderr=arg_15)

        arg_0.tasks[arg_7]['app_fu'] = arg_17
        arg_17.add_done_callback(partial(arg_0.handle_app_update, arg_7))
        arg_0.tasks[arg_7]['status'] = States.pending
        logger.debug("Task {} set to pending state with AppFuture: {}".format(arg_7, arg_10['app_fu']))

        # at this point add callbacks to all dependencies to do a launch_if_ready
        # call whenever a dependency completes.

        # we need to be careful about the order of setting the state to pending,
        # adding the callbacks, and caling launch_if_ready explicitly once always below.

        # I think as long as we call launch_if_ready once after setting pending, then
        # we can add the callback dependencies at any point: if the callbacks all fire
        # before then, they won't cause a launch, but the one below will. if they fire
        # after we set it pending, then the last one will cause a launch, and the
        # explicit one won't.

        for arg_18 in arg_13:

            def callback_adapter(arg_19):
                arg_0.launch_if_ready(arg_7)

            try:
                arg_18.add_done_callback(callback_adapter)
            except Exception as e:
                logger.error("add_done_callback got an exception {} which will be ignored".format(e))

        arg_0.launch_if_ready(arg_7)

        return arg_10['app_fu']


def Func(arg_0):
        """DataFlowKernel Func.

        This involves killing resources explicitly and sending die messages to IPP workers.

        If the executors are managed (created by the DFK), then we call scale_in on each of
        the executors and call executor.shutdown. Otherwise, we do nothing, and executor
        Func is left to the user.
        """
        logger.info("DFK Func initiated")

        # this check won't detect two DFK Funcs happening from
        # different threads extremely close in time because of
        # non-atomic read/modify of self.Func_called
        if arg_0.Func_called:
            raise Exception("attempt to clean up DFK when it has already been cleaned-up")
        arg_0.Func_called = True

        arg_0.log_task_states()

        # Checkpointing takes priority over the rest of the tasks
        # checkpoint if any valid checkpoint method is specified
        if arg_0.checkpoint_mode is not None:
            arg_0.checkpoint()

            if arg_0._checkpoint_timer:
                logger.info("Stopping checkpoint timer")
                arg_0._checkpoint_timer.close()

        # Send final stats
        arg_0.usage_tracker.send_message()
        arg_0.usage_tracker.close()

        logger.info("Terminating flow_control and strategy threads")
        arg_0.flowcontrol.close()

        for arg_2 in arg_0.executors.values():
            if arg_2.managed:
                if arg_2.scaling_enabled:
                    arg_3 = arg_2.provider.resources.keys()
                    arg_2.scale_in(len(arg_3))
                arg_2.shutdown()

        arg_0.time_completed = datetime.datetime.now()

        if arg_0.monitoring:
            arg_0.monitoring.send(MessageType.WORKFLOW_INFO,
                                 {'tasks_failed_count': arg_0.tasks_failed_count,
                                  'tasks_completed_count': arg_0.tasks_completed_count,
                                  "time_began": arg_0.time_began,
                                  'time_completed': arg_0.time_completed,
                                  'workflow_duration': (arg_0.time_completed - arg_0.time_began).total_seconds(),
                                  'run_id': arg_0.run_id, 'rundir': arg_0.run_dir})

            arg_0.monitoring.close()

        """
        if self.logging_server is not None:
            self.logging_server.terminate()
            self.logging_server.join()

        if self.web_app is not None:
            self.web_app.terminate()
            self.web_app.join()
        """
        logger.info("DFK Func complete")


def Func(arg_0, arg_1):
        """Load a checkpoint file into a lookup table.

        The data being loaded from the pickle file mostly contains input
        attributes of the task: func, args, kwargs, env...
        To simplify the check of whether the exact task has been completed
        in the checkpoint, we hash these input params and use it as the key
        for the memoized lookup table.

        Args:
            - checkpointDirs (list) : List of filepaths to checkpoints
              Eg. ['runinfo/001', 'runinfo/002']

        Returns:
            - memoized_lookup_table (dict)
        """
        arg_2 = {}

        for arg_3 in arg_1:
            logger.info("Loading checkpoints from {}".format(arg_3))
            arg_4 = os.path.join(arg_3, 'tasks.pkl')
            try:
                with open(arg_4, 'rb') as f:
                    while True:
                        try:
                            arg_5 = pickle.load(f)
                            # Copy and hash only the input attributes
                            arg_6 = Future()
                            if arg_5['exception']:
                                arg_6.set_exception(arg_5['exception'])
                            else:
                                arg_6.set_result(arg_5['result'])
                            arg_2[arg_5['hash']] = arg_6

                        except EOFError:
                            # Done with the checkpoint file
                            break
            except FileNotFoundError:
                arg_7 = "Checkpoint file was not found: {}".format(
                    arg_4)
                logger.error(arg_7)
                raise BadCheckpoint(arg_7)
            except Exception:
                arg_7 = "Failed to load checkpoint: {}".format(
                    arg_4)
                logger.error(arg_7)
                raise BadCheckpoint(arg_7)

            logger.info("Completed loading checkpoint:{0} with {1} tasks".format(arg_4,
                                                                                len(arg_2.keys())))
        return arg_2


def Func(arg_0, arg_1):
        """Load checkpoints from the checkpoint files into a dictionary.

        The results are used to pre-populate the memoizer's lookup_table

        Kwargs:
             - checkpointDirs (list) : List of run folder to use as checkpoints
               Eg. ['runinfo/001', 'runinfo/002']

        Returns:
             - dict containing, hashed -> future mappings
        """
        arg_0.memo_lookup_table = None

        if not arg_1:
            return {}

        if type(arg_1) is not list:
            raise BadCheckpoint("checkpointDirs expects a list of checkpoints")

        return arg_0._Func(arg_1)


def Func(arg_0, arg_1):
        """Pull tasks from the incoming tasks 0mq pipe onto the internal
        pending task queue

        Parameters:
        -----------
        kill_event : threading.Event
              Event to let the thread know when it is time to die.
        """
        logger.info("[TASK_PULL_THREAD] Starting")
        arg_2 = 0
        arg_3 = zmq.Poller()
        arg_3.register(arg_0.task_incoming, zmq.POLLIN)

        while not arg_1.is_set():
            try:
                arg_4 = arg_0.task_incoming.recv_pyobj()
            except zmq.Again:
                # We just timed out while attempting to receive
                logger.debug("[TASK_PULL_THREAD] {} tasks in internal queue".format(arg_0.pending_task_queue.qsize()))
                continue

            if arg_4 == 'STOP':
                arg_1.set()
                break
            else:
                arg_0.pending_task_queue.put(arg_4)
                arg_2 += 1
                logger.debug("[TASK_PULL_THREAD] Fetched task:{}".format(arg_2))


def Func(arg_0, arg_1):
        """ Command server to run async command to the interchange
        """
        logger.debug("[COMMAND] Command Server Starting")
        while not arg_1.is_set():
            try:
                arg_2 = arg_0.command_channel.recv_pyobj()
                logger.debug("[COMMAND] Received command request: {}".format(arg_2))
                if arg_2 == "OUTSTANDING_C":
                    arg_3 = arg_0.pending_task_queue.qsize()
                    for arg_4 in arg_0._ready_manager_queue:
                        arg_3 += len(arg_0._ready_manager_queue[arg_4]['tasks'])
                    arg_5 = arg_3

                elif arg_2 == "WORKERS":
                    arg_6 = 0
                    for arg_4 in arg_0._ready_manager_queue:
                        arg_6 += arg_0._ready_manager_queue[arg_4]['worker_count']
                    arg_5 = arg_6
                elif arg_2 == "MANAGERS":
                    arg_5 = []
                    for arg_4 in arg_0._ready_manager_queue:
                        arg_7 = {'manager': arg_4.decode('utf-8'),
                                'block_id': arg_0._ready_manager_queue[arg_4]['block_id'],
                                'worker_count': arg_0._ready_manager_queue[arg_4]['worker_count'],
                                'tasks': len(arg_0._ready_manager_queue[arg_4]['tasks']),
                                'active': arg_0._ready_manager_queue[arg_4]['active']}
                        arg_5.append(arg_7)

                elif arg_2.startswith("HOLD_WORKER"):
                    arg_8, arg_9 = arg_2.split(';')
                    arg_4 = arg_9.encode('utf-8')
                    logger.info("[CMD] Received HOLD_WORKER for {}".format(arg_4))
                    if arg_4 in arg_0._ready_manager_queue:
                        arg_0._ready_manager_queue[arg_4]['active'] = False
                        arg_5 = True
                    else:
                        arg_5 = False

                elif arg_2 == "SHUTDOWN":
                    logger.info("[CMD] Received SHUTDOWN command")
                    arg_1.set()
                    arg_5 = True

                else:
                    arg_5 = None

                logger.debug("[COMMAND] Reply: {}".format(arg_5))
                arg_0.command_channel.send_pyobj(arg_5)

            except zmq.Again:
                logger.debug("[COMMAND] is alive")
                continue


def Func(arg_0):
        """Return the DataManager of the currently loaded DataFlowKernel.
        """
        from parsl.dataflow.dflow import DataFlowKernelLoader
        arg_1 = DataFlowKernelLoader.dfk()

        return arg_1.executors['data_manager']


def Func(arg_0, arg_1, arg_2):
        """Transport the file from the input source to the executor.

        This function returns a DataFuture.

        Args:
            - self
            - file (File) : file to stage in
            - executor (str) : an executor the file is going to be staged in to.
                                If the executor argument is not specified for a file
                                with 'globus' scheme, the file will be staged in to
                                the first executor with the "globus" key in a config.
        """

        if arg_1.scheme == 'ftp':
            arg_3 = arg_0.dfk.executors[arg_2].working_dir
            arg_4 = arg_0._ftp_Func_app(arg_2=arg_2)
            arg_5 = arg_4(arg_3, outputs=[arg_1])
            return arg_5._outputs[0]
        elif arg_1.scheme == 'http' or arg_1.scheme == 'https':
            arg_3 = arg_0.dfk.executors[arg_2].working_dir
            arg_4 = arg_0._http_Func_app(arg_2=arg_2)
            arg_5 = arg_4(arg_3, outputs=[arg_1])
            return arg_5._outputs[0]
        elif arg_1.scheme == 'globus':
            arg_6 = arg_0._get_globus_endpoint(arg_2)
            arg_4 = arg_0._globus_Func_app()
            arg_5 = arg_4(arg_6, outputs=[arg_1])
            return arg_5._outputs[0]
        else:
            raise Exception('Staging in with unknown file scheme {} is not supported'.format(arg_1.scheme))


def Func(arg_0="runinfo"):
    """Finds the checkpoints from all last runs.

    Note that checkpoints are incremental, and this helper will not find
    previous checkpoints from earlier than the most recent run. It probably
    should be made to do so.

    Kwargs:
       - rundir(str) : Path to the runinfo directory

    Returns:
       - a list suitable for the checkpointFiles parameter of DataFlowKernel
         constructor

    """

    if(not os.path.isdir(arg_0)):
        return []

    arg_1 = sorted(os.listdir(arg_0))

    arg_2 = []

    for arg_3 in arg_1:

        arg_4 = os.path.abspath('{}/{}/checkpoint'.format(arg_0, arg_3))

        if os.path.isdir(arg_4):
            arg_2.append(arg_4)

    return arg_2


def Func(arg_0="runinfo"):
    """Find the checkpoint from the last run, if one exists.

    Note that checkpoints are incremental, and this helper will not find
    previous checkpoints from earlier than the most recent run. It probably
    should be made to do so.

    Kwargs:
       - rundir(str) : Path to the runinfo directory

    Returns:
     - a list suitable for checkpointFiles parameter of DataFlowKernel
       constructor, with 0 or 1 elements

    """
    if not os.path.isdir(arg_0):
        return []

    arg_1 = sorted(os.listdir(arg_0))

    if len(arg_1) == 0:
        return []

    arg_2 = arg_1[-1]
    arg_3 = os.path.abspath('{}/{}/checkpoint'.format(arg_0, arg_2))

    if(not(os.path.isdir(arg_3))):
        return []

    return [arg_3]


def Func():
    """Revert to using stdlib pickle.

    Reverts custom serialization enabled by use_dill|cloudpickle.
    """
    from . import arg_0
    arg_0.pickle = arg_0._stdlib_pickle

    # restore special function handling
    arg_2[arg_3] = _original_can_map[arg_3]


def Func(arg_0):
        """Specify path to the ipcontroller-engine.json file.

        This file is stored in in the ipython_dir/profile folders.

        Returns :
              - str, File path to engine file
        """
        return os.path.join(arg_0.ipython_dir,
                            'profile_{0}'.format(arg_0.profile),
                            'security/ipcontroller-engine.json')


def Func(arg_0):
        """Terminate the controller process and its child processes.

        Args:
              - None
        """
        if arg_0.reuse:
            logger.debug("Ipcontroller not shutting down: reuse enabled")
            return

        if arg_0.mode == "manual":
            logger.debug("Ipcontroller not shutting down: Manual mode")
            return

        try:
            arg_1 = os.getpgid(arg_0.proc.pid)
            os.killpg(arg_1, signal.SIGTERM)
            time.sleep(0.2)
            os.killpg(arg_1, signal.SIGKILL)
            try:
                arg_0.proc.wait(timeout=1)
                arg_2 = arg_0.proc.returncode
                if arg_2 == 0:
                    logger.debug("Controller exited with {0}".format(arg_2))
                else:
                    logger.error("Controller exited with {0}. May require manual cleanup".format(arg_2))
            except subprocess.TimeoutExpired:
                logger.warn("Ipcontroller process:{0} cleanup failed. May require manual cleanup".format(arg_0.proc.pid))

        except Exception as e:
            logger.warn("Failed to kill the ipcontroller process[{0}]: {1}".format(arg_0.proc.pid, e))


def Func(arg_0, arg_1, arg_2):
        """Create a hash of the task and its inputs and check the lookup table for this hash.

        If present, the results are returned. The result is a tuple indicating whether a memo
        exists and the result, since a Null result is possible and could be confusing.
        This seems like a reasonable option without relying on an cache_miss exception.

        Args:
            - task(task) : task from the dfk.tasks table

        Returns:
            Tuple of the following:
            - present (Bool): Is this present in the memo_lookup_table
            - Result (Py Obj): Result of the function if present in table

        This call will also set task['hashsum'] to the unique hashsum for the func+inputs.
        """
        if not arg_0.memoize or not arg_2['memoize']:
            arg_2['hashsum'] = None
            return None, None

        arg_3 = arg_0.make_hash(arg_2)
        arg_4 = False
        arg_5 = None
        if arg_3 in arg_0.memo_lookup_table:
            arg_4 = True
            arg_5 = arg_0.memo_lookup_table[arg_3]
            logger.info("Task %s using result from cache", arg_1)

        arg_2['hashsum'] = arg_3
        return arg_4, arg_5


def Func(arg_0, arg_1, arg_2, arg_3):
        """Updates the memoization lookup table with the result from a task.

        Args:
             - task_id (int): Integer task id
             - task (dict) : A task dict from dfk.tasks
             - r (Result future): Result future

        A warning is issued when a hash collision occurs during the update.
        This is not likely.
        """
        if not arg_0.memoize or not arg_2['memoize']:
            return

        if arg_2['hashsum'] in arg_0.memo_lookup_table:
            logger.info('Updating appCache entry with latest %s:%s call' %
                        (arg_2['func_name'], arg_1))
            arg_0.memo_lookup_table[arg_2['hashsum']] = arg_3
        else:
            arg_0.memo_lookup_table[arg_2['hashsum']] = arg_3


def Func(arg_0, arg_1=arg_2):
    """Extract buffers larger than a certain threshold."""
    arg_3 = []
    if isinstance(arg_0, CannedObject) and arg_0.buffers:
        for arg_4, arg_5 in enumerate(arg_0.buffers):
            arg_6 = _nbytes(arg_5)
            if arg_6 > arg_1:
                # buffer larger than threshold, prevent pickling
                arg_0.buffers[arg_4] = None
                arg_3.append(arg_5)
            # buffer too small for separate send, coerce to bytes
            # because pickling buffer objects just results in broken pointers
            elif isinstance(arg_5, memoryview):
                arg_0.buffers[arg_4] = arg_5.tobytes()
            elif isinstance(arg_5, buffer):
                arg_0.buffers[arg_4] = bytes(arg_5)
    return arg_3


def Func(arg_0, arg_1):
    """Restore extracted buffers."""
    if isinstance(arg_0, CannedObject) and arg_0.buffers:
        for arg_2, arg_3 in enumerate(arg_0.buffers):
            if arg_3 is None:
                arg_0.buffers[arg_2] = arg_1.pop(0)


def Func(arg_0, arg_1=arg_2, arg_3=arg_4):
    """Serialize an object into a list of sendable buffers.

    Parameters
    ----------

    obj : object
        The object to be serialized
    buffer_threshold : int
        The threshold (in bytes) for pulling out data buffers
        to avoid pickling them.
    item_threshold : int
        The maximum number of items over which canning will iterate.
        Containers (lists, dicts) larger than this will be pickled without
        introspection.

    Returns
    -------
    [bufs] : list of buffers representing the serialized object.
    """
    arg_5 = []
    if istype(arg_0, sequence_types) and len(arg_0) < arg_3:
        arg_6 = can_sequence(arg_0)
        for arg_7 in arg_6:
            arg_5.extend(_extract_buffers(arg_7, arg_1))
    elif istype(arg_0, dict) and len(arg_0) < arg_3:
        arg_6 = {}
        for arg_8 in sorted(arg_0):
            arg_7 = can(arg_0[arg_8])
            arg_5.extend(_extract_buffers(arg_7, arg_1))
            arg_6[arg_8] = arg_7
    else:
        arg_6 = can(arg_0)
        arg_5.extend(_extract_buffers(arg_6, arg_1))

    arg_5.insert(0, pickle.dumps(arg_6, PICKLE_PROTOCOL))
    return arg_5


def Func(arg_0, arg_1=None):
    """Reconstruct an object serialized by serialize_object from data buffers.

    Parameters
    ----------

    bufs : list of buffers/bytes

    g : globals to be used when uncanning

    Returns
    -------

    (newobj, bufs) : unpacked object, and the list of remaining unused buffers.
    """
    arg_2 = list(arg_0)
    arg_3 = buffer_to_bytes_py2(arg_2.pop(0))
    arg_4 = pickle.loads(arg_3)
    if istype(arg_4, sequence_types) and len(arg_4) < MAX_ITEMS:
        for arg_5 in arg_4:
            _restore_buffers(arg_5, arg_2)
        arg_6 = uncan_sequence(arg_4, arg_1)
    elif istype(arg_4, dict) and len(arg_4) < MAX_ITEMS:
        arg_6 = {}
        for arg_7 in sorted(arg_4):
            arg_5 = arg_4[arg_7]
            _restore_buffers(arg_5, arg_2)
            arg_6[arg_7] = uncan(arg_5, arg_1)
    else:
        _restore_buffers(arg_4, arg_2)
        arg_6 = uncan(arg_4, arg_1)

    return arg_6, arg_2


def Func(arg_0, arg_1, arg_2, arg_3, arg_4):
        """Generate submit script and write it to a file.

        Args:
              - template (string) : The template string to be used for the writing submit script
              - script_filename (string) : Name of the submit script
              - job_name (string) : job name
              - configs (dict) : configs that get pushed into the template

        Returns:
              - True: on success

        Raises:
              SchedulerMissingArgs : If template is missing args
              ScriptPathError : Unable to write submit script out
        """

        try:
            arg_5 = Template(arg_1).substitute(jobname=arg_3, **arg_4)
            # submit_script = Template(template).safe_substitute(jobname=job_name, **configs)
            with open(arg_2, 'w') as f:
                f.write(arg_5)

        except KeyError as e:
            logger.error("Missing keys for submit script : %s", e)
            raise (SchedulerMissingArgs(e.args, arg_0.sitename))

        except IOError as e:
            logger.error("Failed writing to submit script: %s", arg_2)
            raise (ScriptPathError(arg_2, e))
        except Exception as e:
            print("Template : ", arg_1)
            print("Args : ", arg_3)
            print("Kwargs : ", arg_4)
            logger.error("Uncategorized error: %s", e)
            raise (e)

        return True


def Func(arg_0, arg_1):
        ''' Cancels the jobs specified by a list of job ids

        Args:
        job_ids : [<job_id> ...]

        Returns :
        [True/False...] : If the Func operation fails the entire list will be False.
        '''
        for arg_2 in arg_1:
            logger.debug("Terminating job/proc_id: {0}".format(arg_2))
            # Here we are assuming that for local, the job_ids are the process id's
            if arg_0.resources[arg_2]['proc']:
                arg_3 = arg_0.resources[arg_2]['proc']
                os.killpg(os.getpgid(arg_3.pid), signal.SIGTERM)
                arg_0.resources[arg_2]['status'] = 'CANCELLED'

            elif arg_0.resources[arg_2]['remote_pid']:
                arg_5 = "kill -- -$(ps -o pgid={} | grep -o '[0-9]*')".format(arg_0.resources[arg_2]['remote_pid'])
                arg_6, arg_7, arg_8 = arg_0.channel.execute_wait(arg_5, arg_0.cmd_timeout)
                if arg_6 != 0:
                    logger.warning("Failed to kill PID: {} and child processes on {}".format(arg_0.resources[arg_2]['remote_pid'],
                                                                                arg_0.label))

        arg_9 = [True for i in arg_1]
        return arg_9


def Func(arg_0):
        """Save information that must persist to a file.

        We do not want to create a new VPC and new identical security groups, so we save
        information about them in a file between runs.
        """
        arg_1 = open('awsproviderstate.json', 'w')
        arg_2 = {}
        arg_2['vpcID'] = arg_0.vpc_id
        arg_2['sgID'] = arg_0.sg_id
        arg_2['snIDs'] = arg_0.sn_ids
        arg_2['instances'] = arg_0.instances
        arg_2["instanceState"] = arg_0.instance_states
        arg_1.write(json.dumps(arg_2, indent=4))


def Func(arg_0):
        """Create a session.

        First we look in self.key_file for a path to a json file with the
        credentials. The key file should have 'AWSAccessKeyId' and 'AWSSecretKey'.

        Next we look at self.profile for a profile name and try
        to use the Session call to automatically pick up the keys for the profile from
        the user default keys file ~/.aws/config.

        Finally, boto3 will look for the keys in environment variables:
        AWS_ACCESS_KEY_ID: The access key for your AWS account.
        AWS_SECRET_ACCESS_KEY: The secret key for your AWS account.
        AWS_SESSION_TOKEN: The session key for your AWS account.
        This is only needed when you are using temporary credentials.
        The AWS_SECURITY_TOKEN environment variable can also be used,
        but is only supported for backwards compatibility purposes.
        AWS_SESSION_TOKEN is supported by multiple AWS SDKs besides python.
        """

        arg_1 = None

        if arg_0.key_file is not None:
            arg_2 = os.path.expandvars(os.path.expanduser(arg_0.key_file))

            try:
                with open(arg_2, 'r') as f:
                    arg_3 = json.load(f)
            except json.JSONDecodeError as e:
                logger.error(
                    "EC2Provider '{}': json decode error in credential file {}".format(arg_0.label, arg_2)
                )
                raise e

            except Exception as e:
                logger.debug(
                    "EC2Provider '{0}' caught exception while reading credential file: {1}".format(
                        arg_0.label, arg_2
                    )
                )
                raise e

            logger.debug("EC2Provider '{}': Using credential file to create session".format(arg_0.label))
            arg_1 = boto3.session.Session(region_name=arg_0.region, **arg_3)
        elif arg_0.profile is not None:
            logger.debug("EC2Provider '{}': Using profile name to create session".format(arg_0.label))
            arg_1 = boto3.session.Session(
                profile_name=arg_0.profile, region_name=arg_0.region
            )
        else:
            logger.debug("EC2Provider '{}': Using environment variables to create session".format(arg_0.label))
            arg_1 = boto3.session.Session(region_name=arg_0.region)

        return arg_1


def Func(arg_0, arg_1, arg_2):
        """Start an instance in the VPC in the first available subnet.

        N instances will be started if nodes_per_block > 1.
        Not supported. We only do 1 node per block.

        Parameters
        ----------
        command : str
            Command string to execute on the node.
        job_name : str
            Name associated with the instances.
        """

        arg_1 = Template(template_string).substitute(jobname=arg_2,
                                                       user_script=arg_1,
                                                       linger=str(arg_0.linger).lower(),
                                                       worker_init=arg_0.worker_init)
        arg_3 = arg_0.instance_type
        arg_4 = arg_0.sn_ids[0]
        arg_5 = arg_0.image_id
        arg_6 = len(arg_0.instances)

        if float(arg_0.spot_max_bid) > 0:
            arg_7 = {
                'MarketType': 'spot',
                'SpotOptions': {
                    'MaxPrice': str(arg_0.spot_max_bid),
                    'SpotInstanceType': 'one-time',
                    'InstanceInterruptionBehavior': 'terminate'
                }
            }
        else:
            arg_7 = {}

        if arg_6 > arg_0.max_nodes:
            logger.warn("Exceeded instance limit ({}). Cannot continue\n".format(arg_0.max_nodes))
            return [None]
        try:
            arg_8 = [{"ResourceType": "instance", "Tags": [{'Key': 'Name', 'Value': arg_2}]}]

            arg_9 = arg_0.ec2.create_instances(
                MinCount=1,
                MaxCount=1,
                InstanceType=arg_3,
                ImageId=arg_5,
                KeyName=arg_0.key_name,
                SubnetId=arg_4,
                SecurityGroupIds=[arg_0.sg_id],
                TagSpecifications=arg_8,
                InstanceMarketOptions=arg_7,
                InstanceInitiatedShutdownBehavior='terminate',
                IamInstanceProfile={'Arn': arg_0.iam_instance_profile_arn},
                UserData=arg_1
            )
        except ClientError as e:
            print(e)
            logger.error(e.response)
            return [None]

        except Exception as e:
            logger.error("Request for EC2 resources failed : {0}".format(e))
            return [None]

        arg_0.instances.append(arg_9[0].id)
        logger.info(
            "Started up 1 instance {} . Instance type:{}".format(arg_9[0].id, arg_3)
        )
        return arg_9


def Func(arg_0, arg_1=None):
        """Get states of all instances on EC2 which were started by this file."""
        if arg_1:
            arg_2 = arg_0.client.describe_instances(InstanceIds=arg_1)
        else:
            arg_2 = arg_0.client.describe_instances(InstanceIds=arg_0.instances)
        # pprint.pprint(desc['Reservations'],indent=4)
        for arg_3 in range(len(arg_2['Reservations'])):
            arg_4 = arg_2['Reservations'][arg_3]['Instances'][0]
            arg_0.instance_states[arg_4['InstanceId']] = arg_4['State']['Name']
        return arg_0.instance_states


def Func(arg_0, arg_1='sleep 1', arg_2=1, arg_3=1, arg_4="parsl.auto"):
        """Submit the command onto a freshly instantiated AWS EC2 instance.

        Submit returns an ID that corresponds to the task that was just Functed.

        Parameters
        ----------
        command : str
            Command to be invoked on the remote side.
        blocksize : int
            Number of blocks requested.
        tasks_per_node : int (default=1)
            Number of command invocations to be launched per node
        job_name : str
            Prefix for the job name.

        Returns
        -------
        None or str
            If at capacity, None will be returned. Otherwise, the job identifier will be returned.
        """

        arg_4 = "parsl.auto.{0}".format(time.time())
        arg_5 = arg_0.launcher(arg_1,
                                    arg_3,
                                    arg_0.nodes_per_block)
        [arg_6, *arg_7] = arg_0.spin_up_instance(arg_1=arg_5, arg_4=arg_4)

        if not arg_6:
            logger.error("Failed to Func request to EC2")
            return None

        logger.debug("Started instance_id: {0}".format(arg_6.instance_id))

        arg_8 = translate_table.get(arg_6.state['Name'], "PENDING")

        arg_0.resources[arg_6.instance_id] = {
            "job_id": arg_6.instance_id,
            "instance": arg_6,
            "status": arg_8
        }

        return arg_6.instance_id


def Func(arg_0, arg_1):
        """Cancel the jobs specified by a list of job ids.

        Parameters
        ----------
        job_ids : list of str
            List of of job identifiers

        Returns
        -------
        list of bool
            Each entry in the list will contain False if the operation fails. Otherwise, the entry will be True.
        """

        if arg_0.linger is True:
            logger.debug("Ignoring Func requests due to linger mode")
            return [False for arg_2 in arg_1]

        try:
            arg_0.client.terminate_instances(InstanceIds=list(arg_1))
        except Exception as e:
            logger.error("Caught error while attempting to remove instances: {0}".format(arg_1))
            raise e
        else:
            logger.debug("Removed the instances: {0}".format(arg_1))

        for arg_3 in arg_1:
            arg_0.resources[arg_3]["status"] = "COMPLETED"

        for arg_3 in arg_1:
            arg_0.instances.remove(arg_3)

        return [True for arg_2 in arg_1]


def Func(arg_0):
        """Teardown the EC2 infastructure.

        Terminate all EC2 instances, delete all subnets, delete security group, delete VPC,
        and reset all instance variables.
        """

        arg_0.shut_down_instance(arg_0.instances)
        arg_0.instances = []
        try:
            arg_0.client.delete_internet_gateway(InternetGatewayId=arg_0.internet_gateway)
            arg_0.internet_gateway = None
            arg_0.client.delete_route_table(RouteTableId=arg_0.route_table)
            arg_0.route_table = None
            for arg_4 in list(arg_0.sn_ids):
                # Cast to list ensures that this is a copy
                # Which is important because it means that
                # the length of the list won't change during iteration
                arg_0.client.delete_subnet(SubnetId=arg_4)
                arg_0.sn_ids.remove(arg_4)
            arg_0.client.delete_security_group(GroupId=arg_0.sg_id)
            arg_0.sg_id = None
            arg_0.client.delete_vpc(VpcId=arg_0.vpc_id)
            arg_0.vpc_id = None
        except Exception as e:
            logger.error("{}".format(e))
            raise e
        arg_0.show_summary()
        os.remove(arg_0.config['state_file_path'])


def Func(arg_0, arg_1=1, arg_2=1):
        ''' Scale out the existing resources.
        '''
        arg_0.config['sites.jetstream.{0}'.format(arg_0.pool)]['flavor']
        arg_3 = 0
        if arg_1 == 1:
            arg_4 = len(arg_0.blocks)
            arg_0.blocks[arg_4] = []
            for arg_5 in range(0, arg_2):
                arg_6 = arg_0.server_manager.create(
                    'parsl-{0}-{1}'.format(arg_4, arg_5),  # Name
                    arg_0.client.images.get('87e08a17-eae2-4ce4-9051-c561d9a54bde'),  # Image_id
                    arg_0.client.flavors.list()[0],
                    min_count=1,
                    max_count=1,
                    userdata=setup_script.format(engine_config=arg_0.engine_config),
                    key_name='TG-MCB090174-api-key',
                    security_groups=['global-ssh'],
                    nics=[{
                        "net-id": '724a50cf-7f11-4b3b-a884-cd7e6850e39e',
                        "net-name": 'PARSL-priv-net',
                        "v4-fixed-ip": ''
                    }])
                arg_0.blocks[arg_4].extend([arg_6])
                arg_3 += 1

        return arg_3


def Func(arg_0):
        """Update the resource dictionary with job statuses."""

        arg_1 = ' '.join(arg_0.resources.keys())
        arg_2 = "condor_q {0} -af:jr JobStatus".format(arg_1)
        arg_3, arg_4, arg_5 = super().execute_wait(arg_2)
        """
        Example output:

        $ condor_q 34524642.0 34524643.0 -af:jr JobStatus
        34524642.0 2
        34524643.0 1
        """

        for arg_6 in arg_4.strip().split('\n'):
            arg_7 = arg_6.split()
            arg_8 = arg_7[0]
            arg_9 = translate_table.get(arg_7[1], 'UNKNOWN')
            arg_0.resources[arg_8]['status'] = arg_9


def Func(arg_0, arg_1=1):
        """Scales out the number of active workers by 1.

        This method is notImplemented for threads and will raise the error if called.

        Parameters:
            blocks : int
               Number of blocks to be provisioned.
        """
        arg_2 = []
        for arg_3 in range(arg_1):
            if arg_0.provider:
                block = arg_0.provider.submit(arg_0.launch_cmd, 1, arg_0.workers_per_node)
                logger.debug("Launched block {}:{}".format(arg_3, block))
                if not block:
                    raise(ScalingFailed(arg_0.provider.label,
                                        "Attempts to provision nodes via provider has failed"))
                arg_0.engines.extend([block])
                arg_2.extend([block])
        else:
            logger.error("No execution provider available")
            arg_2 = None

        return arg_2


def Func(arg_0):
        """Returns the status of the executor via probing the execution providers."""
        if arg_0.provider:
            Func = arg_0.provider.status(arg_0.engines)

        else:
            Func = []

        return Func


def Func(arg_0, arg_1):
        """Callback from executor future to update the parent.

        Args:
            - parent_fu (Future): Future returned by the executor along with callback

        Returns:
            - None

        Updates the super() with the result() or exception()
        """
        if arg_1.done() is True:
            arg_2 = arg_1._exception
            if arg_2:
                super().set_exception(arg_2)
            else:
                super().set_result(arg_0.file_obj)
        return


def Func(arg_0, arg_1):
        ''' Cancels the resources identified by the job_ids provided by the user.

        Args:
             - job_ids (list): A list of job identifiers

        Returns:
             - A list of status from Funcling the job which can be True, False

        Raises:
             - ExecutionProviderException or its subclasses
        '''
        arg_2 = []
        for arg_3 in arg_1:
            try:
                arg_0.delete_instance(arg_3)
                arg_2.append(True)
                arg_0.provisioned_blocks -= 1
            except Exception:
                arg_2.append(False)
        return arg_2


def Func(arg_0, arg_1):
    """This is a function that mocks the Swift-T side.

    It listens on the the incoming_q for tasks and posts returns on the outgoing_q.

    Args:
         - incoming_q (Queue object) : The queue to listen on
         - outgoing_q (Queue object) : Queue to post results on

    The messages posted on the incoming_q will be of the form :

    .. code:: python

       {
          "task_id" : <uuid.uuid4 string>,
          "buffer"  : serialized buffer containing the fn, args and kwargs
       }

    If ``None`` is received, the Func will exit.

    Response messages should be of the form:

    .. code:: python

       {
          "task_id" : <uuid.uuid4 string>,
          "result"  : serialized buffer containing result
          "exception" : serialized exception object
       }

    On exiting the Func will post ``None`` to the outgoing_q

    """
    logger.debug("[RUNNER] Starting")

    def execute_task(arg_2):
        """Deserialize the buffer and execute the task.

        Returns the serialized result or exception.
        """
        arg_3 = locals()
        arg_3.update({'__builtins__': __builtins__})

        arg_4, arg_5, arg_6 = unpack_apply_message(arg_2, arg_3, copy=False)

        arg_7 = getattr(arg_4, '__name__', 'f')
        arg_8 = "parsl_"
        arg_7 = arg_8 + "f"
        arg_9 = arg_8 + "args"
        arg_10 = arg_8 + "kwargs"
        arg_11 = arg_8 + "result"

        arg_3.update({arg_7: arg_4,
                        arg_9: arg_5,
                        arg_10: arg_6,
                        arg_11: arg_11})

        arg_12 = "{0} = {1}(*{2}, **{3})".format(arg_11, arg_7,
                                               arg_9, arg_10)

        try:
            logger.debug("[RUNNER] Executing: {0}".format(arg_12))
            exec(arg_12, arg_3, arg_3)

        except Exception as e:
            logger.warning("Caught exception; will raise it: {}".format(e))
            raise e

        else:
            logger.debug("[RUNNER] Result: {0}".format(arg_3.get(arg_11)))
            return arg_3.get(arg_11)

    while True:
        try:
            # Blocking wait on the queue
            arg_13 = arg_0.get(block=True, timeout=10)

        except queue.Empty:
            # Handle case where no items were in the queue
            logger.debug("[RUNNER] Queue is empty")

        except IOError as e:
            logger.debug("[RUNNER] Broken pipe: {}".format(e))
            try:
                # Attempt to send a stop notification to the management thread
                arg_1.put(None)

            except Exception:
                pass

            break

        except Exception as e:
            logger.debug("[RUNNER] Caught unknown exception: {}".format(e))

        else:
            # Handle received message
            if not arg_13:
                # Empty message is a die request
                logger.debug("[RUNNER] Received exit request")
                arg_1.put(None)
                break
            else:
                # Received a valid message, handle it
                logger.debug("[RUNNER] Got a valid task with ID {}".format(arg_13["task_id"]))
                try:
                    arg_14 = execute_task(arg_13['buffer'])
                    arg_15 = {"task_id": arg_13["task_id"],
                                "result": serialize_object(arg_14)}

                    logger.debug("[RUNNER] Returing result: {}".format(
                                   deserialize_object(arg_15["result"])))

                except Exception as e:
                    logger.debug("[RUNNER] Caught task exception: {}".format(e))
                    arg_15 = {"task_id": arg_13["task_id"],
                                "exception": serialize_object(e)}

                arg_1.put(arg_15)

    logger.debug("[RUNNER] Terminating")


def Func(arg_0):
        """Shutdown method, to kill the threads and workers."""
        arg_0.is_alive = False
        logging.debug("Waking management thread")
        arg_0.incoming_q.put(None)  # Wake up the thread
        arg_0._queue_management_thread.join()  # Force join
        logging.debug("Exiting thread")
        arg_0.worker.join()
        return True


def Func(arg_0, arg_1, *arg_2, **arg_3):
        """Submits work to the the outgoing_q.

        The outgoing_q is an external process listens on this
        queue for new work. This method is simply pass through and behaves like a
        Func call as described here `Python docs: <https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor>`_

        Args:
            - func (callable) : Callable function
            - *args (list) : List of arbitrary positional arguments.

        Kwargs:
            - **kwargs (dict) : A dictionary of arbitrary keyword args for func.

        Returns:
              Future
        """
        arg_4 = uuid.uuid4()

        logger.debug("Pushing function {} to queue with args {}".format(arg_1, arg_2))

        arg_0.tasks[arg_4] = Future()

        arg_6 = pack_apply_message(arg_1, arg_2, arg_3,
                                    buffer_threshold=1024 * 1024,
                                    item_threshold=1024)

        arg_7 = {"task_id": arg_4,
               "buffer": arg_6}

        # Post task to the the outgoing queue
        arg_0.outgoing_q.put(arg_7)

        # Return the future
        return arg_0.tasks[arg_4]


def Func(arg_0):
        """Return the resolved Func on the side where it is called from.

        The appropriate Func will be returned when called from within
        an app running remotely as well as regular python on the client side.

        Args:
            - self
        Returns:
             - Func (string)
        """
        if hasattr(arg_0, 'local_path'):
            return arg_0.local_path

        if arg_0.scheme in ['ftp', 'http', 'https', 'globus']:
            return arg_0.filename
        elif arg_0.scheme in ['file']:
            return arg_0.path
        else:
            raise Exception('Cannot return Func for unknown scheme {}'.format(arg_0.scheme))


def Func(arg_0, arg_1=None, arg_2=60, arg_3=False, arg_4='all'):
    """The Func decorator function.

    Args:
        - apptype (string) : Functype can be bash|python

    Kwargs:
        - data_flow_kernel (DataFlowKernel): The :class:`~parsl.dataflow.dflow.DataFlowKernel` responsible for
          managing this app. This can be omitted only
          after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`.
        - walltime (int) : Walltime for app in seconds,
             default=60
        - executors (str|list) : Labels of the executors that this app can execute over. Default is 'all'.
        - cache (Bool) : Enable caching of the app call
             default=False

    Returns:
         A PythonFunc or BashFunc object, which when called runs the apps through the executor.
    """

    from parsl.app.python import PythonFunc
    from parsl.app.bash import BashFunc

    logger.warning("The 'Func' decorator will be deprecated in Parsl 0.8. Please use 'python_app' or 'bash_app' instead.")

    if arg_0 == 'python':
        arg_5 = PythonFunc
    elif arg_0 == 'bash':
        arg_5 = BashFunc
    else:
        raise InvalidFuncTypeError("Invalid apptype requested {}; must be 'python' or 'bash'".format(arg_0))

    def wrapper(arg_6):
        return arg_5(arg_6,
                         arg_1=arg_1,
                         arg_2=arg_2,
                         arg_3=arg_3,
                         arg_4=arg_4)
    return wrapper