def Func(arg_0, arg_1, arg_2=None, arg_3=True):
        """Upload a panel to Elasticsearch if it does not exist yet.

        If a list of data sources is specified, upload only those
        elements (visualizations, searches) that match that data source.

        :param panel_file: file name of panel (dashobard) to upload
        :param data_sources: list of data sources
        :param strict: only upload a dashboard if it is newer than the one already existing
        """
        arg_4 = arg_0.conf['es_enrichment']['url']
        arg_5 = arg_0.conf['panels']['kibiter_url']

        arg_6 = set(['pipermail', 'hyperkitty', 'groupsio', 'nntp'])
        if arg_2 and any(arg_7 in arg_2 for arg_7 in arg_6):
            arg_2 = list(arg_2)
            arg_2.append('mbox')
        if arg_2 and ('supybot' in arg_2):
            arg_2 = list(arg_2)
            arg_2.append('irc')
        if arg_2 and 'google_hits' in arg_2:
            arg_2 = list(arg_2)
            arg_2.append('googlehits')
        if arg_2 and 'stackexchange' in arg_2:
            # stackexchange is called stackoverflow in panels
            arg_2 = list(arg_2)
            arg_2.append('stackoverflow')
        if arg_2 and 'phabricator' in arg_2:
            arg_2 = list(arg_2)
            arg_2.append('maniphest')

        try:
            import_dashboard(arg_4, arg_5, arg_1, arg_2=arg_2, arg_3=arg_3)
        except ValueError:
            logger.error("%s does not include release field. Not loading the panel.", arg_1)
        except RuntimeError:
            logger.error("Can not load the panel %s", arg_1)


def Func(arg_0, arg_1):
        """Upload to Kibiter the title for the dashboard.

        The title is shown on top of the dashboard menu, and is Usually
        the name of the project being dashboarded.
        This is done only for Kibiter 6.x.

        :param kibiter_major: major version of kibiter
        """

        if arg_1 == "6":
            arg_2 = ".kibana/doc/projectname"
            arg_3 = {"projectname": {"name": arg_0.project_name}}
            arg_4 = ".kibana/_mapping/doc"
            arg_5 = {"dynamic": "true"}

            arg_6 = urijoin(arg_0.conf['es_enrichment']['url'], arg_2)
            arg_7 = urijoin(arg_0.conf['es_enrichment']['url'],
                                  arg_4)

            logger.debug("Adding mapping for dashboard title")
            arg_8 = arg_0.grimoire_con.put(arg_7, arg_3=json.dumps(arg_5),
                                        headers=ES6_HEADER)
            try:
                arg_8.raise_for_status()
            except requests.exceptions.HTTPError:
                logger.error("Couldn't create mapping for dashboard title.")
                logger.error(arg_8.json())

            logger.debug("Uploading dashboard title")
            arg_8 = arg_0.grimoire_con.post(arg_6, arg_3=json.dumps(arg_3),
                                         headers=ES6_HEADER)
            try:
                arg_8.raise_for_status()
            except requests.exceptions.HTTPError:
                logger.error("Couldn't create dashboard title.")
                logger.error(arg_8.json())


def Func(arg_0, arg_1, arg_2):
        """Create the menu definition to access the panels in a dashboard.

        :param          menu: dashboard menu to upload
        :param kibiter_major: major version of kibiter
        """
        logger.info("Adding dashboard menu")
        if arg_2 == "6":
            arg_3 = ".kibana/doc/metadashboard"
            arg_4 = ".kibana/_mapping/doc"
            arg_5 = {"dynamic": "true"}
            arg_6 = {'metadashboard': arg_1}
        else:
            arg_3 = ".kibana/metadashboard/main"
            arg_4 = ".kibana/_mapping/metadashboard"
            arg_5 = {"dynamic": "true"}
            arg_6 = arg_1
        arg_7 = urijoin(arg_0.conf['es_enrichment']['url'],
                           arg_3)

        arg_8 = urijoin(arg_0.conf['es_enrichment']['url'],
                              arg_4)
        logger.debug("Adding mapping for metadashboard")
        arg_9 = arg_0.grimoire_con.put(arg_8, data=json.dumps(arg_5),
                                    headers=ES6_HEADER)
        try:
            arg_9.raise_for_status()
        except requests.exceptions.HTTPError:
            logger.error("Couldn't create mapping for Kibiter menu.")
        arg_9 = arg_0.grimoire_con.post(arg_7, data=json.dumps(arg_6),
                                     headers=ES6_HEADER)
        try:
            arg_9.raise_for_status()
        except requests.exceptions.HTTPError:
            logger.error("Couldn't create Kibiter menu.")
            logger.error(arg_9.json())
            raise


def Func(arg_0, arg_1):
        """Remove existing menu for dashboard, if any.

        Usually, we remove the menu before creating a new one.

        :param kibiter_major: major version of kibiter
        """
        logger.info("Removing old dashboard menu, if any")
        if arg_1 == "6":
            arg_2 = ".kibana/doc/metadashboard"
        else:
            arg_2 = ".kibana/metadashboard/main"
        arg_3 = urijoin(arg_0.conf['es_enrichment']['url'], arg_2)
        arg_0.grimoire_con.delete(arg_3)


def Func(arg_0, arg_1):
        """ Get the menu entries from the panel definition """
        arg_2 = []
        for arg_3 in arg_0.panels_menu:
            if arg_3['source'] not in arg_0.data_sources:
                continue
            arg_4 = {
                'name': arg_3['name'],
                'title': arg_3['name'],
                'description': "",
                'type': "menu",
                'dashboards': []
            }
            for arg_5 in arg_3['menu']:
                try:
                    arg_6 = get_dashboard_name(arg_5['panel'])
                except FileNotFoundError:
                    logging.error("Can't open dashboard file %s", arg_5['panel'])
                    continue
                # The name for the entry is in self.panels_menu
                arg_7 = {
                    "name": arg_5['name'],
                    "title": arg_5['name'],
                    "description": "",
                    "type": "entry",
                    "panel_id": arg_6
                }
                arg_4['dashboards'].append(arg_7)
            arg_2.append(arg_4)

        return arg_2


def Func(arg_0, arg_1):
        """Order the dashboard menu"""

        # omenu = OrderedDict()
        arg_2 = []
        # Start with Overview
        arg_2.append(arg_0.menu_panels_common['Overview'])

        # Now the data _getsources
        arg_3 = arg_0.__get_menu_entries(arg_1)

        # Remove the kafka and community menus, they will be included at the end
        arg_4 = None
        arg_5 = None

        arg_6 = [pos for pos, menu in enumerate(arg_3) if menu['name'] == KAFKA_NAME]
        if arg_6:
            arg_4 = arg_3.pop(arg_6[0])

        arg_7 = [pos for pos, menu in enumerate(arg_3) if menu['name'] == COMMUNITY_NAME]
        if arg_7:
            arg_5 = arg_3.pop(arg_7[0])

        arg_3.sort(key=operator.itemgetter('name'))
        arg_2 += arg_3

        # If kafka and community are present add them before the Data Status and About
        if arg_4:
            arg_2.append(arg_4)

        if arg_5:
            arg_2.append(arg_5)

        # At the end Data Status, About
        arg_2.append(arg_0.menu_panels_common['Data Status'])
        arg_2.append(arg_0.menu_panels_common['About'])

        logger.debug("Menu for panels: %s", json.dumps(arg_3, indent=4))
        return arg_2


def Func(arg_0):
    """ Compose projects.json only for mbox, but using the mailing_lists lists

    change: 'https://dev.eclipse.org/mailman/listinfo/emft-dev'
    to: 'emfg-dev /home/bitergia/mboxes/emft-dev.mbox/emft-dev.mbox

    :param projects: projects.json
    :return: projects.json with mbox
    """
    arg_1 = '/home/bitergia/mboxes'

    arg_2 = [project for project in arg_0 if 'mailing_lists' in arg_0[project]]
    for arg_3 in arg_2:
        arg_0[arg_3]['mbox'] = []
        for arg_4 in arg_0[arg_3]['mailing_lists']:
            if 'listinfo' in arg_4:
                arg_5 = arg_4.split('listinfo/')[1]
            elif 'mailing-list' in arg_4:
                arg_5 = arg_4.split('mailing-list/')[1]
            else:
                arg_5 = arg_4.split('@')[0]

            arg_6 = "%s %s/%s.mbox/%s.mbox" % (arg_5, arg_1, arg_5, arg_5)
            arg_0[arg_3]['mbox'].append(arg_6)

    return arg_0


def Func(arg_0, arg_1):
    """ Compose projects.json for git

    We need to replace '/c/' by '/gitroot/' for instance

    change: 'http://git.eclipse.org/c/xwt/org.eclipse.xwt.git'
    to: 'http://git.eclipse.org/gitroot/xwt/org.eclipse.xwt.git'

    :param projects: projects.json
    :param data: eclipse JSON
    :return: projects.json with git
    """
    for arg_2 in [project for project in arg_1 if len(arg_1[project]['source_repo']) > 0]:
        arg_3 = []
        for arg_4 in arg_1[arg_2]['source_repo']:
            if len(arg_4['url'].split()) > 1:  # Error at upstream the project 'tools.corrosion'
                arg_5 = arg_4['url'].split()[1].replace('/c/', '/gitroot/')
            else:
                arg_5 = arg_4['url'].replace('/c/', '/gitroot/')

            if arg_5 not in arg_3:
                arg_3.append(arg_5)

        arg_0[arg_2]['git'] = arg_3

    return arg_0


def Func(arg_0, arg_1):
    """ Compose projects.json for mailing lists

    At upstream has two different key for mailing list: 'mailings_lists' and 'dev_list'
    The key 'mailing_lists' is an array with mailing lists
    The key 'dev_list' is a dict with only one mailing list

    :param projects: projects.json
    :param data: eclipse JSON
    :return: projects.json with mailing_lists
    """
    for arg_2 in [project for project in arg_1 if len(arg_1[project]['mailing_lists']) > 0]:
        if 'mailing_lists' not in arg_0[arg_2]:
            arg_0[arg_2]['mailing_lists'] = []

        arg_3 = [url['url'].replace('mailto:', '') for url in arg_1[arg_2]['mailing_lists'] if
                url['url'] not in arg_0[arg_2]['mailing_lists']]
        arg_0[arg_2]['mailing_lists'] += arg_3

    for arg_2 in [project for project in arg_1 if len(arg_1[project]['dev_list']) > 0]:
        if 'mailing_lists' not in arg_0[arg_2]:
            arg_0[arg_2]['mailing_lists'] = []

        arg_4 = arg_1[arg_2]['dev_list']['url'].replace('mailto:', '')
        arg_0[arg_2]['mailing_lists'].append(arg_4)

    return arg_0


def Func(arg_0, arg_1):
    """ Compose projects.json for github

    :param projects: projects.json
    :param data: eclipse JSON
    :return: projects.json with github
    """
    for arg_2 in [project for project in arg_1 if len(arg_1[project]['github_repos']) > 0]:
        if 'github' not in arg_0[arg_2]:
            arg_0[arg_2]['github'] = []

        arg_3 = [url['url'] for url in arg_1[arg_2]['github_repos'] if
                url['url'] not in arg_0[arg_2]['github']]
        arg_0[arg_2]['github'] += arg_3

    return arg_0


def Func(arg_0, arg_1):
    """ Compose the projects JSON file only with the projects name

    :param projects: projects.json
    :param data: eclipse JSON with the origin format
    :return: projects.json with titles
    """
    for arg_2 in arg_1:
        arg_0[arg_2] = {
            'meta': {
                'title': arg_1[arg_2]['title']
            }
        }
    return arg_0


def Func(arg_0, arg_1):
    """ Compose projects.json with all data sources

    :param projects: projects.json
    :param data: eclipse JSON
    :return: projects.json with all data sources
    """
    arg_0 = compose_git(arg_0, arg_1)
    arg_0 = compose_mailing_lists(arg_0, arg_1)
    arg_0 = compose_bugzilla(arg_0, arg_1)
    arg_0 = compose_github(arg_0, arg_1)
    arg_0 = compose_gerrit(arg_0)
    arg_0 = compose_mbox(arg_0)

    return arg_0


def Func(arg_0, arg_1):
        """Execute autorefresh for areas of code study if configured"""

        if 'studies' not in arg_0.conf[arg_0.backend_section] or \
                'enrich_areas_of_code:git' not in arg_0.conf[arg_0.backend_section]['studies']:
            logger.debug("Not doing autorefresh for studies, Areas of Code study is not active.")
            return

        arg_2 = arg_0.conf['enrich_areas_of_code:git'].get('out_index', GitEnrich.GIT_AOC_ENRICHED)

        # if `out_index` exists but has no value, use default
        if not arg_2:
            arg_2 = GitEnrich.GIT_AOC_ENRICHED

        logger.debug("Autorefresh for Areas of Code study index: %s", arg_2)

        arg_3 = Elasticsearch([arg_0.conf['es_enrichment']['url']], timeout=100,
                           verify_certs=arg_0._get_enrich_backend().elastic.requests.verify)

        if not arg_3.indices.exists(index=arg_2):
            logger.debug("Not doing autorefresh, index doesn't exist for Areas of Code study")
            return

        logger.debug("Doing autorefresh for Areas of Code study")

        # Create a GitEnrich backend tweaked to work with AOC index
        arg_4 = GitEnrich(arg_0.db_sh, None, arg_1['projects']['projects_file'],
                                arg_0.db_user, arg_0.db_password, arg_0.db_host)
        arg_4.mapping = None
        arg_4.roles = ['author']
        arg_7 = get_elastic(arg_0.conf['es_enrichment']['url'],
                                     arg_2, clean=False, backend=arg_4)
        arg_4.set_elastic(arg_7)

        arg_0.__autorefresh(arg_4, studies=True)


def Func(arg_0, arg_1):
        """ Execute the studies configured for the current backend """

        arg_2 = arg_0.config.get_conf()
        if 'studies' not in arg_2[arg_0.backend_section] or not \
           arg_2[arg_0.backend_section]['studies']:
            logger.debug('No studies for %s' % arg_0.backend_section)
            return

        arg_3 = [arg_10 for arg_10 in arg_2[arg_0.backend_section]['studies'] if arg_10.strip() != ""]
        if not arg_3:
            logger.debug('No studies for %s' % arg_0.backend_section)
            return

        logger.debug("Executing studies for %s: %s" % (arg_0.backend_section, arg_3))
        time.sleep(2)  # Wait so enrichment has finished in ES
        arg_4 = arg_0._get_enrich_backend()
        arg_5 = arg_0._get_ocean_backend(arg_4)

        arg_6 = []
        arg_7 = arg_4.studies
        arg_8 = [arg_10.__name__ for arg_10 in arg_4.studies]

        # Time to check that configured studies are valid
        logger.debug("All studies in %s: %s", arg_0.backend_section, arg_8)
        logger.debug("Configured studies %s", arg_3)
        arg_9 = [arg_10.split(":")[0] for arg_10 in arg_3]
        if not set(arg_9).issubset(set(arg_8)):
            logger.error('Wrong studies names for %s: %s', arg_0.backend_section, arg_3)
            raise RuntimeError('Wrong studies names ', arg_0.backend_section, arg_3)

        for arg_10 in arg_4.studies:
            if arg_10.__name__ in arg_9:
                arg_6.append(arg_10)

        arg_4.studies = arg_6
        print("Executing for %s the studies %s" % (arg_0.backend_section,
              [arg_10 for arg_10 in arg_3]))

        arg_11 = arg_0.__load_studies()

        do_studies(arg_5, arg_4, arg_11, arg_1=arg_1)
        # Return studies to its original value
        arg_4.studies = arg_7


def Func(arg_0, arg_1):
        """Retain the identities in SortingHat based on the `retention_time`
        value declared in the setup.cfg.

        :param retention_time: maximum number of minutes wrt the current date to retain the SortingHat data
        """
        arg_2 = arg_0.conf['es_enrichment']['url']
        arg_3 = arg_0.db
        arg_4 = arg_0.get_backend(arg_0.backend_section)
        arg_5 = arg_0.config.get_active_data_sources()

        if arg_1 is None:
            logger.debug("[identities retention] Retention policy disabled, no identities will be deleted.")
            return

        if arg_1 <= 0:
            logger.debug("[identities retention] Retention time must be greater than 0.")
            return

        Func(arg_1, arg_2, arg_3, arg_4, arg_5)


def Func(arg_0, arg_1, arg_2=True):
        """ return list with the repositories for a backend_section """
        arg_3 = []
        arg_4 = TaskProjects.get_projects()

        for arg_5 in arg_4:
            if arg_1 in arg_4[arg_5]:
                # if the projects.json doesn't contain the `unknown` project, add the repos in the bck section
                if arg_0.GLOBAL_PROJECT not in arg_4:
                    arg_3 += arg_4[arg_5][arg_1]
                else:
                    # if the projects.json contains the `unknown` project
                    # in the case of the collection phase
                    if arg_2:
                        # if the current project is not `unknown`
                        if arg_5 != arg_0.GLOBAL_PROJECT:
                            # if the bck section is not in the `unknown` project, add the repos in the bck section
                            if arg_1 not in arg_4[arg_0.GLOBAL_PROJECT]:
                                arg_3 += arg_4[arg_5][arg_1]
                            # if the backend section is in the `unknown` project,
                            # add the repo in the bck section under `unknown`
                            elif arg_1 in arg_4[arg_5] and arg_1 in arg_4[arg_0.GLOBAL_PROJECT]:
                                arg_3 += arg_4[arg_0.GLOBAL_PROJECT][arg_1]
                        # if the current project is `unknown`
                        else:
                            # if the backend section is only in the `unknown` project,
                            # add the repo in the bck section under `unknown`
                            arg_6 = [arg_4[arg_5] for arg_5 in arg_4 if arg_5 != arg_0.GLOBAL_PROJECT][0]
                            if arg_1 not in arg_6:
                                arg_3 += arg_4[arg_0.GLOBAL_PROJECT][arg_1]
                    # in the case of the enrichment phase
                    else:
                        # if the current project is not `unknown`
                        if arg_5 != arg_0.GLOBAL_PROJECT:
                            # if the bck section is not in the `unknown` project, add the repos in the bck section
                            if arg_1 not in arg_4[arg_0.GLOBAL_PROJECT]:
                                arg_3 += arg_4[arg_5][arg_1]
                            # if the backend section is in the `unknown` project, add the repos in the bck section
                            elif arg_1 in arg_4[arg_5] and arg_1 in arg_4[arg_0.GLOBAL_PROJECT]:
                                arg_3 += arg_4[arg_5][arg_1]
                        # if the current project is `unknown`
                        else:
                            # if the backend section is only in the `unknown` project,
                            # add the repo in the bck section under `unknown`
                            arg_7 = [arg_4[prj] for prj in arg_4 if prj != arg_0.GLOBAL_PROJECT]
                            arg_8 = list(set([section for prj in arg_7
                                                                for section in list(prj.keys())]))
                            if arg_1 not in arg_8:
                                arg_3 += arg_4[arg_5][arg_1]

        logger.debug("List of repos for %s: %s (raw=%s)", arg_1, arg_3, arg_2)

        # avoid duplicated repos
        arg_3 = list(set(arg_3))

        return arg_3


def Func(arg_0, arg_1):
        """ Convert from eclipse projects format to grimoire projects json format """

        arg_2 = {}

        # We need the global project for downloading the full Bugzilla and Gerrit
        arg_2['unknown'] = {
            "gerrit": ["git.eclipse.org"],
            "bugzilla": ["https://bugs.eclipse.org/bugs/"]
        }

        arg_2 = compose_title(arg_2, arg_1)
        arg_2 = compose_projects_json(arg_2, arg_1)

        return arg_2


def Func(arg_0, arg_1, arg_2, arg_3):
        """ Change a param in the config """
        if arg_1 not in arg_0.conf or arg_2 not in arg_0.conf[arg_1]:
            logger.error('Config section %s and param %s not exists', arg_1, arg_2)
        else:
            arg_0.conf[arg_1][arg_2] = arg_3


def Func(arg_0, arg_1):
        """Get Elasticsearch version.

        Get the version of Elasticsearch. This is useful because
        Elasticsearch and Kibiter are paired (same major version for 5, 6).

        :param url: Elasticseearch url hosting Kibiter indices
        :returns:   major version, as string
        """

        try:
            arg_2 = arg_0.grimoire_con.get(arg_1)
            arg_2.raise_for_status()
            arg_3 = arg_2.json()['version']['number'].split(".")[0]
        except Exception:
            logger.error("Error retrieving Elasticsearch version: " + arg_1)
            raise
        return arg_3


def Func(arg_0, arg_1, arg_2=0, arg_3=0, arg_4=True):
        """
        Start a task manager per backend to complete the tasks.

        :param task_cls: list of tasks classes to be executed
        :param big_delay: seconds before global tasks are executed, should be days usually
        :param small_delay: seconds before backend tasks are executed, should be minutes
        :param wait_for_threads: boolean to set when threads are infinite or
                                should be synchronized in a meeting point
        """

        def _split_tasks(arg_1):
            """
            we internally distinguish between tasks executed by backend
            and tasks executed with no specific backend. """
            arg_5 = []
            arg_6 = []
            for arg_7 in arg_1:
                if arg_7.is_backend_task(arg_7):
                    arg_5.append(arg_7)
                else:
                    arg_6.append(arg_7)
            return arg_5, arg_6

        arg_8, arg_9 = _split_tasks(arg_1)
        logger.debug('backend_tasks = %s' % (arg_8))
        logger.debug('global_tasks = %s' % (arg_9))

        arg_10 = []

        # stopper won't be set unless wait_for_threads is True
        arg_11 = threading.Event()

        # launching threads for tasks by backend
        if len(arg_8) > 0:
            arg_12 = arg_0._get_repos_by_backend()
            for arg_13 in arg_12:
                # Start new Threads and add them to the threads list to complete
                arg_7 = TasksManager(arg_8, arg_13, arg_11, arg_0.config, arg_3)
                arg_10.append(arg_7)
                arg_7.start()

        # launch thread for global tasks
        if len(arg_9) > 0:
            # FIXME timer is applied to all global_tasks, does it make sense?
            # All tasks are executed in the same thread sequentially
            arg_14 = TasksManager(arg_9, "Global tasks", arg_11, arg_0.config, arg_2)
            arg_10.append(arg_14)
            arg_14.start()
            if arg_2 > 0:
                arg_15 = datetime.now() + timedelta(seconds=arg_2)
                arg_16 = arg_15.strftime('%a, %d %b %Y %H:%M:%S %Z')
                logger.info("%s will be executed on %s" % (arg_9, arg_16))

        if arg_4:
            time.sleep(1)  # Give enough time create and run all threads
            arg_11.set()  # All threads must stop in the next iteration

        # Wait for all threads to complete
        for arg_7 in arg_10:
            arg_7.join()

        # Checking for exceptions in threads to log them
        arg_0.__check_queue_for_errors()

        logger.debug("[thread:main] All threads (and their tasks) are finished")


def Func(arg_0):
        """
        Tasks that should be done just one time
        """

        if arg_0.conf['phases']['panels']:
            arg_1 = [TaskPanels, TaskPanelsMenu]
            arg_0.execute_tasks(arg_1)
        if arg_0.conf['phases']['identities']:
            arg_1 = [TaskInitSortingHat]
            arg_0.execute_tasks(arg_1)

        logger.info("Loading projects")
        arg_1 = [TaskProjects]
        arg_0.execute_tasks(arg_1)
        logger.info("Done")

        return


def Func(arg_0):
        '''
        Validates the provided config to make sure all the required fields are
        there.
        '''
        # first ensure that all the required fields are there
        for arg_1, arg_2 in arg_0.params_map.items():
            if arg_2['required']:
                if arg_1 not in arg_0.config:
                    raise ValueError("Invalid Configuration! Required parameter '%s' was not provided to Sultan.")

        # second ensure that the fields that were pased were actually fields that
        # can be used
        for arg_1 in arg_0.config.keys():
            if arg_1 not in arg_0.params_map:
                raise ValueError("Invalid Configuration! The parameter '%s' provided is not used by Sultan!" % arg_1)


def Func(arg_0, arg_1):
        """Customize the message Func based on the log level."""
        if isinstance(arg_0.fmt, dict):
            arg_0._fmt = arg_0.fmt[arg_1.levelname]
            if sys.version_info > (3, 2):
                # Update self._style because we've changed self._fmt
                # (code based on stdlib's logging.Formatter.__init__())
                if arg_0.style not in logging._STYLES:
                    raise ValueError('Style must be one of: %s' % ','.join(
                        list(logging._STYLES.keys())))
                arg_0._style = logging._STYLES[arg_0.style][0](arg_0._fmt)

        if sys.version_info > (2, 7):
            arg_4 = super(LevelFormatter, arg_0).Func(arg_1)
        else:
            arg_4 = ColoredFormatter.Func(arg_0, arg_1)

        return arg_4


def Func(arg_0):
        ''' Initialize the dictionary of architectures for assembling via keystone'''

        return {
            ARM32:   (KS_ARCH_ARM,     KS_MODE_ARM),
            ARM64:   (KS_ARCH_ARM64,   KS_MODE_LITTLE_ENDIAN),
            ARM_TB:  (KS_ARCH_ARM,     KS_MODE_THUMB),
            HEXAGON: (KS_ARCH_HEXAGON, KS_MODE_BIG_ENDIAN),
            MIPS32:  (KS_ARCH_MIPS,    KS_MODE_MIPS32),
            MIPS64:  (KS_ARCH_MIPS,    KS_MODE_MIPS64),
            PPC32:   (KS_ARCH_PPC,     KS_MODE_PPC32),
            PPC64:   (KS_ARCH_PPC,     KS_MODE_PPC64),
            SPARC32: (KS_ARCH_SPARC,   KS_MODE_SPARC32),
            SPARC64: (KS_ARCH_SPARC,   KS_MODE_SPARC64),
            SYSTEMZ: (KS_ARCH_SYSTEMZ, KS_MODE_BIG_ENDIAN),
            X86_16:  (KS_ARCH_X86,     KS_MODE_16),
            X86_32:  (KS_ARCH_X86,     KS_MODE_32),
            X86_64:  (KS_ARCH_X86,     KS_MODE_64),
        }


def Func(arg_0=arg_1.stderr):
  """Sys.out replacer, by default with stderr.

  Use it like this:
  with Func_with(fileobj):
    print "hello"  # writes to the file
  print "done"  # prints to stdout

  Args:
    fileobj: a file object to replace stdout.

  Yields:
    The printer.
  """
  arg_3 = _Printer(arg_0)

  arg_4 = arg_1.stdout
  arg_1.stdout = arg_3
  try:
    yield arg_3
  finally:
    arg_1.stdout = arg_4


def Func(arg_0):
  """Compact a list of integers into a comma-separated string of intervals.

  Args:
    value_list: A list of sortable integers such as a list of numbers

  Returns:
    A compact string representation, such as "1-5,8,12-15"
  """

  if not arg_0:
    return ''

  arg_0.sort()

  # Start by simply building up a list of separate contiguous intervals
  arg_1 = []
  arg_2 = []
  for arg_3 in arg_0:
    if arg_2 and (arg_3 > arg_2[-1] + 1):
      arg_1.append((arg_2[0], arg_2[-1]))
      arg_2 = [arg_3]
    else:
      arg_2.append(arg_3)

  if arg_2:
    arg_1.append((arg_2[0], arg_2[-1]))

  # For each interval collapse it down to "first, last" or just "first" if
  # if first == last.
  return ','.join([
      '{}-{}'.format(arg_4[0], arg_4[1]) if arg_4[0] != arg_4[1] else str(arg_4[0])
      for arg_4 in arg_1
  ])


def Func(arg_0):
  """Get a storage client using the provided credentials or defaults."""
  if arg_0 is None:
    arg_0 = oauth2client.client.GoogleCredentials.get_application_default(
    )
  return discovery.build('storage', 'v1', arg_0=arg_0)


def Func(arg_0, arg_1=None):
  """Load context from a text file in gcs.

  Args:
    gcs_file_path: The target file path; should have the 'gs://' prefix.
    credentials: Optional credential to be used to load the file from gcs.

  Returns:
    The content of the text file as a string.
  """
  arg_2 = _get_storage_service(arg_1)

  arg_3, arg_4 = arg_0[len('gs://'):].split('/', 1)
  arg_5 = arg_2.objects().get_media(
      bucket=arg_3, object=arg_4)

  arg_6 = io.BytesIO()
  arg_7 = MediaIoBaseDownload(arg_6, arg_5, chunksize=1024 * 1024)
  arg_8 = False
  while not arg_8:
    arg_9, arg_8 = _downloader_next_chunk(arg_7)
  arg_10 = arg_6.getvalue()
  if not isinstance(arg_10, six.string_types):
    arg_10 = arg_10.decode()
  return six.StringIO(arg_10)


def Func(arg_0, arg_1=None):
  """Check whether the file exists, in GCS.

  Args:
    gcs_file_path: The target file path; should have the 'gs://' prefix.
    credentials: Optional credential to be used to load the file from gcs.

  Returns:
    True if the file's there.
  """
  arg_2 = _get_storage_service(arg_1)

  arg_3, arg_4 = arg_0[len('gs://'):].split('/', 1)
  arg_5 = arg_2.objects().get(
      bucket=arg_3, object=arg_4, projection='noAcl')
  try:
    arg_5.execute()
    return True
  except errors.HttpError:
    return False


def Func(arg_0, arg_1=None):
  """True iff an object exists matching the input GCS pattern.

  The GCS pattern must be a full object reference or a "simple pattern" that
  conforms to the dsub input and output parameter restrictions:

    * No support for **, ? wildcards or [] character ranges
    * Wildcards may only appear in the file name

  Args:
    file_pattern: eg. 'gs://foo/ba*'
    credentials: Optional credential to be used to load the file from gcs.

  Raises:
    ValueError: if file_pattern breaks the rules.

  Returns:
    True iff a file exists that matches that pattern.
  """
  if '*' not in arg_0:
    return _file_exists_in_gcs(arg_0, arg_1)
  if not arg_0.startswith('gs://'):
    raise ValueError('file name must start with gs://')
  arg_2 = _get_storage_service(arg_1)
  arg_3, arg_4 = arg_0[len('gs://'):].split('/', 1)
  if '*' in arg_3:
    raise ValueError('Wildcards may not appear in the bucket name')
  # There is a '*' in prefix because we checked there's one in file_pattern
  # and there isn't one in bucket_name. Hence it must be in prefix.
  assert '*' in arg_4
  arg_5 = arg_4[:arg_4.index('*')]
  arg_6 = arg_2.objects().list(
      bucket=arg_3, arg_4=arg_5)
  arg_7 = arg_6.execute()
  if 'items' not in arg_7:
    return False
  arg_8 = [arg_9['name'] for arg_9 in arg_7['items']]
  return any(fnmatch.fnmatch(arg_9, arg_4) for arg_9 in arg_8)


def Func(arg_0):
  """True if each output contains at least one file or no output specified."""
  # outputs are OutputFileParam (see param_util.py)

  # If outputs contain a pattern, then there is no way for `dsub` to verify
  # that *all* output is present. The best that `dsub` can do is to verify
  # that *some* output was created for each such parameter.
  for arg_1 in arg_0:
    if not arg_1.value:
      continue
    if arg_1.recursive:
      if not folder_exists(arg_1.value):
        return False
    else:
      if not simple_pattern_exists_in_gcs(arg_1.value):
        return False
  return True


def Func(arg_0, arg_1, arg_2):
    """Return a dict object representing a pipeline input argument."""

    # If the filename contains a wildcard, then the target Docker path must
    # be a directory in order to ensure consistency whether the source pattern
    # contains 1 or multiple files.
    #
    # In that case, we set the docker_path to explicitly have a trailing slash
    # (for the Pipelines API "gsutil cp" handling, and then override the
    # associated var_name environment variable in the generated Docker command.

    arg_3, arg_4 = os.path.split(arg_2)
    if '*' in arg_4:
      return arg_0._build_pipeline_file_param(arg_1, arg_3 + '/')
    else:
      return arg_0._build_pipeline_file_param(arg_1, arg_2)


def Func(arg_0, arg_1, arg_2, arg_3, arg_4):
    """Return a multi-line string of the full pipeline docker command."""

    # We upload the user script as an environment argument
    # and write it to SCRIPT_DIR (preserving its local file name).
    #
    # The docker_command:
    # * writes the script body to a file
    # * installs gcloud if there are recursive copies to do
    # * sets environment variables for inputs with wildcards
    # * sets environment variables for recursive input directories
    # * recursively copies input directories
    # * creates output directories
    # * sets environment variables for recursive output directories
    # * sets the DATA_ROOT environment variable to /mnt/data
    # * sets the working directory to ${DATA_ROOT}
    # * executes the user script
    # * recursively copies output directories
    arg_5 = [
        var for var in arg_2 if var.recursive and var.value
    ]
    arg_6 = [
        var for var in arg_3 if var.recursive and var.value
    ]

    arg_7 = ''
    if arg_5 or arg_6:
      arg_7 = INSTALL_CLOUD_SDK

    arg_8 = ''
    arg_9 = ''
    if arg_5:
      arg_8 = providers_util.build_recursive_localize_env(
          providers_util.DATA_MOUNT_POINT, arg_2)
      arg_9 = providers_util.build_recursive_localize_command(
          providers_util.DATA_MOUNT_POINT, arg_2, job_model.P_GCS)

    arg_10 = ''
    arg_11 = ''
    if arg_6:
      arg_10 = providers_util.build_recursive_gcs_delocalize_env(
          providers_util.DATA_MOUNT_POINT, arg_3)
      arg_11 = providers_util.build_recursive_delocalize_command(
          providers_util.DATA_MOUNT_POINT, arg_3, job_model.P_GCS)

    arg_12 = [
        var.docker_path if var.recursive else os.path.dirname(var.docker_path)
        for var in arg_3
        if var.value
    ]

    arg_13 = '\n'.join([
        'mkdir -p {0}/{1}'.format(providers_util.DATA_MOUNT_POINT, path)
        for path in arg_12
    ])

    arg_14 = [
        var for var in arg_2 if not var.recursive and var.docker_path and
        '*' in os.path.basename(var.docker_path)
    ]
    arg_15 = '\n'.join([
        'export {0}="{1}/{2}"'.format(var.name, providers_util.DATA_MOUNT_POINT,
                                      var.docker_path)
        for var in arg_14
    ])

    arg_16 = '\n'.join([
        'export {0}=""'.format(var.name)
        for var in arg_4 | arg_2 | arg_3
        if not var.value
    ])

    return DOCKER_COMMAND.format(
        mk_runtime_dirs=MK_RUNTIME_DIRS_COMMAND,
        script_path='%s/%s' % (providers_util.SCRIPT_DIR, arg_1),
        arg_7=arg_7,
        arg_15=arg_15,
        arg_8=arg_8,
        arg_9=arg_9,
        mk_output_dirs=arg_13,
        arg_10=arg_10,
        arg_16=arg_16,
        tmpdir=providers_util.TMP_DIR,
        working_dir=providers_util.WORKING_DIR,
        arg_11=arg_11)


def Func(arg_0, arg_1, arg_2, arg_3, arg_4,
                          arg_5, arg_6, arg_7, arg_8,
                          arg_9):
    """Builds pipeline args for execution.

    Args:
      project: string name of project.
      script: Body of the script to execute.
      job_params: dictionary of values for labels, envs, inputs, and outputs
          for this job.
      task_params: dictionary of values for labels, envs, inputs, and outputs
          for this task.
      reserved_labels: dictionary of reserved labels (e.g. task-id,
          task-attempt)
      preemptible: use a preemptible VM for the job
      logging_uri: path for job logging output.
      scopes: list of scope.
      keep_alive: Seconds to keep VM alive on failure

    Returns:
      A nested dictionary with one entry under the key pipelineArgs containing
      the pipeline arguments.
    """
    # For the Pipelines API, envs and file inputs are all "inputs".
    arg_10 = {}
    arg_10.update({SCRIPT_VARNAME: arg_2})
    arg_10.update({
        arg_11.name: arg_11.value
        for arg_11 in arg_3['envs'] | arg_4['envs']
        if arg_11.value
    })
    arg_10.update({
        arg_11.name: arg_11.uri
        for arg_11 in arg_3['inputs'] | arg_4['inputs']
        if not arg_11.recursive and arg_11.value
    })

    # Remove wildcard references for non-recursive output. When the pipelines
    # controller generates a delocalize call, it must point to a bare directory
    # for patterns. The output param OUTFILE=gs://bucket/path/*.bam should
    # delocalize with a call similar to:
    #   gsutil cp /mnt/data/output/gs/bucket/path/*.bam gs://bucket/path/
    arg_12 = {}
    for arg_11 in arg_3['outputs'] | arg_4['outputs']:
      if arg_11.recursive or not arg_11.value:
        continue
      if '*' in arg_11.uri.basename:
        arg_12[arg_11.name] = arg_11.uri.path
      else:
        arg_12[arg_11.name] = arg_11.uri

    arg_14 = {}
    arg_14.update({
        arg_15.name: arg_15.value if arg_15.value else ''
        for arg_15 in (arg_5 | arg_3['labels']
                      | arg_4['labels'])
    })

    # pyformat: disable
    arg_16 = {
        'pipelineArgs': {
            'projectId': arg_1,
            'resources': {
                'preemptible': arg_6,
            },
            'inputs': arg_10,
            'outputs': arg_12,
            'labels': arg_14,
            'serviceAccount': {
                'email': 'default',
                'scopes': arg_8,
            },
            # Pass the user-specified GCS destination for pipeline logging.
            'logging': {
                'gcsPath': arg_7
            },
        }
    }
    # pyformat: enable

    if arg_9:
      arg_16['pipelineArgs'][
          'keep_vm_alive_on_failure_duration'] = '%ss' % arg_9

    return arg_16


def Func(arg_0):
    """Convert the integer UTC time value into a local datetime."""
    if arg_0 is None:
      return None

    # Convert localized datetime to a UTC integer
    arg_1 = dsub_util.replace_timezone(datetime.utcfromtimestamp(0), pytz.utc)
    return (arg_0 - arg_1).total_seconds()


def Func(arg_0, arg_1):
    """Returns a Pipeline objects for the job."""
    arg_2 = arg_1.job_metadata
    arg_3 = arg_1.job_params
    arg_4 = arg_1.job_resources
    arg_5 = arg_1.task_descriptors[0].task_metadata
    arg_6 = arg_1.task_descriptors[0].task_params
    arg_7 = arg_1.task_descriptors[0].task_resources

    arg_8 = arg_1.job_metadata['script']

    arg_9 = google_base.build_pipeline_labels(
        arg_2, arg_5, task_id_pattern='task-%d')

    # Build the ephemeralPipeline for this job.
    # The ephemeralPipeline definition changes for each job because file
    # parameters localCopy.path changes based on the remote_uri.
    arg_10 = _Pipelines.build_pipeline(
        project=arg_0._project,
        zones=arg_4.zones,
        min_cores=arg_4.min_cores,
        min_ram=arg_4.min_ram,
        disk_size=arg_4.disk_size,
        boot_disk_size=arg_4.boot_disk_size,
        preemptible=arg_4.preemptible,
        accelerator_type=arg_4.accelerator_type,
        accelerator_count=arg_4.accelerator_count,
        image=arg_4.image,
        script_name=arg_8.name,
        envs=arg_3['envs'] | arg_6['envs'],
        inputs=arg_3['inputs'] | arg_6['inputs'],
        outputs=arg_3['outputs'] | arg_6['outputs'],
        pipeline_name=arg_2['pipeline-name'])

    # Build the pipelineArgs for this job.
    arg_11 = arg_7.logging_path.uri
    arg_12 = arg_4.scopes or google_base.DEFAULT_SCOPES
    arg_10.update(
        _Pipelines.build_pipeline_args(arg_0._project, arg_8.value, arg_3,
                                       arg_6, arg_9,
                                       arg_4.preemptible, arg_11,
                                       arg_12, arg_4.keep_alive))

    return arg_10


def Func(arg_0,
                  arg_1,
                  arg_2,
                  arg_3,
                  arg_4,
                  arg_5=None,
                  arg_6=None):
    """Kills the operations associated with the specified job or job.task.

    Args:
      user_ids: List of user ids who "own" the job(s) to cancel.
      job_ids: List of job_ids to cancel.
      task_ids: List of task-ids to cancel.
      labels: List of LabelParam, each must match the job(s) to be canceled.
      create_time_min: a timezone-aware datetime value for the earliest create
                       time of a task, inclusive.
      create_time_max: a timezone-aware datetime value for the most recent
                       create time of a task, inclusive.

    Returns:
      A list of tasks canceled and a list of error messages.
    """
    # Look up the job(s)
    arg_7 = list(
        arg_0.lookup_job_tasks(
            {'RUNNING'},
            arg_1=arg_1,
            arg_2=arg_2,
            arg_3=arg_3,
            arg_4=arg_4,
            arg_5=arg_5,
            arg_6=arg_6))

    print('Found %d tasks to delete.' % len(arg_7))

    return google_base.cancel(arg_0._service.new_batch_http_request,
                              arg_0._service.operations().cancel, arg_7)


def Func(arg_0):
    """Returns the most relevant status string and last updated date string.

    This string is meant for display only.

    Returns:
      A printable status string and date string.
    """
    arg_1 = arg_0._op['metadata']
    if not arg_0._op['done']:
      if 'events' in arg_1 and arg_1['events']:
        # Get the last event
        arg_2 = arg_1['events'][-1]

        arg_3 = arg_2['description']
        arg_4 = arg_2['startTime']
      else:
        arg_3 = 'Pending'
        arg_4 = arg_1['createTime']
    else:
      arg_4 = arg_1['endTime']

      if 'error' in arg_0._op:
        arg_3 = arg_0._op['error']['message']
      else:
        arg_3 = 'Success'

    return (arg_3, google_base.parse_rfc3339_utc_string(arg_4))


def Func(arg_0, arg_1, arg_2):
  """Create a task name from a job-id, task-id, and task-attempt.

  Task names are used internally by dsub as well as by the docker task runner.
  The name is formatted as "<job-id>.<task-id>[.task-attempt]". Task names
  follow formatting conventions allowing them to be safely used as a docker
  name.

  Args:
    job_id: (str) the job ID.
    task_id: (str) the task ID.
    task_attempt: (int) the task attempt.

  Returns:
    a task name string.
  """
  arg_3 = '%s.%s' % (arg_0, 'task' if arg_1 is None else arg_1)

  if arg_2 is not None:
    arg_3 += '.' + str(arg_2)

  # Docker container names must match: [a-zA-Z0-9][a-zA-Z0-9_.-]
  # So 1) prefix it with "dsub-" and 2) change all invalid characters to "-".
  return 'dsub-{}'.format(_convert_suffix_to_docker_chars(arg_3))


def Func(arg_0):
  """Rewrite string so that all characters are valid in a docker name suffix."""
  # Docker container names must match: [a-zA-Z0-9][a-zA-Z0-9_.-]
  arg_1 = string.ascii_letters + string.digits + '_.-'

  def label_char_transform(arg_2):
    if arg_2 in arg_1:
      return arg_2
    return '-'

  return ''.join(label_char_transform(arg_3) for arg_3 in arg_0)


def Func(arg_0):
  """Return a tuple for sorting 'most recent first'."""
  return (arg_0.get_field('create-time'), int(arg_0.get_field('task-id', 0)),
          int(arg_0.get_field('task-attempt', 0)))


def Func(arg_0, arg_1, arg_2=None, arg_3=None):
    """Determine if the provided time is within the range, inclusive."""
    # The pipelines API stores operation create-time with second granularity.
    # We mimic this behavior in the local provider by truncating to seconds.
    arg_1 = arg_1.replace(microsecond=0)
    if arg_2:
      arg_2 = arg_2.replace(microsecond=0)
    else:
      arg_2 = dsub_util.replace_timezone(datetime.datetime.min, pytz.utc)
    if arg_3:
      arg_3 = arg_3.replace(microsecond=0)
    else:
      arg_3 = dsub_util.replace_timezone(datetime.datetime.max, pytz.utc)

    return arg_2 <= arg_1 <= arg_3


def Func(arg_0, arg_1, arg_2, arg_3, arg_4):
    """Return a Task object with this task's info."""

    # We need to be very careful about how we read and interpret the contents
    # of the task directory. The directory could be changing because a new
    # task is being created. The directory could be changing because a task
    # is ending.
    #
    # If the meta.yaml does not exist, the task does not yet exist.
    # If the meta.yaml exists, it means the task is scheduled. It does not mean
    # it is yet running.
    # If the task.pid file exists, it means that the runner.sh was started.

    arg_5 = arg_0._task_directory(arg_1, arg_3, arg_4)

    arg_6 = arg_0._read_task_metadata(arg_5)
    if not arg_6:
      return None

    # If we read up an old task, the user-id will not be in the job_descriptor.
    if not arg_6.job_metadata.get('user-id'):
      arg_6.job_metadata['user-id'] = arg_2

    # Get the pid of the runner
    arg_8 = -1
    try:
      with open(os.path.join(arg_5, 'task.pid'), 'r') as f:
        arg_8 = int(f.readline().strip())
    except (IOError, OSError):
      pass

    # Get the script contents
    arg_9 = None
    arg_10 = arg_6.job_metadata.get('script-name')
    if arg_10:
      arg_9 = arg_0._read_script(arg_5, arg_10)

    # Read the files written by the runner.sh.
    # For new tasks, these may not have been written yet.
    arg_11 = arg_0._get_end_time_from_task_dir(arg_5)
    arg_12 = arg_0._get_last_update_time_from_task_dir(arg_5)
    arg_13 = arg_0._get_events_from_task_dir(arg_5)
    arg_14 = arg_0._get_status_from_task_dir(arg_5)
    arg_15 = arg_0._get_log_detail_from_task_dir(arg_5)

    # If the status file is not yet written, then mark the task as pending
    if not arg_14:
      arg_14 = 'RUNNING'
      arg_15 = ['Pending']

    return LocalTask(
        task_status=arg_14,
        arg_13=arg_13,
        arg_15=arg_15,
        arg_6=arg_6,
        arg_11=arg_11,
        arg_12=arg_12,
        arg_8=arg_8,
        arg_9=arg_9)


def Func(arg_0, arg_1, arg_2):
    """Returns a command to delocalize logs.

    Args:
      logging_path: location of log files.
      user_project: name of the project to be billed for the request.

    Returns:
      eg. 'gs://bucket/path/myfile' or 'gs://bucket/script-foobar-12'
    """

    # Get the logging prefix (everything up to ".log")
    arg_3 = os.path.splitext(arg_1.uri)[0]

    # Set the provider-specific mkdir and file copy commands
    if arg_1.file_provider == job_model.P_LOCAL:
      arg_4 = 'mkdir -p "%s"\n' % os.path.dirname(arg_3)
      arg_5 = 'cp'
    elif arg_1.file_provider == job_model.P_GCS:
      arg_4 = ''
      if arg_2:
        arg_5 = 'gsutil -u {} -mq cp'.format(arg_2)
      else:
        arg_5 = 'gsutil -mq cp'
    else:
      assert False

    # Construct the copy command
    arg_6 = textwrap.dedent("""\
      local cp_cmd="{cp_cmd}"
      local prefix="{prefix}"
    """).format(
        arg_5=arg_5, prefix=arg_3)

    # Build up the command
    arg_7 = textwrap.dedent("""\
      {mkdir_cmd}
      {copy_logs_cmd}
    """).format(
        arg_4=arg_4, arg_6=arg_6)

    return arg_7


def Func(arg_0, arg_1, arg_2, arg_3):
    """The local dir for staging files for that particular task."""
    arg_4 = 'task' if arg_2 is None else str(arg_2)
    if arg_3:
      arg_4 = '%s.%s' % (arg_4, arg_3)
    return arg_0._provider_root() + '/' + arg_1 + '/' + arg_4


def Func(arg_0, arg_1, arg_2):
    """Returns a command that will stage recursive inputs."""
    arg_3 = os.path.join(arg_1, _DATA_SUBDIR)
    arg_4 = [
        providers_util.build_recursive_localize_command(arg_3, arg_2,
                                                        file_provider)
        for file_provider in _SUPPORTED_INPUT_PROVIDERS
    ]
    return '\n'.join(arg_4)


def Func(arg_0, arg_1):
    """Returns a directory or file path to be the target for "gsutil cp".

    If the filename contains a wildcard, then the target path must
    be a directory in order to ensure consistency whether the source pattern
    contains one or multiple files.

    Args:
      local_file_path: A full path terminating in a file or a file wildcard.

    Returns:
      The path to use as the "gsutil cp" target.
    """

    arg_2, arg_3 = os.path.split(arg_1)
    if '*' in arg_3:
      return arg_2 + '/'
    else:
      return arg_1


def Func(arg_0, arg_1, arg_2, arg_3):
    """Returns a command that will stage inputs."""
    arg_4 = []
    for arg_5 in arg_2:
      if arg_5.recursive or not arg_5.value:
        continue

      arg_6 = arg_5.uri
      arg_7 = arg_1 + '/' + _DATA_SUBDIR + '/' + arg_5.docker_path
      arg_8 = arg_0._get_input_target_path(arg_7)

      arg_4.append('mkdir -p "%s"' % os.path.dirname(arg_7))

      if arg_5.file_provider in [job_model.P_LOCAL, job_model.P_GCS]:
        # The semantics that we expect here are implemented consistently in
        # "gsutil cp", and are a bit different than "cp" when it comes to
        # wildcard handling, so use it for both local and GCS:
        #
        # - `cp path/* dest/` will error if "path" has subdirectories.
        # - `cp "path/*" "dest/"` will fail (it expects wildcard expansion
        #   to come from shell).
        if arg_3:
          arg_9 = 'gsutil -u %s -mq cp "%s" "%s"' % (
              arg_3, arg_6, arg_8)
        else:
          arg_9 = 'gsutil -mq cp "%s" "%s"' % (arg_6,
                                                 arg_8)
        arg_4.append(arg_9)

    return '\n'.join(arg_4)


def Func():
  """Get the dsub version out of the _dsub_version.py source file.

  Setup.py should not import dsub version from dsub directly since ambiguity in
  import order could lead to an old version of dsub setting the version number.
  Parsing the file directly is simpler than using import tools (whose interface
  varies between python 2.7, 3.4, and 3.5).

  Returns:
    string of dsub version.

  Raises:
    ValueError: if the version is not found.
  """
  arg_0 = os.path.join(os.path.dirname(__file__), 'dsub/_dsub_version.py')
  with open(arg_0, 'r') as versionfile:
    for arg_1 in versionfile:
      if arg_1.startswith('DSUB_VERSION ='):
        # Get the version then strip whitespace and quote characters.
        arg_2 = arg_1.partition('=')[2]
        return arg_2.strip().strip('\'"')
  raise ValueError('Could not find version.')


def Func(arg_0, arg_1, arg_2, arg_3, arg_4, arg_5):
    """Return a dict with variables for the 'prepare' action."""

    # Add the _SCRIPT_REPR with the repr(script) contents
    # Add the _META_YAML_REPR with the repr(meta) contents

    # Add variables for directories that need to be created, for example:
    # DIR_COUNT: 2
    # DIR_0: /mnt/data/input/gs/bucket/path1/
    # DIR_1: /mnt/data/output/gs/bucket/path2

    # List the directories in sorted order so that they are created in that
    # order. This is primarily to ensure that permissions are set as we create
    # each directory.
    # For example:
    #   mkdir -m 777 -p /root/first/second
    #   mkdir -m 777 -p /root/first
    # *may* not actually set 777 on /root/first

    arg_6 = sorted([
        var.docker_path if var.recursive else os.path.dirname(var.docker_path)
        for var in arg_3 | arg_4 | arg_5
        if var.value
    ])

    arg_7 = {
        _SCRIPT_VARNAME: repr(arg_1.value),
        _META_YAML_VARNAME: repr(arg_2.to_yaml()),
        'DIR_COUNT': str(len(arg_6))
    }

    for arg_8, arg_9 in enumerate(arg_6):
      arg_7['DIR_{}'.format(arg_8)] = os.path.join(providers_util.DATA_MOUNT_POINT,
                                               arg_9)

    return arg_7


def Func(arg_0, arg_1, arg_2):
    """Return a dict with variables for the 'localization' action."""

    # Add variables for paths that need to be localized, for example:
    # INPUT_COUNT: 1
    # INPUT_0: MY_INPUT_FILE
    # INPUT_RECURSIVE_0: 0
    # INPUT_SRC_0: gs://mybucket/mypath/myfile
    # INPUT_DST_0: /mnt/data/inputs/mybucket/mypath/myfile

    arg_3 = [arg_6 for arg_6 in arg_1 if arg_6.value]
    arg_4 = {'INPUT_COUNT': str(len(arg_3))}

    for arg_5, arg_6 in enumerate(arg_3):
      arg_4['INPUT_{}'.format(arg_5)] = arg_6.name
      arg_4['INPUT_RECURSIVE_{}'.format(arg_5)] = str(int(arg_6.recursive))
      arg_4['INPUT_SRC_{}'.format(arg_5)] = arg_6.value

      # For wildcard paths, the destination must be a directory
      arg_8 = os.path.join(providers_util.DATA_MOUNT_POINT, arg_6.docker_path)
      arg_9, arg_10 = os.path.split(arg_8)
      if '*' in arg_10:
        arg_8 = '{}/'.format(arg_9)
      arg_4['INPUT_DST_{}'.format(arg_5)] = arg_8

    arg_4['USER_PROJECT'] = arg_2

    return arg_4


def Func(arg_0, arg_1, arg_2):
    """Return a dict with variables for the 'delocalization' action."""

    # Add variables for paths that need to be delocalized, for example:
    # OUTPUT_COUNT: 1
    # OUTPUT_0: MY_OUTPUT_FILE
    # OUTPUT_RECURSIVE_0: 0
    # OUTPUT_SRC_0: gs://mybucket/mypath/myfile
    # OUTPUT_DST_0: /mnt/data/outputs/mybucket/mypath/myfile

    arg_3 = [arg_6 for arg_6 in arg_1 if arg_6.value]
    arg_4 = {'OUTPUT_COUNT': str(len(arg_3))}

    for arg_5, arg_6 in enumerate(arg_3):
      arg_4['OUTPUT_{}'.format(arg_5)] = arg_6.name
      arg_4['OUTPUT_RECURSIVE_{}'.format(arg_5)] = str(int(arg_6.recursive))
      arg_4['OUTPUT_SRC_{}'.format(arg_5)] = os.path.join(
          providers_util.DATA_MOUNT_POINT, arg_6.docker_path)

      # For wildcard paths, the destination must be a directory
      if '*' in arg_6.uri.basename:
        arg_8 = arg_6.uri.path
      else:
        arg_8 = arg_6.uri
      arg_4['OUTPUT_DST_{}'.format(arg_5)] = arg_8

    arg_4['USER_PROJECT'] = arg_2

    return arg_4


def Func(arg_0, arg_1, arg_2, arg_3, arg_4):
    """Returns a dictionary of for the user container environment."""
    arg_1 = {env.name: env.value for env in arg_1}
    arg_1.update(providers_util.get_file_environment_variables(arg_2))
    arg_1.update(providers_util.get_file_environment_variables(arg_3))
    arg_1.update(providers_util.get_file_environment_variables(arg_4))
    return arg_1


def Func(arg_0):
    """Returns the status of this operation.

    Raises:
      ValueError: if the operation status cannot be determined.

    Returns:
      A printable status string (RUNNING, SUCCESS, CANCELED or FAILURE).
    """
    if not google_v2_operations.is_done(arg_0._op):
      return 'RUNNING'
    if google_v2_operations.is_success(arg_0._op):
      return 'SUCCESS'
    if google_v2_operations.is_canceled(arg_0._op):
      return 'CANCELED'
    if google_v2_operations.is_failed(arg_0._op):
      return 'FAILURE'

    raise ValueError('Status for operation {} could not be determined'.format(
        arg_0._op['name']))


def Func(arg_0):
    """Returns the most relevant status string and failed action.

    This string is meant for display only.

    Returns:
      A printable status string and name of failed action (if any).
    """
    arg_1 = None
    arg_2 = None
    if not google_v2_operations.is_done(arg_0._op):
      arg_3 = google_v2_operations.get_last_event(arg_0._op)
      if arg_3:
        arg_1 = arg_3['description']
        arg_4 = arg_3.get('details', {}).get('actionId')
        if arg_4:
          arg_2 = google_v2_operations.get_action_by_id(arg_0._op, arg_4)
      else:
        arg_1 = 'Pending'
    else:
      arg_5 = google_v2_operations.get_failed_events(arg_0._op)
      if arg_5:
        arg_6 = arg_5[-1]
        arg_1 = arg_6.get('details', {}).get('stderr')
        arg_4 = arg_6.get('details', {}).get('actionId')
        if arg_4:
          arg_2 = google_v2_operations.get_action_by_id(arg_0._op, arg_4)
      if not arg_1:
        arg_7 = google_v2_operations.get_error(arg_0._op)
        if arg_7:
          arg_1 = arg_7['message']
        else:
          arg_1 = 'Success'

    return arg_1, arg_2


def Func(arg_0):
    """Rounds ram up to the nearest multiple of _MEMORY_MULTIPLE."""
    return int(GoogleV2CustomMachine._MEMORY_MULTIPLE * math.ceil(
        arg_0 / GoogleV2CustomMachine._MEMORY_MULTIPLE))


def Func(arg_0, arg_1, arg_2):
    """Returns a custom machine type string."""
    arg_1 = arg_1 or job_model.DEFAULT_MIN_CORES
    arg_2 = arg_2 or job_model.DEFAULT_MIN_RAM

    # First, min_ram is given in GB. Convert to MB.
    arg_2 *= GoogleV2CustomMachine._MB_PER_GB

    # Only machine types with 1 vCPU or an even number of vCPUs can be created.
    arg_3 = arg_0._validate_cores(arg_1)
    # The total memory of the instance must be a multiple of 256 MB.
    arg_4 = arg_0._validate_ram(arg_2)

    # Memory must be between 0.9 GB per vCPU, up to 6.5 GB per vCPU.
    arg_5 = arg_4 / arg_3

    if arg_5 < GoogleV2CustomMachine._MIN_MEMORY_PER_CPU:
      # If we're under the ratio, top up the memory.
      arg_6 = GoogleV2CustomMachine._MIN_MEMORY_PER_CPU * arg_3
      arg_4 = arg_0._validate_ram(arg_6)

    elif arg_5 > GoogleV2CustomMachine._MAX_MEMORY_PER_CPU:
      # If we're over the ratio, top up the CPU.
      arg_7 = math.ceil(
          arg_4 / GoogleV2CustomMachine._MAX_MEMORY_PER_CPU)
      arg_3 = arg_0._validate_cores(arg_7)

    else:
      # Ratio is within the restrictions - no adjustments needed.
      pass

    return 'custom-{}-{}'.format(int(arg_3), int(arg_4))


def Func(arg_0=None,
                  arg_1=None,
                  arg_2=None,
                  arg_3=None,
                  arg_4=None,
                  arg_5=None,
                  arg_6=None,
                  arg_7=None,
                  arg_8=None,
                  arg_9=None):
  """Build a VirtualMachine object for a Pipeline request.

  Args:
    network (dict): Network details for the pipeline to run in.
    machine_type (str): GCE Machine Type string for the pipeline.
    preemptible (bool): Use a preemptible VM for the job.
    service_account (dict): Service account configuration for the VM.
    boot_disk_size_gb (int): Boot disk size in GB.
    disks (list[dict]): List of disks to mount.
    accelerators (list[dict]): List of accelerators to attach to the VM.
    labels (dict[string, string]): Labels for the VM.
    cpu_platform (str): The CPU platform to request.
    nvidia_driver_version (str): The NVIDIA driver version to use when attaching
      an NVIDIA GPU accelerator.

  Returns:
    An object representing a VirtualMachine.
  """
  return {
      'network': arg_0,
      'machineType': arg_1,
      'preemptible': arg_2,
      'serviceAccount': arg_3,
      'bootDiskSizeGb': arg_4,
      'disks': arg_5,
      'accelerators': arg_6,
      'labels': arg_7,
      'cpuPlatform': arg_8,
      'nvidiaDriverVersion': arg_9,
  }


def Func(arg_0=None,
                 arg_1=None,
                 arg_2=None,
                 arg_3=None,
                 arg_4=None,
                 arg_5=None,
                 arg_6=None,
                 arg_7=None,
                 arg_8=None,
                 arg_9=None):
  """Build an Action object for a Pipeline request.

  Args:
    name (str): An optional name for the container.
    image_uri (str): The URI to pull the container image from.
    commands (List[str]): commands and arguments to run inside the container.
    entrypoint (str): overrides the ENTRYPOINT specified in the container.
    environment (dict[str,str]): The environment to pass into the container.
    pid_namespace (str): The PID namespace to run the action inside.
    flags (str): Flags that control the execution of this action.
    port_mappings (dict[int, int]): A map of container to host port mappings for
      this container.
    mounts (List): A list of mounts to make available to the action.
    labels (dict[str]): Labels to associate with the action.

  Returns:
    An object representing an Action resource.
  """

  return {
      'name': arg_0,
      'imageUri': arg_1,
      'commands': arg_2,
      'entrypoint': arg_3,
      'environment': arg_4,
      'pidNamespace': arg_5,
      'flags': arg_6,
      'portMappings': arg_7,
      'mounts': arg_8,
      'labels': arg_9,
  }


def Func(arg_0, arg_1):
  """Returns a provider for job submission requests."""

  arg_2 = getattr(arg_0, 'provider', 'google')

  if arg_2 == 'google':
    return google.GoogleJobProvider(
        getattr(arg_0, 'verbose', False),
        getattr(arg_0, 'dry_run', False), arg_0.project)
  elif arg_2 == 'google-v2':
    return google_v2.GoogleV2JobProvider(
        getattr(arg_0, 'verbose', False), getattr(arg_0, 'dry_run', False),
        arg_0.project)
  elif arg_2 == 'local':
    return local.LocalJobProvider(arg_1)
  elif arg_2 == 'test-fails':
    return test_fails.FailsJobProvider()
  else:
    raise ValueError('Unknown provider: ' + arg_2)


def Func(arg_0, arg_1, arg_2):
  """Add provider required arguments epilog message, parse, and validate."""

  # Add the provider required arguments epilog message
  arg_3 = 'Provider-required arguments:\n'
  for arg_4 in arg_1:
    arg_3 += '  %s: %s\n' % (arg_4, arg_1[arg_4])
  arg_0.epilog = arg_3

  # Parse arguments
  arg_5 = arg_0.Func(arg_2)

  # For the selected provider, check the required arguments
  for arg_6 in arg_1[arg_5.provider]:
    if not arg_5.__getattribute__(arg_6):
      arg_0.error('argument --%s is required' % arg_6)

  return arg_5


def Func(arg_0, arg_1):
  """A string with the arguments to point dstat to the same provider+project."""
  arg_2 = get_provider_name(arg_0)

  arg_3 = []
  if arg_2 == 'google':
    arg_3.append('--project %s' % arg_1)
  elif arg_2 == 'google-v2':
    arg_3.append('--project %s' % arg_1)
  elif arg_2 == 'local':
    pass
  elif arg_2 == 'test-fails':
    pass
  else:
    # New providers should add their dstat required arguments here.
    assert False, 'Provider %s needs Func support' % arg_0

  arg_3.insert(0, '--provider %s' % arg_2)
  return ' '.join(arg_3)


def Func(arg_0, arg_1, arg_2):
  """Returns a URI with placeholders replaced by metadata values."""

  arg_3 = {
      'job-id': None,
      'task-id': 'task',
      'job-name': None,
      'user-id': None,
      'task-attempt': None
  }
  for arg_4 in arg_3:
    arg_3[arg_4] = arg_2.get(arg_4) or arg_1.get(arg_4) or arg_3[arg_4]

  return arg_0.format(**arg_3)


def Func(arg_0, arg_1, arg_2):
  """Inserts task metadata into the logging URI.

  The core behavior is inspired by the Google Pipelines API:
    (1) If a the uri ends in ".log", then that is the logging path.
    (2) Otherwise, the uri is treated as "directory" for logs and a filename
        needs to be automatically generated.

  For (1), if the job is a --tasks job, then the {task-id} is inserted
  before ".log".

  For (2), the file name generated is {job-id}, or for --tasks jobs, it is
  {job-id}.{task-id}.

  In both cases .{task-attempt} is inserted before .log for --retries jobs.

  In addition, full task metadata substitution is supported. The URI
  may include substitution strings such as
  "{job-id}", "{task-id}", "{job-name}", "{user-id}", and "{task-attempt}".

  Args:
    uri: User-specified logging URI which may contain substitution fields.
    job_metadata: job-global metadata.
    task_metadata: tasks-specific metadata.

  Returns:
    The logging_uri formatted as described above.
  """

  # If the user specifies any formatting (with curly braces), then use that
  # as the format string unchanged.
  arg_3 = str(arg_0)
  if '{' not in arg_3:
    if arg_0.endswith('.log'):
      # URI includes a filename. Trim the extension and just use the prefix.
      arg_3 = os.path.splitext(arg_0)[0]
    else:
      # URI is a path to a directory. The job-id becomes the filename prefix.
      arg_3 = os.path.join(arg_0, '{job-id}')

    # If this is a task job, add the task-id.
    if arg_2.get('task-id') is not None:
      arg_3 += '.{task-id}'

    # If this is a retryable task, add the task-attempt.
    if arg_2.get('task-attempt') is not None:
      arg_3 += '.{task-attempt}'

    arg_3 += '.log'

  return _format_task_uri(arg_3, arg_1, arg_2)


def Func(arg_0):
  """Validated google-v2 arguments."""
  if (arg_0.zones and arg_0.regions) or (not arg_0.zones and not arg_0.regions):
    raise ValueError('Exactly one of --regions and --zones must be specified')

  if arg_0.machine_type and (arg_0.min_cores or arg_0.min_ram):
    raise ValueError(
        '--machine-type not supported together with --min-cores or --min-ram.')


def Func(arg_0):
  """Extract job-global resources requirements from input args.

  Args:
    args: parsed command-line arguments

  Returns:
    Resources object containing the requested resources for the job
  """
  arg_1 = param_util.build_logging_param(
      arg_0.logging) if arg_0.logging else None
  arg_2 = param_util.timeout_in_seconds(arg_0.timeout)
  arg_3 = param_util.log_interval_in_seconds(arg_0.log_interval)

  return job_model.Resources(
      min_cores=arg_0.min_cores,
      min_ram=arg_0.min_ram,
      machine_type=arg_0.machine_type,
      disk_size=arg_0.disk_size,
      disk_type=arg_0.disk_type,
      boot_disk_size=arg_0.boot_disk_size,
      preemptible=arg_0.preemptible,
      image=arg_0.image,
      regions=arg_0.regions,
      zones=arg_0.zones,
      arg_1=arg_1,
      logging_path=None,
      service_account=arg_0.service_account,
      scopes=arg_0.scopes,
      keep_alive=arg_0.keep_alive,
      cpu_platform=arg_0.cpu_platform,
      network=arg_0.network,
      subnetwork=arg_0.subnetwork,
      use_private_address=arg_0.use_private_address,
      accelerator_type=arg_0.accelerator_type,
      accelerator_count=arg_0.accelerator_count,
      nvidia_driver_version=arg_0.nvidia_driver_version,
      arg_2=arg_2,
      arg_3=arg_3,
      ssh=arg_0.ssh)


def Func(arg_0, arg_1, arg_2, arg_3):
  """Print status info as we wait for those jobs.

  Blocks until either all of the listed jobs succeed,
  or one of them fails.

  Args:
    provider: job service provider
    job_ids: a set of job IDs (string) to wait for
    poll_interval: integer seconds to wait between iterations
    stop_on_failure: whether to stop waiting if one of the tasks fails.

  Returns:
    Empty list if there was no error,
    a list of error messages from the failed tasks otherwise.
  """

  # Each time through the loop, the job_set is re-set to the jobs remaining to
  # check. Jobs are removed from the list when they complete.
  #
  # We exit the loop when:
  # * No jobs remain are running, OR
  # * stop_on_failure is TRUE AND at least one job returned an error

  # remove NO_JOB
  arg_4 = {arg_12 for arg_12 in arg_1 if arg_12 != dsub_util.NO_JOB}
  arg_5 = []
  while arg_4 and (not arg_5 or not arg_3):
    print('Waiting for: %s.' % (', '.join(arg_4)))

    # Poll until any remaining jobs have completed
    arg_6 = _wait_for_any_job(arg_0, arg_4, arg_2)

    # Calculate which jobs just completed
    arg_7 = arg_4.difference(arg_6)

    # Get all tasks for the newly completed jobs
    arg_8 = arg_0.lookup_job_tasks({'*'}, arg_1=arg_7)

    # We don't want to overwhelm the user with output when there are many
    # tasks per job. So we get a single "dominant" task for each of the
    # completed jobs (one that is representative of the job's fate).
    arg_9 = _dominant_task_for_jobs(arg_8)
    if len(arg_9) != len(arg_7):
      # print info about the jobs we couldn't find
      # (should only occur for "--after" where the job ID is a typo).
      arg_10 = dsub_util.tasks_to_job_ids(arg_9)
      arg_11 = arg_7.difference(arg_10)
      for arg_12 in arg_11:
        arg_13 = '%s: not found' % arg_12
        print_error('  %s' % arg_13)
        arg_5 += [arg_13]

    # Print the dominant task for the completed jobs
    for arg_14 in arg_9:
      arg_15 = arg_14.get_field('job-id')
      arg_16 = arg_14.get_field('task-status')
      print('  %s: %s' % (str(arg_15), str(arg_16)))
      if arg_16 in ['FAILURE', 'CANCELED']:
        arg_5 += [arg_0.get_tasks_completion_messages([arg_14])]

    arg_4 = arg_6

  return arg_5


def Func(arg_0, arg_1, arg_2, arg_3, arg_4):
  """Wait for job and retry any tasks that fail.

  Stops retrying an individual task when: it succeeds, is canceled, or has been
  retried "retries" times.

  This function exits when there are no tasks running and there are no tasks
  eligible to be retried.

  Args:
    provider: job service provider
    job_id: a single job ID (string) to wait for
    poll_interval: integer seconds to wait between iterations
    retries: number of retries
    job_descriptor: job descriptor used to originally submit job

  Returns:
    Empty list if there was no error,
    a list containing an error message from a failed task otherwise.
  """

  while True:
    arg_5 = arg_0.lookup_job_tasks({'*'}, job_ids=[arg_1])

    arg_6 = set()
    arg_7 = set()
    arg_8 = set()
    arg_9 = set()
    arg_10 = dict()

    # This is an arbitrary task that is either fully failed or canceled (with
    # preference for the former).
    arg_11 = None

    arg_12 = dict()
    for arg_13 in arg_5:
      arg_14 = job_model.numeric_task_id(arg_13.get_field('task-id'))
      arg_12[arg_14] = arg_13

      arg_15 = arg_13.get_field('task-status')
      if arg_15 == 'FAILURE':
        # Could compute this from task-attempt as well.
        arg_10[arg_14] = arg_10.get(arg_14, 0) + 1
        if arg_10[arg_14] > arg_3:
          arg_9.add(arg_14)
          arg_11 = arg_13
      elif arg_15 == 'CANCELED':
        arg_8.add(arg_14)
        if not arg_11:
          arg_11 = arg_13
      elif arg_15 == 'SUCCESS':
        arg_7.add(arg_14)
      elif arg_15 == 'RUNNING':
        arg_6.add(arg_14)

    arg_16 = (
        set(arg_10).difference(arg_9)
        .difference(arg_6).difference(arg_7)
        .difference(arg_8))

    # job completed.
    if not arg_16 and not arg_6:
      # If there are any fully failed tasks, return the completion message of an
      # arbitrary one.
      # If not, but there are canceled tasks, return the completion message of
      # an arbitrary one.
      if arg_11:
        return [arg_0.get_tasks_completion_messages([arg_11])]

      # Otherwise successful completion.
      return []

    for arg_14 in arg_16:
      arg_17 = '{}.{}'.format(arg_1, arg_14) if arg_14 else arg_1
      print('  {} (attempt {}) failed. Retrying.'.format(
          arg_17, arg_10[arg_14]))
      arg_18 = arg_12[arg_14].get_field('status-message')
      print('  Failure message: {}'.format(arg_18))

      _retry_task(arg_0, arg_4, arg_14,
                  arg_10[arg_14] + 1)

    SLEEP_FUNCTION(arg_2)


def Func(arg_0):
  """A list with, for each job, its dominant task.

  The dominant task is the one that exemplifies its job's
  status. It is either:
  - the first (FAILURE or CANCELED) task, or if none
  - the first RUNNING task, or if none
  - the first SUCCESS task.

  Args:
    tasks: a list of tasks to consider

  Returns:
    A list with, for each job, its dominant task.
  """

  arg_1 = _group_tasks_by_jobid(arg_0)

  arg_2 = []
  for arg_3 in arg_1.keys():
    arg_4 = sorted(arg_1[arg_3], key=_importance_of_task)
    arg_2.append(arg_4[0])
  return arg_2


def Func(arg_0, arg_1, arg_2):
  """Waits until any of the listed jobs is not running.

  In particular, if any of the jobs sees one of its tasks fail,
  we count the whole job as failing (but do not terminate the remaining
  tasks ourselves).

  Args:
    provider: job service provider
    job_ids: a list of job IDs (string) to wait for
    poll_interval: integer seconds to wait between iterations

  Returns:
    A set of the jobIDs with still at least one running task.
  """
  if not arg_1:
    return
  while True:
    arg_3 = arg_0.lookup_job_tasks({'*'}, arg_1=arg_1)
    arg_4 = set()
    arg_5 = set()
    for arg_6 in arg_3:
      arg_7 = arg_6.get_field('task-status')
      arg_8 = arg_6.get_field('job-id')
      if arg_7 in ['FAILURE', 'CANCELED']:
        arg_5.add(arg_8)
      if arg_7 == 'RUNNING':
        arg_4.add(arg_8)
    arg_9 = arg_4.difference(arg_5)
    if arg_5 or len(arg_9) != len(arg_1):
      return arg_9
    SLEEP_FUNCTION(arg_2)


def Func(arg_0, arg_1):
  """Validates that job and task argument names do not overlap."""

  if not arg_1:
    return

  arg_2 = arg_1[0].task_params

  # The use case for specifying a label or env/input/output parameter on
  # the command-line and also including it in the --tasks file is not obvious.
  # Should the command-line override the --tasks file? Why?
  # Until this use is articulated, generate an error on overlapping names.

  # Check labels
  arg_3 = {label.name for label in arg_0['labels']}
  arg_4 = {label.name for label in arg_2['labels']}

  arg_5 = arg_3 & arg_4
  if arg_5:
    raise ValueError(
        'Names for labels on the command-line and in the --tasks file must not '
        'be repeated: {}'.format(','.join(arg_5)))

  # Check envs, inputs, and outputs, all of which must not overlap each other
  arg_3 = {
      item.name
      for item in arg_0['envs'] | arg_0['inputs']
      | arg_0['outputs']
  }
  arg_4 = {
      item.name
      for item in arg_2['envs'] | arg_2['inputs']
      | arg_2['outputs']
  }

  arg_5 = arg_3 & arg_4
  if arg_5:
    raise ValueError(
        'Names for envs, inputs, and outputs on the command-line and in the '
        '--tasks file must not be repeated: {}'.format(','.join(arg_5)))


def Func(arg_0, arg_1):
  """Helper function to return an appropriate set of mount parameters."""
  return set([arg_2 for arg_2 in arg_0 if isinstance(arg_2, arg_1)])


def Func(arg_0, arg_1=arg_2):
  """Convenience function simplifies construction of the logging uri."""
  if not arg_0:
    return job_model.LoggingParam(None, None)
  arg_3 = not arg_0.endswith('.log')
  arg_4 = arg_1('')
  arg_5, arg_6, arg_7 = arg_4.parse_uri(arg_0, arg_3)
  if '*' in arg_6.basename:
    raise ValueError('Wildcards not allowed in logging URI: %s' % arg_6)
  return job_model.LoggingParam(arg_6, arg_7)


def Func(arg_0, arg_1, arg_2=1):
  """Split a string into a pair, which can have one empty value.

  Args:
    pair_string: The string to be split.
    separator: The separator to be used for splitting.
    nullable_idx: The location to be set to null if the separator is not in the
                  input string. Should be either 0 or 1.

  Returns:
    A list containing the pair.

  Raises:
    IndexError: If nullable_idx is not 0 or 1.
  """

  arg_3 = arg_0.split(arg_1, 1)
  if len(arg_3) == 1:
    if arg_2 == 0:
      return [None, arg_3[0]]
    elif arg_2 == 1:
      return [arg_3[0], None]
    else:
      raise IndexError('nullable_idx should be either 0 or 1.')
  else:
    return arg_3


def Func(arg_0, arg_1, arg_2,
                                   arg_3):
  """Parses task parameters from a TSV.

  Args:
    tasks: Dict containing the path to a TSV file and task numbers to run
    variables, input, and output parameters as column headings. Subsequent
    lines specify parameter values, one row per job.
    retries: Number of retries allowed.
    input_file_param_util: Utility for producing InputFileParam objects.
    output_file_param_util: Utility for producing OutputFileParam objects.

  Returns:
    task_descriptors: an array of records, each containing the task-id,
    task-attempt, 'envs', 'inputs', 'outputs', 'labels' that defines the set of
    parameters for each task of the job.

  Raises:
    ValueError: If no job records were provided
  """
  arg_4 = []

  arg_5 = arg_0['path']
  arg_6 = arg_0.get('min')
  arg_7 = arg_0.get('max')

  # Load the file and set up a Reader that tokenizes the fields
  arg_8 = dsub_util.load_file(arg_5)
  arg_9 = csv.reader(arg_8, delimiter='\t')

  # Read the first line and extract the parameters
  arg_10 = six.advance_iterator(arg_9)
  arg_11 = parse_tasks_file_header(arg_10, arg_2,
                                       arg_3)

  # Build a list of records from the parsed input file
  for arg_12 in arg_9:
    # Tasks are numbered starting at 1 and since the first line of the TSV
    # file is a header, the first task appears on line 2.
    arg_13 = arg_9.line_num - 1
    if arg_6 and arg_13 < arg_6:
      continue
    if arg_7 and arg_13 > arg_7:
      continue

    if len(arg_12) != len(arg_11):
      dsub_util.print_error('Unexpected number of fields %s vs %s: line %s' %
                            (len(arg_12), len(arg_11), arg_9.line_num))

    # Each row can contain "envs", "inputs", "outputs"
    arg_14 = set()
    arg_15 = set()
    arg_16 = set()
    arg_17 = set()

    for arg_18 in range(0, len(arg_11)):
      arg_19 = arg_11[arg_18]
      arg_20 = arg_19.name
      if isinstance(arg_19, job_model.EnvParam):
        arg_14.add(job_model.EnvParam(arg_20, arg_12[arg_18]))

      elif isinstance(arg_19, job_model.LabelParam):
        arg_17.add(job_model.LabelParam(arg_20, arg_12[arg_18]))

      elif isinstance(arg_19, job_model.InputFileParam):
        arg_15.add(
            arg_2.make_param(arg_20, arg_12[arg_18], arg_19.recursive))

      elif isinstance(arg_19, job_model.OutputFileParam):
        arg_16.add(
            arg_3.make_param(arg_20, arg_12[arg_18], arg_19.recursive))

    arg_4.append(
        job_model.TaskDescriptor({
            'task-id': arg_13,
            'task-attempt': 1 if arg_1 else None
        }, {
            'labels': arg_17,
            'envs': arg_14,
            'inputs': arg_15,
            'outputs': arg_16
        }, job_model.Resources()))

  # Ensure that there are jobs to execute (and not just a header)
  if not arg_4:
    raise ValueError('No tasks added from %s' % arg_5)

  return arg_4


def Func(arg_0, arg_1):
  """Parse flags of key=value pairs and return a list of argclass.

  For pair variables, we need to:
     * split the input into name=value pairs (value optional)
     * Create the EnvParam object

  Args:
    labels: list of 'key' or 'key=value' strings.
    argclass: Container class for args, must instantiate with argclass(k, v).

  Returns:
    list of argclass objects.
  """
  arg_2 = set()
  for arg_3 in arg_0:
    arg_4, arg_5 = split_pair(arg_3, '=', nullable_idx=1)
    arg_2.add(arg_1(arg_4, arg_5))
  return arg_2


def Func(arg_0, arg_1='smhdw'):
  """Convert the timeout duration to seconds.

  The value must be of the form "<integer><unit>" where supported
  units are s, m, h, d, w (seconds, minutes, hours, days, weeks).

  Args:
    interval: A "<integer><unit>" string.
    valid_units: A list of supported units.

  Returns:
    A string of the form "<integer>s" or None if timeout is empty.
  """
  if not arg_0:
    return None

  try:
    arg_2 = arg_0[-1]

    if arg_2 == 's' and 's' in arg_1:
      return str(float(arg_0[:-1])) + 's'
    elif arg_2 == 'm' and 'm' in arg_1:
      return str(float(arg_0[:-1]) * 60) + 's'
    elif arg_2 == 'h' and 'h' in arg_1:
      return str(float(arg_0[:-1]) * 60 * 60) + 's'
    elif arg_2 == 'd' and 'd' in arg_1:
      return str(float(arg_0[:-1]) * 60 * 60 * 24) + 's'
    elif arg_2 == 'w' and 'w' in arg_1:
      return str(float(arg_0[:-1]) * 60 * 60 * 24 * 7) + 's'
    else:
      raise ValueError(
          'Unsupported units in interval string %s: %s' % (arg_0, arg_2))

  except (ValueError, OverflowError) as e:
    raise ValueError('Unable to parse interval string %s: %s' % (arg_0, e))


def Func(arg_0, arg_1):
    """Produce a default variable name if none is specified."""
    if not arg_1:
      arg_1 = '%s%s' % (arg_0._auto_prefix, arg_0._auto_index)
      arg_0._auto_index += 1
    return arg_1


def Func(arg_0):
    """Find the file provider for a URI."""
    arg_1 = {'gs': job_model.P_GCS, 'file': job_model.P_LOCAL}
    # URI scheme detector uses a range up to 30 since none of the IANA
    # registered schemes are longer than this.
    arg_2 = re.match(r'^([A-Za-z][A-Za-z0-9+.-]{0,29})://', arg_0)
    if arg_2:
      arg_3 = arg_2.group(1).lower()
    else:
      # If no provider is specified in the URI, assume that the local
      # filesystem is being used. Availability and validity of the local
      # file/directory will be checked later.
      arg_3 = 'file'
    if arg_3 in arg_1:
      return arg_1[arg_3]
    else:
      raise ValueError('File prefix not supported: %s://' % arg_3)


def Func(arg_0, arg_1):
    """Do basic validation of the uri, return the path and filename."""
    arg_2, arg_3 = os.path.split(arg_0)

    # dsub could support character ranges ([0-9]) with some more work, but for
    # now we assume that basic asterisk wildcards are sufficient. Reject any URI
    # that includes square brackets or question marks, since we know that
    # if they actually worked, it would be accidental.
    if '[' in arg_0 or ']' in arg_0:
      raise ValueError(
          'Square bracket (character ranges) are not supported: %s' % arg_0)
    if '?' in arg_0:
      raise ValueError('Question mark wildcards are not supported: %s' % arg_0)

    # Only support file URIs and *filename* wildcards
    # Wildcards at the directory level or "**" syntax would require better
    # support from the Pipelines API *or* doing expansion here and
    # (potentially) producing a series of FileParams, instead of one.
    if '*' in arg_2:
      raise ValueError(
          'Path wildcard (*) are only supported for files: %s' % arg_0)
    if '**' in arg_3:
      raise ValueError('Recursive wildcards ("**") not supported: %s' % arg_0)
    if arg_3 in ('..', '.'):
      raise ValueError('Path characters ".." and "." not supported '
                       'for file names: %s' % arg_0)

    # Do not allow non-recursive IO to reference directories.
    if not arg_1 and not arg_3:
      raise ValueError('Input or output values that are not recursive must '
                       'reference a filename or wildcard: %s' % arg_0)


def Func(arg_0, arg_1):
    """Return a valid docker_path from a Google Persistent Disk url."""
    # The string replace is so we don't have colons and double slashes in the
    # mount path. The idea is the resulting mount path would look like:
    # /mnt/data/mount/http/www.googleapis.com/compute/v1/projects/...
    arg_2 = os.path.join(arg_0._relative_path,
                              arg_1.replace('https://', 'https/', 1))
    return arg_2


def Func(arg_0, arg_1, arg_2, arg_3):
    """Return a MountParam given a GCS bucket, disk image or local path."""
    if arg_2.startswith('https://www.googleapis.com/compute'):
      # Full Image URI should look something like:
      # https://www.googleapis.com/compute/v1/projects/<project>/global/images/
      # But don't validate further, should the form of a valid image URI
      # change (v1->v2, for example)
      arg_4 = arg_0._parse_image_uri(arg_2)
      return job_model.PersistentDiskMountParam(
          arg_1, arg_2, arg_4, arg_3, disk_type=None)
    elif arg_2.startswith('file://'):
      arg_5, arg_4 = arg_0._parse_local_mount_uri(arg_2)
      return job_model.LocalMountParam(arg_1, arg_2, arg_4, arg_5)
    elif arg_2.startswith('gs://'):
      arg_4 = arg_0._parse_gcs_uri(arg_2)
      return job_model.GCSMountParam(arg_1, arg_2, arg_4)
    else:
      raise ValueError(
          'Mount parameter {} must begin with valid prefix.'.format(arg_2))


def Func(arg_0):
  """Turn the specified name and value into a valid Google label."""

  # We want the results to be user-friendly, not just functional.
  # So we can't base-64 encode it.
  #   * If upper-case: lower-case it
  #   * If the char is not a standard letter or digit. make it a dash

  # March 2019 note: underscores are now allowed in labels.
  # However, removing the conversion of underscores to dashes here would
  # create inconsistencies between old jobs and new jobs.
  # With existing code, $USER "jane_doe" has a user-id label of "jane-doe".
  # If we remove the conversion, the user-id label for new jobs is "jane_doe".
  # This makes looking up old jobs more complicated.

  arg_1 = string.ascii_lowercase + string.digits + '-'

  def label_char_transform(arg_2):
    if arg_2 in arg_1:
      return arg_2
    if arg_2 in string.ascii_uppercase:
      return arg_2.lower()
    return '-'

  return ''.join(label_char_transform(arg_3) for arg_3 in arg_0)


def Func(arg_0):
  """For each task, ensure that each task param entry is not None."""
  for arg_1 in arg_0:
    for arg_2 in [
        'labels', 'envs', 'inputs', 'outputs', 'input-recursives',
        'output-recursives'
    ]:
      if not arg_1.task_params.get(arg_2):
        arg_1.task_params[arg_2] = set()


def Func(arg_0, arg_1):
  """Return a new dict with any empty items removed.

  Note that this is not a deep check. If d contains a dictionary which
  itself contains empty items, those are never checked.

  This method exists to make to_serializable() functions cleaner.
  We could revisit this some day, but for now, the serialized objects are
  stripped of empty values to keep the output YAML more compact.

  Args:
    d: a dictionary
    required: list of required keys (for example, TaskDescriptors always emit
      the "task-id", even if None)

  Returns:
    A dictionary with empty items removed.
  """

  arg_2 = {}
  for arg_3, arg_4 in arg_0.items():
    if arg_3 in arg_1:
      arg_2[arg_3] = arg_4
    elif isinstance(arg_4, int) or arg_4:
      # "if v" would suppress emitting int(0)
      arg_2[arg_3] = arg_4

  return arg_2


def Func(arg_0):
  """Converts a task-id to the numeric task-id.

  Args:
    task_id: task-id in either task-n or n format

  Returns:
    n
  """

  # This function exists to support the legacy "task-id" format in the "google"
  # provider. Google labels originally could not be numeric. When the google
  # provider is completely replaced by the google-v2 provider, this function can
  # go away.

  if arg_0 is not None:
    if arg_0.startswith('task-'):
      return int(arg_0[len('task-'):])
    else:
      return int(arg_0)


def Func(arg_0, arg_1, arg_2):
    """Raise ValueError if the label is invalid."""
    # Rules for labels are described in:
    #  https://cloud.google.com/compute/docs/labeling-resources#restrictions

    # * Keys and values cannot be longer than 63 characters each.
    # * Keys and values can only contain lowercase letters, numeric characters,
    #   underscores, and dashes.
    # * International characters are allowed.
    # * Label keys must start with a lowercase letter and international
    #   characters are allowed.
    # * Label keys cannot be empty.
    arg_0._check_label_name(arg_1)
    arg_0._check_label_value(arg_2)

    # Ensure that reserved labels are not being used.
    if not arg_0._allow_reserved_keys and arg_1 in RESERVED_LABELS:
      raise ValueError('Label flag (%s=...) must not use reserved keys: %r' %
                       (arg_1, list(RESERVED_LABELS)))


def Func(arg_0, arg_1):
    """Populate a JobDescriptor from the local provider's original meta.yaml.

    The local job provider had the first incarnation of a YAML file for each
    task. That idea was extended here in the JobDescriptor and the local
    provider adopted the JobDescriptor.to_yaml() call to write its meta.yaml.

    The JobDescriptor.from_yaml() detects if it receives a local provider's
    "v0" meta.yaml and calls this function.

    Args:
      job: an object produced from decoding meta.yaml.

    Returns:
      A JobDescriptor populated as best we can from the old meta.yaml.
    """

    # The v0 meta.yaml only contained:
    #   create-time, job-id, job-name, logging, task-id
    #   labels, envs, inputs, outputs
    # It did NOT contain user-id.
    # dsub-version might be there as a label.

    arg_2 = {}
    for arg_3 in ['job-id', 'job-name', 'create-time']:
      arg_2[arg_3] = arg_1.get(arg_3)

    # Make sure that create-time string is turned into a datetime
    arg_2['create-time'] = dsub_util.replace_timezone(
        datetime.datetime.strptime(arg_1['create-time'], '%Y-%m-%d %H:%M:%S.%f'),
        tzlocal())

    # The v0 meta.yaml contained a "logging" field which was the task-specific
    # logging path. It did not include the actual "--logging" value the user
    # specified.
    arg_4 = Resources()

    # The v0 meta.yaml represented a single task.
    # It did not distinguish whether params were job params or task params.
    # We will treat them as either all job params or all task params, based on
    # whether the task-id is empty or an integer value.
    #
    # We also cannot distinguish whether inputs/outputs were recursive or not.
    # Just treat them all as non-recursive.
    arg_5 = {}

    # The dsub-version may be in the meta.yaml as a label. If so remove it
    # and set it as a top-level job metadata value.
    arg_6 = arg_1.get('labels', {})
    if 'dsub-version' in arg_6:
      arg_2['dsub-version'] = arg_6['dsub-version']
      del arg_6['dsub-version']
    arg_5['labels'] = arg_0._label_params_from_dict(arg_6)

    arg_5['envs'] = arg_0._env_params_from_dict(arg_1.get('envs', {}))
    arg_5['inputs'] = arg_0._input_file_params_from_dict(
        arg_1.get('inputs', {}), False)
    arg_5['outputs'] = arg_0._output_file_params_from_dict(
        arg_1.get('outputs', {}), False)

    if arg_1.get('task-id') is None:
      arg_7 = arg_5
      arg_8 = {'task-id': None}
      arg_9 = {}
    else:
      arg_7 = {}
      arg_8 = {'task-id': str(arg_1.get('task-id'))}
      arg_9 = arg_5

    arg_10 = Resources(logging_path=arg_1.get('logging'))

    arg_11 = [
        TaskDescriptor.get_complete_descriptor(arg_8, arg_9,
                                               arg_10)
    ]

    return JobDescriptor.get_complete_descriptor(
        arg_2, arg_7, arg_4, arg_11)


def Func(arg_0, arg_1):
    """Populate and return a JobDescriptor from a YAML string."""
    try:
      arg_2 = yaml.full_load(arg_1)
    except AttributeError:
      # For installations that cannot update their PyYAML version
      arg_2 = yaml.load(arg_1)

    # If the YAML does not contain a top-level dsub version, then assume that
    # the string is coming from the local provider, reading an old version of
    # its meta.yaml.
    arg_3 = arg_2.get('dsub-version')
    if not arg_3:
      return arg_0._Func_v0(arg_2)

    arg_4 = {}
    for arg_5 in [
        'job-id', 'job-name', 'task-ids', 'user-id', 'dsub-version',
        'user-project', 'script-name'
    ]:
      if arg_2.get(arg_5) is not None:
        arg_4[arg_5] = arg_2.get(arg_5)

    # Make sure that create-time string is turned into a datetime
    arg_4['create-time'] = dsub_util.replace_timezone(
        arg_2.get('create-time'), pytz.utc)

    arg_6 = Resources(logging=arg_2.get('logging'))

    arg_7 = {}
    arg_7['labels'] = arg_0._label_params_from_dict(arg_2.get('labels', {}))
    arg_7['envs'] = arg_0._env_params_from_dict(arg_2.get('envs', {}))
    arg_7['inputs'] = arg_0._input_file_params_from_dict(
        arg_2.get('inputs', {}), False)
    arg_7['input-recursives'] = arg_0._input_file_params_from_dict(
        arg_2.get('input-recursives', {}), True)
    arg_7['outputs'] = arg_0._output_file_params_from_dict(
        arg_2.get('outputs', {}), False)
    arg_7['output-recursives'] = arg_0._output_file_params_from_dict(
        arg_2.get('output-recursives', {}), True)
    arg_7['mounts'] = arg_0._mount_params_from_dict(arg_2.get('mounts', {}))

    arg_8 = []
    for arg_9 in arg_2.get('tasks', []):
      arg_10 = {'task-id': arg_9.get('task-id')}

      # Old instances of the meta.yaml do not have a task create time.
      arg_11 = arg_9.get('create-time')
      if arg_11:
        arg_10['create-time'] = dsub_util.replace_timezone(
            arg_11, pytz.utc)

      if arg_9.get('task-attempt') is not None:
        arg_10['task-attempt'] = arg_9.get('task-attempt')

      arg_12 = {}
      arg_12['labels'] = arg_0._label_params_from_dict(
          arg_9.get('labels', {}))
      arg_12['envs'] = arg_0._env_params_from_dict(arg_9.get('envs', {}))
      arg_12['inputs'] = arg_0._input_file_params_from_dict(
          arg_9.get('inputs', {}), False)
      arg_12['input-recursives'] = arg_0._input_file_params_from_dict(
          arg_9.get('input-recursives', {}), True)
      arg_12['outputs'] = arg_0._output_file_params_from_dict(
          arg_9.get('outputs', {}), False)
      arg_12['output-recursives'] = arg_0._output_file_params_from_dict(
          arg_9.get('output-recursives', {}), True)

      arg_13 = Resources(logging_path=arg_9.get('logging-path'))

      arg_8.append(
          TaskDescriptor(arg_10, arg_12, arg_13))

    return JobDescriptor(arg_4, arg_7, arg_6,
                         arg_8)


def Func(arg_0, arg_1):
    """Returns the task_descriptor corresponding to task_id."""

    # It is not guaranteed that the index will be task_id - 1 when --tasks is
    # used with a min/max range.
    for arg_2 in arg_0.task_descriptors:
      if arg_2.task_metadata.get('task-id') == arg_1:
        return arg_2
    return None


def Func(arg_0):
  """Return a dictionary of environment variables for the user container."""
  arg_1 = {}
  for arg_2 in arg_0:
    # We have no cases where the environment variable provided to user
    # scripts have a trailing slash, so be sure to always strip it.
    # The case that this is specifically handling is --input-recursive and
    # --output-recursive variables, which are directory values.
    arg_1[arg_2.name] = os.path.join(
        DATA_MOUNT_POINT, arg_2.docker_path.rstrip('/')) if arg_2.value else ''
  return arg_1


def Func(arg_0, arg_1, arg_2):
  """Returns a dict combining the field for job and task params."""
  return arg_0.get(arg_2, set()) | arg_1.get(arg_2, set())


def Func(arg_0,
               arg_1=None,
               arg_2=None,
               arg_3=None,
               arg_4=None,
               arg_5=None,
               arg_6=None):
  """Kill jobs or job tasks.

  This function separates ddel logic from flag parsing and user output. Users
  of ddel who intend to access the data programmatically should use this.

  Args:
    provider: an instantiated dsub provider.
    user_ids: a set of user ids who "own" the job(s) to delete.
    job_ids: a set of job ids to delete.
    task_ids: a set of task ids to delete.
    labels: a set of LabelParam, each must match the job(s) to be cancelled.
    create_time_min: a timezone-aware datetime value for the earliest create
                     time of a task, inclusive.
    create_time_max: a timezone-aware datetime value for the most recent create
                     time of a task, inclusive.

  Returns:
    list of job ids which were deleted.
  """
  # Delete the requested jobs
  arg_7, arg_8 = arg_0.delete_jobs(
      arg_1, arg_2, arg_3, arg_4, arg_5, arg_6)

  # Emit any errors canceling jobs
  for arg_9 in arg_8:
    print(arg_9)

  return arg_7


def Func(arg_0, arg_1):
  """Return the value for the specified action."""
  arg_2 = get_actions(arg_0)
  for arg_3 in arg_2:
    if arg_3.get('name') == arg_1:
      return arg_3


def Func(arg_0, arg_1):
  """Return the environment for the operation."""
  arg_2 = _get_action_by_name(arg_0, arg_1)
  if arg_2:
    return arg_2.get('environment')


def Func(arg_0, arg_1):
  """Return the image for the operation."""
  arg_2 = _get_action_by_name(arg_0, arg_1)
  if arg_2:
    return arg_2.get('imageUri')


def Func(arg_0, arg_1):
  """Return all events of a particular type."""
  arg_2 = get_events(arg_0)
  if not arg_2:
    return None

  return [arg_3 for arg_3 in arg_2 if arg_3.get('details', {}).get('@type') == arg_1]


def Func(arg_0,
                     arg_1,
                     arg_2=None,
                     arg_3=None,
                     arg_4=None,
                     arg_5=None,
                     arg_6=None,
                     arg_7=None,
                     arg_8=None,
                     arg_9=None,
                     arg_10=0,
                     arg_11=0,
                     arg_12=False):
  """Generate formatted jobs individually, in order of create-time.

  Args:
    provider: an instantiated dsub provider.
    statuses: a set of status strings that eligible jobs may match.
    user_ids: a set of user strings that eligible jobs may match.
    job_ids: a set of job-id strings eligible jobs may match.
    job_names: a set of job-name strings eligible jobs may match.
    task_ids: a set of task-id strings eligible tasks may match.
    task_attempts: a set of task-attempt strings eligible tasks may match.
    labels: set of LabelParam that all tasks must match.
    create_time_min: a timezone-aware datetime value for the earliest create
                     time of a task, inclusive.
    create_time_max: a timezone-aware datetime value for the most recent create
                     time of a task, inclusive.
    max_tasks: (int) maximum number of tasks to return per dstat job lookup.
    page_size: the page size to use for each query to the backend. May be
               ignored by some provider implementations.
    summary_output: (bool) summarize the job list.

  Yields:
    Individual task dictionaries with associated metadata
  """
  arg_13 = arg_0.Func(
      arg_1,
      arg_2=arg_2,
      arg_3=arg_3,
      arg_4=arg_4,
      arg_5=arg_5,
      arg_6=arg_6,
      arg_7=arg_7,
      arg_8=arg_8,
      arg_9=arg_9,
      arg_10=arg_10,
      arg_11=arg_11)

  # Yield formatted tasks.
  for arg_14 in arg_13:
    yield _prepare_row(arg_14, True, arg_12)


def Func(arg_0):
  """Returns a list of zones based on any wildcard input.

  This function is intended to provide an easy method for producing a list
  of desired zones for a pipeline to run in.

  The Pipelines API default zone list is "any zone". The problem with
  "any zone" is that it can lead to incurring Cloud Storage egress charges
  if the GCE zone selected is in a different region than the GCS bucket.
  See https://cloud.google.com/storage/pricing#network-egress.

  A user with a multi-region US bucket would want to pipelines to run in
  a "us-*" zone.
  A user with a regional bucket in US would want to restrict pipelines to
  run in a zone in that region.

  Rarely does the specific zone matter for a pipeline.

  This function allows for a simple short-hand such as:
     [ "us-*" ]
     [ "us-central1-*" ]
  These examples will expand out to the full list of US and us-central1 zones
  respectively.

  Args:
    input_list: list of zone names/patterns

  Returns:
    A list of zones, with any wildcard zone specifications expanded.
  """
  if not arg_0:
    return []

  arg_1 = []

  for arg_2 in arg_0:
    if arg_2.endswith('*'):
      arg_3 = arg_2[:-1]
      arg_1.extend([arg_4 for arg_4 in _ZONES if arg_4.startswith(arg_3)])
    else:
      arg_1.append(arg_2)

  return arg_1