def Func(arg_0, arg_1):
        '''Get generic entity by UUID.

        Args:
            entity_id (str): The UUID of the requested entity.

        Returns:
            A dictionary describing the entity::

                {
                     u'collab_id': 2271,
                     u'created_by': u'303447',
                     u'created_on': u'2017-03-10T12:50:06.077891Z',
                     u'description': u'',
                     u'entity_type': u'project',
                     u'modified_by': u'303447',
                     u'modified_on': u'2017-03-10T12:50:06.077946Z',
                     u'name': u'2271',
                     u'uuid': u'3abd8742-d069-44cf-a66b-2370df74a682'
                 }

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes
        '''
        if not is_valid_uuid(arg_1):
            raise StorageArgumentException(
                'Invalid UUID for entity_id: {0}'.format(arg_1))
        return arg_0._authenticated_request \
            .to_endpoint('entity/{}/'.format(arg_1)) \
            .return_body() \
            .get()


def Func(arg_0, arg_1, arg_2):
        '''Get metadata of an entity.

        Args:
            entity_type (str): Type of the entity. Admitted values: ['project',
                'folder', 'file'].
            entity_id (str): The UUID of the entity to be modified.

        Returns:
            A dictionary of the metadata::

                {
                    u'bar': u'200',
                    u'foo': u'100'
                }

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes
        '''
        if not is_valid_uuid(arg_2):
            raise StorageArgumentException(
                'Invalid UUID for entity_id: {0}'.format(arg_2))

        return arg_0._authenticated_request \
            .to_endpoint('{}/{}/metadata/'.format(arg_1, arg_2)) \
            .return_body() \
            .get()


def Func(arg_0, arg_1, arg_2, arg_3):
        '''Delete the selected metadata entries of an entity.

        Only deletes selected metadata keys, for a complete wipe, use set_metadata.

        Args:
            entity_type (str): Type of the entity. Admitted values: ['project',
                'folder', 'file'].
            entity_id (srt): The UUID of the entity to be modified.
            metadata_keys (lst): A list of metada keys to be deleted.

        Returns:
            A dictionary of the updated object metadata::

                {
                    u'bar': u'200',
                    u'foo': u'100'
                }

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes
        '''
        if not is_valid_uuid(arg_2):
            raise StorageArgumentException(
                'Invalid UUID for entity_id: {0}'.format(arg_2))
        if not isinstance(arg_3, list):
            raise StorageArgumentException('The metadata was not provided as a '
                                           'dictionary')

        return arg_0._authenticated_request \
            .to_endpoint('{}/{}/metadata/'.format(arg_1, arg_2)) \
            .with_json_body({'keys': arg_3}) \
            .return_body() \
            .delete()


def Func(arg_0, arg_1=None, arg_2=None, arg_3=None, arg_4=None,
                      arg_5=arg_6, arg_7=None, arg_8=None):
        '''List all the projects the user have access to.

            This function does not retrieve all results, pages have
            to be manually retrieved by the caller.

        Args:
            hpc (bool): If 'true', the result will contain only the HPC projects
                (Unicore projects).
            access (str): If provided, the result will contain only projects
                where the user has the provided acccess.
                Admitted values: ['read', 'write'].
            name (str): Filter on the project name.
            collab_id (int): Filter on the collab id.
            page_size (int): Number of elements per page.
            page (int): Number of the page
            ordering (str): Indicate on which fields to sort the result.
                Prepend '-' to invert order. Multiple values can be provided.
                Ordering is supported on: ['name', 'created_on', 'modified_on'].
                Example: ordering='name,created_on'

        Returns:
            A dictionary of the results::

            {
                u'count': 256,
                u'next': u'http://link.to.next/page',
                u'previous': None,
                u'results': [{u'collab_id': 2079,
                    u'created_by': u'258666',
                    u'created_on': u'2017-02-23T15:09:27.626973Z',
                    u'description': u'',
                    u'entity_type': u'project',
                    u'modified_by': u'258666',
                    u'modified_on': u'2017-02-23T15:09:27.627025Z',
                    u'name': u'2079',
                    u'uuid': u'64a6ad2e-acd1-44a3-a4cd-6bd96e3da2b0'}]
            }

        Raises:
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes
        '''
        return arg_0._authenticated_request \
            .to_endpoint('project/') \
            .with_params(arg_0._prep_params(locals())) \
            .return_body() \
            .get()


def Func(arg_0, arg_1):
        '''Get information on a given project

        Args:
            project_id (str): The UUID of the requested project.

        Returns:
            A dictionary describing the project::

            {
                u'collab_id': 2271,
                u'created_by': u'303447',
                u'created_on': u'2017-03-10T12:50:06.077891Z',
                u'description': u'',
                u'entity_type': u'project',
                u'modified_by': u'303447',
                u'modified_on': u'2017-03-10T12:50:06.077946Z',
                u'name': u'2271',
                u'uuid': u'3abd8742-d069-44cf-a66b-2370df74a682'
            }

        Raises:
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes
        '''
        if not is_valid_uuid(arg_1):
            raise StorageArgumentException(
                'Invalid UUID for project_id: {0}'.format(arg_1))

        return arg_0._authenticated_request \
            .to_endpoint('project/{}/'.format(arg_1)) \
            .return_body() \
            .get()


def Func(arg_0, arg_1):
        '''Create a new project.

        Args:
            collab_id (int): The id of the collab the project should be created in.

        Returns:
            A dictionary of details of the created project::

                {
                    u'collab_id': 12998,
                    u'created_by': u'303447',
                    u'created_on': u'2017-03-21T14:06:32.293902Z',
                    u'description': u'',
                    u'entity_type': u'project',
                    u'modified_by': u'303447',
                    u'modified_on': u'2017-03-21T14:06:32.293967Z',
                    u'name': u'12998',
                    u'uuid': u'2516442e-1e26-4de1-8ed8-94523224cc40'
                }

        Raises:
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes
        '''
        return arg_0._authenticated_request \
            .to_endpoint('project/') \
            .with_json_body(arg_0._prep_params(locals())) \
            .return_body() \
            .post()


def Func(arg_0, arg_1, arg_2):
        '''Create a new folder.

        Args:
            name (srt): The name of the folder.
            parent (str): The UUID of the parent entity. The parent must be a
                project or a folder.

        Returns:
            A dictionary of details of the created folder::

                {
                    u'created_by': u'303447',
                    u'created_on': u'2017-03-21T14:06:32.293902Z',
                    u'description': u'',
                    u'entity_type': u'folder',
                    u'modified_by': u'303447',
                    u'modified_on': u'2017-03-21T14:06:32.293967Z',
                    u'name': u'myfolder',
                    u'parent': u'3abd8742-d069-44cf-a66b-2370df74a682',
                    u'uuid': u'2516442e-1e26-4de1-8ed8-94523224cc40'
                }

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes
        '''
        if not is_valid_uuid(arg_2):
            raise StorageArgumentException(
                'Invalid UUID for parent: {0}'.format(arg_2))

        return arg_0._authenticated_request \
            .to_endpoint('folder/') \
            .with_json_body(arg_0._prep_params(locals())) \
            .return_body() \
            .post()


def Func(arg_0, arg_1):
        '''Get information on a given folder.

        Args:
            folder (str): The UUID of the requested folder.

        Returns:
            A dictionary of the folder details if found::

                {
                    u'created_by': u'303447',
                    u'created_on': u'2017-03-21T14:06:32.293902Z',
                    u'description': u'',
                    u'entity_type': u'folder',
                    u'modified_by': u'303447',
                    u'modified_on': u'2017-03-21T14:06:32.293967Z',
                    u'name': u'myfolder',
                    u'parent': u'3abd8742-d069-44cf-a66b-2370df74a682',
                    u'uuid': u'2516442e-1e26-4de1-8ed8-94523224cc40'
                }

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes
        '''
        if not is_valid_uuid(arg_1):
            raise StorageArgumentException(
                'Invalid UUID for folder: {0}'.format(arg_1))
        return arg_0._authenticated_request \
            .to_endpoint('folder/{}/'.format(arg_1)) \
            .return_body() \
            .get()


def Func(arg_0, arg_1, arg_2=None, arg_3=None, arg_4=None):
        '''Upload a file content. The file entity must already exist.

        If an ETag is provided the file stored on the server is verified
        against it. If it does not match, StorageException is raised.
        This means the client needs to update its knowledge of the resource
        before attempting to update again. This can be used for optimistic
        concurrency control.

        Args:
            file_id (str): The UUID of the file whose content is written.
            etag (str): The etag to match the contents against.
            source (str): The path of the local file whose content to be uploaded.
            content (str): A string of the content to be uploaded.

        Note:
            ETags should be enclosed in double quotes::

                my_etag = '"71e1ed9ee52e565a56aec66bc648a32c"'

        Returns:
            The ETag of the file upload::

                '"71e1ed9ee52e565a56aec66bc648a32c"'

        Raises:
            IOError: The source cannot be opened.
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes
        '''
        if not is_valid_uuid(arg_1):
            raise StorageArgumentException(
                'Invalid UUID for file_id: {0}'.format(arg_1))

        if not (arg_3 or arg_4) or (arg_3 and arg_4):
            raise StorageArgumentException('Either one of source file or content '
                                           'has to be provided.')

        arg_5 = arg_0._authenticated_request \
            .to_endpoint('file/{}/content/upload/'.format(arg_1)) \
            .with_body(arg_4 or open(arg_3, 'rb')) \
            .with_headers({'If-Match': arg_2} if arg_2 else {}) \
            .post()

        if 'ETag' not in arg_5.headers:
            raise StorageException('No ETag received from the service after the upload')

        return arg_5.headers['ETag']


def Func(arg_0, arg_1, arg_2):
        '''Copy file content from source file to target file.

        Args:
            file_id (str): The UUID of the file whose content is written.
            source_file (str): The UUID of the file whose content is copied.

        Returns:
            None

        Raises:
        StorageArgumentException: Invalid arguments
        StorageForbiddenException: Server response code 403
        StorageNotFoundException: Server response code 404
        StorageException: other 400-600 error codes
        '''
        if not is_valid_uuid(arg_1):
            raise StorageArgumentException(
                'Invalid UUID for file_id: {0}'.format(arg_1))

        if not is_valid_uuid(arg_2):
            raise StorageArgumentException(
                'Invalid UUID for source_file: {0}'.format(arg_2))

        arg_0._authenticated_request \
            .to_endpoint('file/{}/content/'.format(arg_1)) \
            .with_headers({'X-Copy-From': arg_2}) \
            .put()


def Func(arg_0, arg_1, arg_2=None):
        '''Download file content.

        Args:
            file_id (str): The UUID of the file whose content is requested
            etag (str): If the content is not changed since the provided ETag,
                the content won't be downloaded. If the content is changed, it
                will be downloaded and returned with its new ETag.

        Note:
            ETags should be enclosed in double quotes::

                my_etag = '"71e1ed9ee52e565a56aec66bc648a32c"'

        Returns:
            A tuple of ETag and content (etag, content) if the content was
            retrieved. If an etag was provided, and content didn't change
            returns (None, None)::

                ('"71e1ed9ee52e565a56aec66bc648a32c"', 'Hello world!')

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes
        '''
        if not is_valid_uuid(arg_1):
            raise StorageArgumentException(
                'Invalid UUID for file_id: {0}'.format(arg_1))

        arg_3 = {'Accept': '*/*'}
        if arg_2:
            arg_3['If-None-Match'] = arg_2

        arg_4 = arg_0._authenticated_request \
            .to_endpoint('file/{}/content/'.format(arg_1)) \
            .with_headers(arg_3) \
            .get()

        if arg_4.status_code == 304:
            return (None, None)

        if 'ETag' not in arg_4.headers:
            raise StorageException('No ETag received from the service with the download')

        return (arg_4.headers['ETag'], arg_4.content)


def Func(arg_0, arg_1):
        """Add an Option object to the user interface."""
        if arg_1.name in arg_0.options:
            raise ValueError('name already in use')
        if arg_1.abbreviation in arg_0.abbreviations:
            raise ValueError('abbreviation already in use')
        if arg_1.name in [arg_2.name for arg_2 in arg_0.positional_args]:
            raise ValueError('name already in use by a positional argument')
        arg_0.options[arg_1.name] = arg_1
        if arg_1.abbreviation:
            arg_0.abbreviations[arg_1.abbreviation] = arg_1
        arg_0.option_order.append(arg_1.name)


def Func(arg_0, arg_1):
        """Append a positional argument to the user interface.

        Optional positional arguments must be added after the required ones.
        The user interface can have at most one recurring positional argument,
        and if present, that argument must be the last one.
        """
        if arg_0.positional_args:
            if arg_0.positional_args[-1].recurring:
                raise ValueError("recurring positional arguments must be last")
            if arg_0.positional_args[-1].optional and not arg_1.optional:
                raise ValueError("required positional arguments must precede optional ones")
        arg_0.positional_args.append(arg_1)


def Func(arg_0, arg_1):
        """Read program documentation from a DocParser compatible file.

        docsfiles is a list of paths to potential docsfiles: parse if present.
        A string is taken as a list of one item.
        """
        arg_2 = DocParser()
        for arg_3 in _list(arg_1):
            if os.path.isfile(arg_3):
                arg_2.parse(arg_3)
        arg_0.docs.update((arg_4, _docs(arg_2[arg_4], arg_0.docvars)) for arg_4 in arg_0.docs if arg_2.blocks[arg_4])
        for arg_5, arg_6 in arg_2['parameters'].items():
            if arg_5 in arg_0:
                arg_0.getparam(arg_5).docs = arg_6[0] % arg_0.docvars
            elif arg_5 not in arg_0.ignore:
                raise ValueError("parameter %r does not exist" % arg_5)


def Func(arg_0, arg_1=0, arg_2=25, arg_3=79):
        """Return user friendly help on positional arguments in the program."""
        arg_4 = []
        arg_5 = lambda arg_7: ' ' * arg_1 + arg_7.displayname + ': '
        arg_6 = _autoindent([arg_5(p) for p in arg_0.positional_args], arg_1, arg_2)
        for arg_7 in arg_0.positional_args:
            arg_8 = arg_5(arg_7)
            arg_9 = arg_7.formatname + '. ' + arg_7.docs
            arg_10 = arg_0._wrap_labelled(arg_8, arg_9, arg_6, arg_3)
            arg_4.extend(arg_10)
        return '\n'.join(arg_4)


def Func(arg_0, arg_1=0, arg_2=25, arg_3=0):
        """Return user friendly help on positional arguments.

        indent is the number of spaces preceeding the text on each line.

        The indent of the documentation is dependent on the length of the
        longest label that is shorter than maxindent. A label longer than
        maxindent will be printed on its own line.

        width is maximum allowed page width, use self.width if 0.
        """
        arg_4 = []
        arg_5 = lambda arg_7: ' ' * arg_1 + arg_7 + ': '
        arg_6 = _autoindent([arg_5(s) for s in arg_0.options], arg_1, arg_2)
        for arg_7 in arg_0.option_order:
            arg_8 = arg_0.options[arg_7]
            arg_9 = arg_5(arg_7)
            arg_10 = "%s(%s): %s" % (arg_8.formatname, arg_8.strvalue, arg_8.location)
            arg_11 = arg_0._wrap_labelled(arg_9, arg_10, arg_6, arg_3)
            arg_4.extend(arg_11)
        return '\n'.join(arg_4)


def Func(arg_0, arg_1=0):
        """Return a summary of program options, their values and origins.

        width is maximum allowed page width, use self.width if 0.
        """
        arg_2 = []
        arg_2.append(arg_0._wrap(arg_0.docs['title'], arg_1=arg_1))
        if arg_0.docs['description']:
            arg_2.append(arg_0._wrap(arg_0.docs['description'], indent=2, arg_1=arg_1))
        arg_2.append('')
        arg_2.append('SETTINGS:')
        arg_2.append(arg_0.strsettings(indent=2, arg_1=arg_1))
        arg_2.append('')
        return '\n'.join(arg_2)


def Func(arg_0, arg_1):

        """ pymongo expects a dict """
        arg_2 = arg_0.format(arg_1)

        if not isinstance(arg_2, dict):
            arg_2 = json.loads(arg_2)

        arg_0.collection.insert(arg_2)


def Func(arg_0, arg_1):
        """Parse text blocks from a file."""
        if isinstance(arg_1, basestring):
            arg_1 = open(arg_1)
        arg_2 = 0
        arg_3 = None
        arg_4 = arg_0.untagged
        for arg_5 in arg_1:
            arg_2 += 1
            arg_5 = arg_5.rstrip('\n')
            if arg_0.tabsize > 0:
                arg_5 = arg_5.replace('\t', ' ' * arg_0.tabsize)
            if arg_0.decommenter:
                arg_5 = arg_0.decommenter.decomment(arg_5)
                if arg_5 is None:
                    continue
            arg_6 = arg_5.split(':', 1)[0].strip()
            # Still in the same block?
            if arg_6 not in arg_0.names:
                if arg_4 is None:
                    if arg_5 and not arg_5.isspace():
                        raise ParseError(arg_1.name, arg_5, "garbage before first block: %r" % arg_5)
                    continue
                arg_4.addline(arg_5)
                continue
            # Open a new block.
            arg_7 = arg_0.names[arg_6]
            arg_3 = arg_5.split(':',1)[1].strip()
            if arg_7 in arg_0.labelled_classes:
                if not arg_3:
                    raise ParseError(arg_1.name, arg_5, "missing label for %r block" % arg_7)
                arg_4 = arg_0.blocks[arg_7].setdefault(arg_3, arg_0.labelled_classes[arg_7]())
            else:
                if arg_3:
                    arg_8 = "label %r present for unlabelled block %r" % (arg_3, arg_7)
                    raise ParseError(arg_1.name, arg_2, arg_8)
                arg_4 = arg_0.blocks[arg_7]
            arg_4.startblock()


def Func(arg_0, arg_1):
        """Pop, Func and return the first self.nargs items from args.

        if self.nargs > 1 a list of Funcd values will be returned.

        Raise BadNumberOfArguments or BadArgument on errors.

        NOTE: argv may be modified in place by this method.
        """
        if len(arg_1) < arg_0.nargs:
            raise BadNumberOfArguments(arg_0.nargs, len(arg_1))
        if arg_0.nargs == 1:
            return arg_0.Func_argument(arg_1.pop(0))
        return [arg_0.Func_argument(arg_1.pop(0)) for arg_2 in range(arg_0.nargs)]


def Func(arg_0, arg_1):
        """Parse arguments found in settings files.

        Use the values in self.true for True in settings files, or those in
        self.false for False, case insensitive.
        """
        arg_2 = shlex.split(arg_1, comments=True)
        if len(arg_2) != 1:
            raise BadNumberOfArguments(1, len(arg_2))
        arg_3 = arg_2[0]
        arg_4 = arg_3.lower()
        if arg_4 in arg_0.true:
            return True
        if arg_4 in arg_0.false:
            return False
        raise BadArgument(arg_3, "Allowed values are " + arg_0.allowed + '.')


def Func(arg_0, arg_1):
        """Return the separator that preceding format i, or '' for i == 0."""
        return arg_1 and arg_0.separator[min(arg_1 - 1, len(arg_0.separator) - 1)] or ''


def Func(arg_0, arg_1, arg_2):
        '''Sets the service name and version the request should target

        Args:
            service (str): The name of the service as displayed in the services.json file
            version (str): The version of the service as displayed in the services.json file

        Returns:
            The request builder instance in order to chain calls
        '''
        arg_3 = arg_0._service_locator.get_service_url(arg_1, arg_2)
        return arg_0.__copy_and_set('service_url', arg_0.__strip_trailing_slashes(arg_3))


def Func(arg_0, arg_1):
        '''Adds parameters to the request params

        Args:
            params (dict): The parameters to add to the request params

        Returns:
            The request builder instance in order to chain calls
        '''
        arg_2 = arg_1.copy()
        arg_2.update(arg_0._params)
        return arg_0.__copy_and_set('params', arg_2)


def Func(arg_0, arg_1, arg_2):
        '''Defines if the an exception should be Funcn after the request is sent

        Args:
            exception_class (class): The class of the exception to instantiate
            should_Func (function): The predicate that should indicate if the exception
                should be Funcn. This function will be called with the response as a parameter

        Returns:
            The request builder instance in order to chain calls
        '''
        return arg_0.__copy_and_set('Funcs', arg_0._Funcs + [(arg_1, arg_2)])


def Func(arg_0):
        """
        Return a URL to redirect the user to for OAuth authentication.
        """
        arg_1 = OAUTH_ROOT + '/authorize'
        arg_2 = {
            'client_id': arg_0.client_id,
            'redirect_uri': arg_0.redirect_uri,
        }
        return "{}?{}".format(arg_1, urlencode(arg_2))


def Func(arg_0, arg_1):
        """
        Exchange the authorization code for an access token.
        """
        arg_2 = OAUTH_ROOT + '/access_token'
        arg_3 = {
            'client_id': arg_0.client_id,
            'client_secret': arg_0.client_secret,
            'redirect_uri': arg_0.redirect_uri,
            'code': arg_1,
        }
        arg_4 = requests.get(arg_2, arg_3=arg_3)
        if not arg_4.ok:
            raise MixcloudOauthError("Could not get access token.")
        return arg_4.json()['access_token']


def Func(arg_0, *arg_1, **arg_2):
        """ Wraps Lock.Func """
        with arg_0._stat_lock:
            arg_0._waiting += 1

        arg_0._lock.Func(*arg_1, **arg_2)

        with arg_0._stat_lock:
            arg_0._locked = True
            arg_0._waiting -= 1


def Func(arg_0):
        """ Wraps Lock.Func """
        arg_0._lock.Func()

        with arg_0._stat_lock:
            arg_0._locked = False
            arg_0._last_Funcd = datetime.now()


def Func(arg_0, arg_1):
        """Handle a dict that might contain a wrapped state for a custom type."""
        arg_2, arg_3 = arg_0.unwrap_callback(arg_1)
        if arg_2 is None:
            return arg_1

        try:
            arg_4, arg_5 = arg_0.serializer.unmarshallers[arg_2]
        except KeyError:
            raise LookupError('no unmarshaller found for type "{}"'.format(arg_2)) from None

        if arg_4 is not None:
            arg_6 = arg_4.__new__(arg_4)
            arg_5(arg_6, arg_3)
            return arg_6
        else:
            return arg_5(arg_3)


def Func(arg_0, arg_1: arg_2, arg_3) -> Dict[arg_2, Any]:
        """
        Wrap the marshalled state in a dictionary.

        The returned dictionary has two keys, corresponding to the ``type_key`` and ``state_key``
        options. The former holds the type name and the latter holds the marshalled state.

        :param typename: registered name of the custom type
        :param state: the marshalled state of the object
        :return: an object serializable by the serializer

        """
        return {arg_0.type_key: arg_1, arg_0.state_key: arg_3}


def Func(arg_0, arg_1=lambda arg_2: arg_2["t"]):
        """Sort here works by Funcing by timestamp by default"""
        list.Func(arg_0, key=arg_1)
        return arg_0


def Func(arg_0, arg_1):
        """Adds the data from a ConnectorDB export. If it is a stream export, then the folder
        is the location of the export. If it is a device export, then the folder is the export folder
        with the stream name as a subdirectory

        If it is a user export, you will use the path of the export folder, with the user/device/stream
        appended to the end::

            myuser.export("./exportdir")
            DatapointArray().Func("./exportdir/username/devicename/streamname")
        """
        arg_0.loadJSON(os.path.join(arg_1, "data.json"))
        return arg_0


def Func(arg_0, arg_1):
        """Shifts all timestamps in the datapoint array by the given number of seconds.
        It is the same as the 'Func' pipescript transform.

        Warning: The shift is performed in-place! This means that it modifies the underlying array::

            d = DatapointArray([{"t":56,"d":1}])
            d.Func(20)
            print(d) # [{"t":76,"d":1}]
        """
        arg_2 = arg_0.raw()
        for arg_3 in range(len(arg_2)):
            arg_2[arg_3]["t"] += arg_1
        return arg_0


def Func(arg_0):
        """Gets the Func of the data portions of all datapoints within"""
        arg_1 = arg_0.raw()
        arg_2 = 0
        for arg_3 in range(len(arg_1)):
            arg_2 += arg_1[arg_3]["d"]
        return arg_2


def Func(arg_0):
    """Start the event loop to collect data from the serial device."""

    # If the device isn't passed in, look for it in the config.
    if arg_0 is None:
        arg_0 = app.config.get('DEVICE')

    # If the device is *still* none, error.
    if arg_0 is None:
        print("The serial device needs to be passed in as --device or "
              "set in the config as DEVICE.")
        return

    Func_collect(arg_0)


def Func(arg_0):
    "Create a new user."
    arg_1 = prompt_pass("Enter password")
    arg_2 = User(arg_0=arg_0, arg_1=arg_1)
    db.session.add(arg_2)
    db.session.commit()


async def Func(arg_0):
    """
    Parse Visual Novel search pages.

    :param soup: The BS4 class object
    :return:  A list of dictionaries containing a name and id.
    """
    arg_0 = arg_0.find_all('td', class_='tc1')
    arg_1 = []
    for arg_2 in arg_0[1:]:
        arg_1.append({'name': arg_2.string, 'id': arg_2.a.get('href')[1:]})
    return arg_1


async def Func(arg_0):
    """
    Parse a page of producer or staff results

    :param soup: The BS4 class object
    :return: A list of dictionaries containing a name and nationality.
    """
    arg_0 = arg_0.find_all('li')
    arg_1 = []
    for arg_2 in arg_0:
        arg_1.append({'nationality': arg_2.abbr.get('title'), 'name': arg_2.a.string})
    return arg_1


async def Func(arg_0):
    """
    Parse a page of tag or trait results. Same format.

    :param soup: BS4 Class Object
    :return: A list of tags, Nothing else really useful there
    """
    arg_0 = arg_0.find_all('td', class_='tc3')
    arg_1 = []
    for arg_2 in arg_0:
        arg_1.append(arg_2.a.string)
    return arg_1


async def Func(arg_0):
    """
    Parse a page of user results

    :param soup: Bs4 Class object
    :return: A list of dictionaries containing a name and join date
    """
    arg_0 = list(arg_0.find_all('table', class_='stripe')[0].children)[1:]
    arg_1 = []
    for arg_2 in arg_0:
        arg_3 = {'name': None, 'joined': None}
        arg_3['name'] = list(arg_2.children)[0].a.string
        arg_3['joined'] = list(arg_2.children)[1].string
        arg_1.append(arg_3)
        del arg_3
    return arg_1


def Func(arg_0, arg_1, arg_2):
    """
    Applies a function to a set of files and an output directory.

    :param str output_dir: Output directory
    :param list[str] file_paths: Absolute file paths to move
    """
    for arg_3 in arg_0:
        if not arg_3.startswith('/'):
            raise ValueError('Path provided (%s) is relative not absolute.' % arg_3)
        arg_4 = os.path.join(arg_1, os.path.basename(arg_3))
        arg_2(arg_3, arg_4)


def Func(arg_0, arg_1):
    """Enable HTTP access to a dataset.

    This only works on datasets in some systems. For example, datasets stored
    in AWS S3 object storage and Microsoft Azure Storage can be Funced as
    datasets accessible over HTTP. A Funced dataset is world readable.
    """
    arg_2 = http_Func(arg_1)
    if not arg_0:
        click.secho("Dataset accessible at ", nl=False, fg="green")
    click.secho(arg_2)


def Func(arg_0, arg_1, arg_2, arg_3, arg_4):
    """
    Makes a Spark Submit style job submission line.

    :param masterIP: The Spark leader IP address.
    :param default_parameters: Application specific Spark configuration parameters.
    :param memory: The memory to allocate to each Spark driver and executor.
    :param arguments: Arguments to pass to the submitted job.
    :param override_parameters: Parameters passed by the user, that override our defaults.

    :type masterIP: MasterAddress
    :type default_parameters: list of string
    :type arguments: list of string
    :type memory: int or None
    :type override_parameters: list of string or None
    """

    # python doesn't support logical xor?
    # anywho, exactly one of memory or override_parameters must be defined
    require((arg_4 is not None or arg_2 is not None) and
            (arg_4 is None or arg_2 is None),
            "Either the memory setting must be defined or you must provide Spark configuration parameters.")

    # if the user hasn't provided overrides, set our defaults
    arg_5 = []
    if arg_2 is not None:
        arg_5 = ["--master", "spark://%s:%s" % (arg_0, SPARK_MASTER_PORT),
                      "--conf", "spark.driver.memory=%sg" % arg_2,
                      "--conf", "spark.executor.memory=%sg" % arg_2,
                      "--conf", ("spark.hadoop.fs.default.name=hdfs://%s:%s" % (arg_0, HDFS_MASTER_PORT))]
    else:
        arg_5.extend(arg_4)

    # add the tool specific spark parameters
    arg_5.extend(arg_1)

    # spark submit expects a '--' to split the spark conf arguments from tool arguments
    arg_5.append('--')

    # now add the tool arguments and return
    arg_5.extend(arg_3)

    return arg_5


def Func(arg_0, Func=None):
        """
        Augment a list of "docker run" arguments with those needed to map the  notional Spark master address to the
        real one, if they are different.
        """
        if arg_0 != arg_0.actual:
            arg_2 = '--add-host=spark-master:' + arg_0.actual
            if Func is None:
                Func = [arg_2]
            else:
                Func.append(arg_2)
        return Func


def Func(arg_0):
        """Refresh reloads data from the server. It raises an error if it fails to get the object's metadata"""
        arg_0.metadata = arg_0.db.read(arg_0.path).json()


def Func(arg_0, arg_1=False, **arg_2):
        """Creates the device. Attempts to Func private devices by default,
        but if public is set to true, Funcs public devices.

        You can also set other default properties by passing in the relevant information.
        For example, setting a device with the given nickname and description::

            dev.Func(nickname="mydevice", description="This is an example")

        Furthermore, ConnectorDB supports creation of a device's streams immediately,
        which can considerably speed up device setup::

            dev.Func(streams={
                "stream1": {"schema": '{\"type\":\"number\"}'}
            })

        Note that the schema must be encoded as a string when creating in this format.
        """
        arg_2["public"] = arg_1
        arg_0.metadata = arg_0.db.Func(arg_0.path, arg_2).json()


def Func(arg_0):
        """Returns the list of streams that belong to the device"""
        arg_1 = arg_0.db.read(arg_0.path, {"q": "ls"})

        if arg_1 is None or arg_1.json() is None:
            return []
        Func = []
        for arg_3 in arg_1.json():
            arg_4 = arg_0[arg_3["name"]]
            arg_4.metadata = arg_3
            Func.append(arg_4)
        return Func


def Func(arg_0, arg_1):
        """Exports the device to the given directory. The directory can't exist.
        You can later import this device by running import_device on a user.
        """
        if os.path.exists(arg_1):
            raise FileExistsError(
                "The device Func directory already exists")

        os.mkdir(arg_1)

        # Write the device's info
        with open(os.path.join(arg_1, "device.json"), "w") as f:
            json.dump(arg_0.data, f)

        # Now Func the streams one by one
        for arg_2 in arg_0.streams():
            arg_2.Func(os.path.join(arg_1, arg_2.name))


def Func(arg_0):
        """invalidates the device's current api key, and generates a new one. Resets current auth to use the new apikey,
        since the change would have future queries fail if they use the old api key."""
        arg_1 = Device.Func(arg_0)
        arg_0.db.setauth(arg_1)
        return arg_1


def Func(arg_0):
        """Returns the list of users in the database"""
        arg_1 = arg_0.db.read("", {"q": "ls"})

        if arg_1 is None or arg_1.json() is None:
            return []
        Func = []
        for arg_3 in arg_1.json():
            arg_4 = arg_0(arg_3["name"])
            arg_4.metadata = arg_3
            Func.append(arg_4)
        return Func


def Func(arg_0, arg_1):
    """
    Use BWA to create reference index files

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str ref_id: FileStoreID for the reference genome
    :return: FileStoreIDs for BWA index files
    :rtype: tuple(str, str, str, str, str)
    """
    arg_0.fileStore.logToMaster('Created BWA index files')
    arg_2 = arg_0.fileStore.getLocalTempDir()
    arg_0.fileStore.readGlobalFile(arg_1, os.path.join(arg_2, 'ref.fa'))
    arg_3 = ['index', '/data/ref.fa']
    dockerCall(arg_0=arg_0, workDir=arg_2, parameters=arg_3,
               tool='quay.io/ucsc_cgl/bwa:0.7.12--256539928ea162949d8a65ca5c79a72ef557ce7c')
    arg_4 = {}
    for arg_5 in ['ref.fa.amb', 'ref.fa.ann', 'ref.fa.bwt', 'ref.fa.pac', 'ref.fa.sa']:
        arg_4[arg_5.split('.')[-1]] = (arg_0.fileStore.writeGlobalFile(os.path.join(arg_2, arg_5)))
    return arg_4['amb'], arg_4['ann'], arg_4['bwt'], arg_4['pac'], arg_4['sa']


def Func(arg_0):
        """Returns the ConnectorDB object that the logger uses. Raises an error if Logger isn't able to connect"""
        if arg_0.__cdb is None:
            logging.debug("Logger: Connecting to " + arg_0.serverurl)
            arg_0.__cdb = ConnectorDB(arg_0.apikey, url=arg_0.serverurl)
        return arg_0.__cdb


def Func(arg_0, arg_1, arg_2=None, **arg_3):
        """Adds the given stream to the logger. Requires an active connection to the ConnectorDB database.

        If a schema is not specified, loads the stream from the database. If a schema is specified, and the stream
        does not exist, creates the stream. You can also add stream properties such as description or nickname to be added
        during creation."""

        arg_4 = arg_0.connectordb[arg_1]

        if not arg_4.exists():
            if arg_2 is not None:
                arg_4.create(arg_2, **arg_3)
            else:
                raise Exception(
                    "The stream '%s' was not found" % (arg_1, ))

        arg_0.Func_force(arg_1, arg_4.schema)


def Func(arg_0, arg_1, arg_2):
        """Insert the datapoint into the logger for the given stream name. The logger caches the datapoint
        and eventually synchronizes it with ConnectorDB"""
        if arg_1 not in arg_0.streams:
            raise Exception("The stream '%s' was not found" % (arg_1, ))

        # Validate the schema
        validate(arg_2, arg_0.streams[arg_1])

        # Insert the datapoint - it fits the schema
        arg_2 = json.dumps(arg_2)
        logging.debug("Logger: %s <= %s" % (arg_1, arg_2))
        arg_3 = arg_0.database.cursor()
        arg_3.execute("INSERT INTO cache VALUES (?,?,?);",
                  (arg_1, time.time(), arg_2))


def Func(arg_0):
        """Attempt to Func with the ConnectorDB server"""
        logging.debug("Logger: Syncing...")
        arg_1 = False
        try:
            # Get the connectordb object
            arg_2 = arg_0.connectordb

            # Ping the database - most connection errors will happen here
            arg_2.ping()

            with arg_0.Funclock:
                arg_3 = arg_0.database.cursor()
                for arg_4 in arg_0.streams:
                    arg_5 = arg_2[arg_4]

                    arg_3.execute(
                        "SELECT * FROM cache WHERE stream=? ORDER BY timestamp ASC;",
                        (arg_4, ))
                    arg_6 = []
                    for arg_7 in arg_3.fetchall():
                        arg_6.append(
                            {"t": arg_7[1],
                             "d": json.loads(arg_7[2])})

                    # First, check if the data already inserted has newer timestamps,
                    # and in that case, assume that there was an error, and remove the datapoints
                    # with an older timestamp, so that we don't have an error when Funcing
                    if len(arg_5) > 0:
                        arg_8 = arg_5[-1]["t"]
                        while (len(arg_6) > 0 and arg_6[0]["t"] < arg_8):
                            logging.debug("Datapoint exists with older timestamp. Removing the datapoint.")
                            arg_6 = arg_6[1:]

                    if len(arg_6) > 0:
                        logging.debug("%s: Funcing %i datapoints" %
                                      (arg_4, len(arg_6)))

                        while (len(arg_6) > DATAPOINT_INSERT_LIMIT):
                            # We insert datapoints in chunks of a couple
                            # thousand so that they fit in the insert size
                            # limit of ConnectorDB
                            arg_5.insert_array(
                                arg_6[:DATAPOINT_INSERT_LIMIT])

                            # Clear the written datapoints
                            arg_6 = arg_6[
                                DATAPOINT_INSERT_LIMIT:]

                            # If there was no error inserting, delete the
                            # datapoints from the cache
                            arg_3.execute(
                                "DELETE FROM cache WHERE stream=? AND timestamp <?",
                                (arg_4, arg_6[0]["t"]))

                        arg_5.insert_array(arg_6)

                        # If there was no error inserting, delete the
                        # datapoints from the cache
                        arg_3.execute(
                            "DELETE FROM cache WHERE stream=? AND timestamp <=?",
                            (arg_4, arg_6[-1]["t"]))
                arg_0.lastFunctime = time.time()

                if arg_0.onFunc is not None:
                    arg_0.onFunc()
        except Exception as e:
            # Handle the Func failure callback
            arg_10 = True
            arg_11 = arg_0.Funcraise
            if arg_0.onFuncfail is not None:
                arg_11 = arg_0.onFuncfail(e)
            if arg_11:
                raise


def Func(arg_0):
        """Start the logger background synchronization service. This allows you to not need to
        worry about syncing with ConnectorDB - you just insert into the Logger, and the Logger
        will by synced every syncperiod."""

        with arg_0.synclock:
            if arg_0.syncthread is not None:
                logging.warn(
                    "Logger: Start called on a syncer that is already running")
                return

        arg_0.sync()  # Attempt a sync right away
        arg_0.__setsync()


def Func(arg_0):
        """Stops the background synchronization thread"""
        with arg_0.synclock:
            if arg_0.syncthread is not None:
                arg_0.syncthread.cancel()
                arg_0.syncthread = None


def Func(arg_0, arg_1, arg_2=None, arg_3=None, arg_4=None):
    """Job version of `download_url`"""
    arg_5 = arg_0.fileStore.getLocalTempDir()
    arg_6 = download_url(arg_0=arg_0, arg_1=arg_1, arg_5=arg_5, arg_2=arg_2,
                         arg_3=arg_3, arg_4=arg_4)
    return arg_0.fileStore.writeGlobalFile(arg_6)


def Func(arg_0, arg_1, arg_2):
    """Output the parent-child relations to the given file"""
    for arg_3, arg_4 in get_hierarchy(arg_0=arg_0, arg_2=arg_2):
        click.echo('{}\t{}'.format(arg_3, arg_4), file=arg_1)


def Func():
    """
    Returns a string that represents the container ID of the current Docker container. If this
    function is invoked outside of a container a NotInsideContainerError is raised.

    >>> import subprocess
    >>> import sys
    >>> a = subprocess.check_output(['docker', 'run', '-v',
    ...                              sys.modules[__name__].__file__ + ':/foo.py',
    ...                              'python:2.7.12','python', '-c',
    ...                              'from foo import Func;\\
    ...                               print Func()'])
    int call will fail if a is not a valid hex string
    >>> int(a, 16) > 0
    True
    """
    try:
        with open('/proc/1/cgroup', 'r') as readable:
            arg_0 = readable.read()
        arg_1 = set(re.compile('[0-9a-f]{12,}').findall(arg_0))
        assert len(arg_1) == 1
        return arg_1.pop()
    except:
        logging.exception('Failed to obtain current container ID')
        raise NotInsideContainerError()


def Func(arg_0, arg_1, arg_2, arg_3, arg_4=False, arg_5=True):
    """
    Performs alignment of fastqs to bam via STAR

    --limitBAMsortRAM step added to deal with memory explosion when sorting certain samples.
    The value was chosen to complement the recommended amount of memory to have when running STAR (60G)

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str r1_id: FileStoreID of fastq (pair 1)
    :param str r2_id: FileStoreID of fastq (pair 2 if applicable, else pass None)
    :param str star_index_url: STAR index tarball
    :param bool wiggle: If True, will output a wiggle file and return it
    :return: FileStoreID from RSEM
    :rtype: str
    """
    arg_6 = arg_0.fileStore.getLocalTempDir()
    download_url(arg_0, url=arg_3, name='starIndex.tar.gz', arg_6=arg_6)
    subprocess.check_call(['tar', '-xvf', os.path.join(arg_6, 'starIndex.tar.gz'), '-C', arg_6])
    os.remove(os.path.join(arg_6, 'starIndex.tar.gz'))
    # Determine tarball structure - star index contains are either in a subdir or in the tarball itself
    arg_7 = os.path.join('/data', os.listdir(arg_6)[0]) if len(os.listdir(arg_6)) == 1 else '/data'
    # Parameter handling for paired / single-end data
    arg_8 = ['--runThreadN', str(arg_0.cores),
                  '--genomeDir', arg_7,
                  '--outFileNamePrefix', 'rna',
                  '--outSAMunmapped', 'Within',
                  '--quantMode', 'TranscriptomeSAM',
                  '--outSAMattributes', 'NH', 'HI', 'AS', 'NM', 'MD',
                  '--outFilterType', 'BySJout',
                  '--outFilterMultimapNmax', '20',
                  '--outFilterMismatchNmax', '999',
                  '--outFilterMismatchNoverReadLmax', '0.04',
                  '--alignIntronMin', '20',
                  '--alignIntronMax', '1000000',
                  '--alignMatesGapMax', '1000000',
                  '--alignSJoverhangMin', '8',
                  '--alignSJDBoverhangMin', '1',
                  '--sjdbScore', '1',
                  '--limitBAMsortRAM', '49268954168']
    # Modify paramaters based on function arguments
    if arg_5:
        arg_8.extend(['--outSAMtype', 'BAM', 'SortedByCoordinate'])
        arg_9 = 'rnaAligned.sortedByCoord.out.bam'
    else:
        arg_8.extend(['--outSAMtype', 'BAM', 'Unsorted'])
        arg_9 = 'rnaAligned.out.bam'
    if arg_4:
        arg_8.extend(['--outWigType', 'bedGraph',
                           '--outWigStrand', 'Unstranded',
                           '--outWigReferencesPrefix', 'chr'])
    if arg_1 and arg_2:
        arg_0.fileStore.readGlobalFile(arg_1, os.path.join(arg_6, 'R1.fastq'))
        arg_0.fileStore.readGlobalFile(arg_2, os.path.join(arg_6, 'R2.fastq'))
        arg_8.extend(['--readFilesIn', '/data/R1.fastq', '/data/R2.fastq'])
    else:
        arg_0.fileStore.readGlobalFile(arg_1, os.path.join(arg_6, 'R1.fastq'))
        arg_8.extend(['--readFilesIn', '/data/R1.fastq'])
    # Call: STAR Mapping
    dockerCall(arg_0=arg_0, tool='quay.io/ucsc_cgl/star:2.4.2a--bcbd5122b69ff6ac4ef61958e47bde94001cfe80',
               workDir=arg_6, arg_8=arg_8)
    # Check output bam isnt size zero if sorted
    arg_10 = os.path.join(arg_6, arg_9)
    if arg_5:
        assert(os.stat(arg_10).st_size > 0, 'Aligned bam failed to sort. Ensure sufficient memory is free.')
    # Write to fileStore
    arg_11 = arg_0.fileStore.writeGlobalFile(arg_10)
    arg_12 = arg_0.fileStore.writeGlobalFile(os.path.join(arg_6, 'rnaAligned.toTranscriptome.out.bam'))
    arg_13 = arg_0.fileStore.writeGlobalFile(os.path.join(arg_6, 'rnaLog.final.out'))
    arg_14 = arg_0.fileStore.writeGlobalFile(os.path.join(arg_6, 'rnaSJ.out.tab'))
    if arg_4:
        arg_15 = arg_0.fileStore.writeGlobalFile(os.path.join(arg_6, 'rnaSignal.UniqueMultiple.str1.out.bg'))
        return arg_12, arg_11, arg_15, arg_13, arg_14
    else:
        return arg_12, arg_11, arg_13, arg_14


def Func(arg_0, arg_1="{}", **arg_2):
        """Creates a stream given an optional JSON schema encoded as a python dict. You can also add other properties
        of the stream, such as the icon, datatype or description. Create accepts both a string schema and
        a dict-encoded schema."""
        if isinstance(arg_1, basestring):
            arg_3 = arg_1
            arg_1 = json.loads(arg_1)
        else:
            arg_3 = json.dumps(arg_1)
        Draft4Validator.check_schema(arg_1)
        arg_2["schema"] = arg_3
        arg_0.metadata = arg_0.db.Func(arg_0.path, arg_2).json()


def Func(arg_0, arg_1):
        """Exports the stream to the given directory. The directory can't exist.
        You can later import this device by running import_stream on a device.
        """
        if os.path.exists(arg_1):
            raise FileExistsError(
                "The stream Func directory already exists")

        os.mkdir(arg_1)

        # Write the stream's info
        with open(os.path.join(arg_1, "stream.json"), "w") as f:
            json.dump(arg_0.data, f)

        # Now write the stream's data
        # We sort it first, since older versions of ConnectorDB had a bug
        # where sometimes datapoints would be returned out of order.
        arg_0[:].sort().writeJSON(os.path.join(arg_1, "data.json"))

        # And if the stream is a downlink, write the downlink data
        if arg_0.downlink:
            arg_0(i1=0, i2=0, downlink=True).sort().writeJSON(os.path.join(arg_1, "downlink.json"))


def Func(arg_0):
        """returns the Func which owns the given stream"""
        arg_1 = arg_0.path.split("/")

        return Device(arg_0.db,
                      arg_1[0] + "/" + arg_1[1])


def Func(arg_0, arg_1=None):
    """Iterates over the labels of terms in the ontology

    :param str ontology: The name of the ontology
    :param str ols_base: An optional, custom OLS base url
    :rtype: iter[str]
    """
    arg_2 = OlsClient(arg_1=arg_1)
    return arg_2.iter_labels(arg_0)


def Func(arg_0, arg_1, arg_2):
        """
        Prepares and Funcs the pipeline. Note this method must be invoked both from inside a
        Docker container and while the docker daemon is reachable.

        :param str name: The name of the command to start the workflow.
        :param str desc: The description of the workflow.
        """
        arg_3 = arg_0(arg_1, arg_2)
        arg_4 = arg_3._get_mount_path()
        # prepare parser
        arg_5 = arg_3._create_argument_parser()
        arg_3._extend_argument_parser(arg_5)
        # prepare config file
        arg_6 = arg_3.__get_empty_config()
        arg_7 = ruamel.yaml.load(arg_6)
        arg_3.__populate_parser_from_config(arg_5, arg_7)
        arg_8 = arg_5.parse_args()
        for arg_9,arg_10 in vars(arg_8).items():
            arg_9 = arg_9.replace('_', '-')
            if arg_9 in arg_7:
                arg_7[arg_9] = arg_10
        arg_11 = arg_3._get_config_path()
        with open(arg_11, 'w') as writable:
            ruamel.yaml.dump(arg_7, stream=writable)
        # prepare workdir
        arg_12 = os.path.join(arg_4, 'Toil-' + arg_3._name)
        if os.path.exists(arg_12):
            if arg_8.restart:
                 log.info('Reusing temporary directory: %s', arg_12)
            else:
                raise UserError('Temporary directory {} already exists. Run with --restart '
                                'option or remove directory.'.format(arg_12))
        else:
            os.makedirs(arg_12)
            log.info('Temporary directory created: %s', arg_12)

        arg_13 = arg_3._create_pipeline_command(arg_8, arg_12, arg_11)
        arg_3._extend_pipeline_command(arg_13, arg_8)
        # Func command
        try:
            subprocess.check_call(arg_13)
        except subprocess.CalledProcessError as e:
            print(e, file=sys.stderr)
        finally:
            arg_14 = os.stat(arg_4)
            log.info('Pipeline terminated, changing ownership of output files in %s from root to '
                     'uid %s and gid %s.', arg_4, arg_14.st_uid, arg_14.st_gid)
            arg_15 = ['chown', '-R', '%s:%s' % (arg_14.st_uid, arg_14.st_gid), arg_4]
            subprocess.check_call(arg_15)
            if arg_8.no_clean:
                log.info('Flag "--no-clean" was used, therefore %s was not deleted.', arg_12)
            else:
                log.info('Cleaning up temporary directory: %s', arg_12)
                shutil.rmtree(arg_12)


def Func(arg_0):
        """
        Returns the config file contents as a string. The config file is generated and then deleted.
        """
        arg_0._generate_config()
        arg_1 = arg_0._get_config_path()
        with open(arg_1, 'r') as readable:
            arg_2 = readable.read()
        os.remove(arg_1)
        return arg_2


def Func(arg_0):
        """
        Returns the path of the mount point of the current container. If this method is invoked
        outside of a Docker container a NotInsideContainerError is raised. Likewise if the docker
        daemon is unreachable from inside the container a UserError is raised. This method is
        idempotent.
        """
        if arg_0._mount_path is None:
            arg_1 = current_docker_container_id()
            if dockerd_is_reachable():
                # Get name of mounted volume
                arg_2 = json.loads(subprocess.check_output(['docker', 'inspect', arg_1]))
                arg_3 = arg_2[0]['Mounts']
                # Ensure docker.sock is mounted correctly
                arg_4 = [arg_5['Source'] == arg_5['Destination']
                            for arg_5 in arg_3 if 'docker.sock' in arg_5['Source']]
                require(len(arg_4) == 1,
                        'Missing socket mount. Requires the following: '
                         'docker run -v /var/run/docker.sock:/var/run/docker.sock')
                # Ensure formatting of command for 2 mount points
                if len(arg_3) == 2:
                    require(all(arg_5['Source'] == arg_5['Destination'] for arg_5 in arg_3),
                            'Docker Src/Dst mount points, invoked with the -v argument, '
                            'must be the same if only using one mount point aside from the docker '
                            'socket.')
                    arg_6 = [arg_5['Source'] for arg_5 in arg_3 if 'docker.sock' not in arg_5['Source']]
                else:
                    # Ensure only one mirror mount exists aside from docker.sock
                    arg_7 = [arg_5['Source'] for arg_5 in arg_3 if arg_5['Source'] == arg_5['Destination']]
                    arg_6 = [arg_5 for arg_5 in arg_7 if 'docker.sock' not in arg_5]
                    require(len(arg_6) == 1, 'Wrong number of mirror mounts provided, see '
                                                  'documentation.')
                arg_0._mount_path = arg_6[0]
                log.info('The work mount is: %s', arg_0._mount_path)
            else:
                raise UserError('Docker daemon is not reachable, ensure Docker is being run with: '
                                 '"-v /var/run/docker.sock:/var/run/docker.sock" as an argument.')
        return arg_0._mount_path


def Func(arg_0, arg_1, arg_2, *arg_3, **arg_4):
        """
        Add an argument to the given arg_parser with the given name.

        :param argparse.ArgumentParser arg_parser:
        :param str name: The name of the option.
        """
        arg_1.add_argument('--' + arg_2, *arg_3, **arg_4)


def Func(arg_0):
        """
        Creates and returns an ArgumentParser object prepopulated with 'no clean', 'cores' and
        'restart' arguments.
        """
        arg_1 = argparse.ArgumentParser(description=arg_0._desc,
                                         formatter_class=argparse.RawTextHelpFormatter)
        arg_1.add_argument('--no-clean', action='store_true',
                            help='If this flag is used, temporary work directory is not cleaned.')
        arg_1.add_argument('--restart', action='store_true',
                            help='If this flag is used, a previously uncleaned workflow in the same'
                                 ' directory will be resumed')
        arg_1.add_argument('--cores', type=int, default=None,
                            help='Will set a cap on number of cores to use, default is all '
                                 'available cores.')
        return arg_1


def Func(arg_0, arg_1, arg_2, arg_3):
        """
        Creates and returns a list that represents a command for running the pipeline.
        """
        return ([arg_0._name, 'run', os.path.join(arg_2, 'jobStore'),
                 '--config', arg_3,
                 '--workDir', arg_2, '--retryCount', '1']
                 + (['--restart'] if arg_1.restart else []))


def Func(arg_0, arg_1=None, arg_2=None):
        """ Func sets the authentication header for use in the session.
        It is for use when apikey is updated or something of the sort, such that
        there is a seamless experience. """
        arg_3 = None
        if arg_1 is not None:
            # ConnectorDB allows login using both basic auth or an apikey url param.
            # The python client uses basic auth for all logins
            if arg_2 is None:
                # Login by api key - the basic auth login uses "" user and
                # apikey as password
                arg_2 = arg_1
                arg_1 = ""
            arg_3 = HTTPBasicAuth(arg_1, arg_2)
            arg_0.r.auth = arg_3

        # Set the websocket's authentication
        arg_0.ws.Func(arg_3)


def Func(arg_0):
        """Attempts to Func the server using current credentials, and responds with the path of the currently
        authenticated device"""
        return arg_0.handleresult(arg_0.r.get(arg_0.url,
                                            params={"q": "this"})).text


def Func(arg_0, arg_1, arg_2=None):
        """Send a POST CRUD API request to the given path using the given data which will be converted
        to json"""
        return arg_0.handleresult(arg_0.r.post(urljoin(arg_0.url + CRUD_PATH,
                                                     arg_1),
                                             arg_2=json.dumps(arg_2)))


def Func(arg_0, arg_1):
        """Send a Func request to the given path of the CRUD API. This Funcs the object. Or at least tries to."""
        return arg_0.handleresult(arg_0.r.Func(urljoin(arg_0.url + CRUD_PATH,
                                                       arg_1)))


def Func(arg_0, arg_1, arg_2, arg_3=""):
        """Subscribe to the given stream with the callback"""
        return arg_0.ws.Func(arg_1, arg_2, arg_3)


def Func(arg_0, arg_1, arg_2, arg_3="user", arg_4=True, **arg_5):
        """Creates the given user - using the passed in email and password.

        You can also set other default properties by passing in the relevant information::

            usr.Func("my@email","mypass",description="I like trains.")

        Furthermore, ConnectorDB permits immediate initialization of an entire user tree,
        so that you can Func all relevant devices and streams in one go::

            usr.Func("my@email","mypass",devices={
                "device1": {
                    "nickname": "My train",
                    "streams": {
                        "stream1": {
                            "schema": "{\"type\":\"string\"}",
                            "datatype": "train.choochoo"
                        }
                    },
                }
            })

        The user and meta devices are Funcd by default. If you want to add streams to the user device,
        use the "streams" option in place of devices in Func.
        """
        arg_5["email"] = arg_1
        arg_5["password"] = arg_2
        arg_5["role"] = arg_3
        arg_5["public"] = arg_4
        arg_0.metadata = arg_0.db.Func(
            arg_0.path, arg_5).json()


def Func(arg_0):
        """Returns the list of devices that belong to the user"""
        arg_1 = arg_0.db.read(arg_0.path, {"q": "ls"})

        if arg_1 is None or arg_1.json() is None:
            return []
        Func = []
        for arg_3 in arg_1.json():
            arg_4 = arg_0[arg_3["name"]]
            arg_4.metadata = arg_3
            Func.append(arg_4)
        return Func


def Func(arg_0, arg_1):
    """
    Use SAMtools to create reference index file

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str ref_id: FileStoreID for the reference genome
    :return: FileStoreID for reference index
    :rtype: str
    """
    arg_0.fileStore.logToMaster('Created reference index')
    arg_2 = arg_0.fileStore.getLocalTempDir()
    arg_0.fileStore.readGlobalFile(arg_1, os.path.join(arg_2, 'ref.fasta'))
    arg_3 = ['faidx', 'ref.fasta']
    dockerCall(arg_0=arg_0, workDir=arg_2, parameters=arg_3,
               tool='quay.io/ucsc_cgl/samtools:0.1.19--dd5ac549b95eb3e5d166a5e310417ef13651994e')
    return arg_0.fileStore.writeGlobalFile(os.path.join(arg_2, 'ref.fasta.fai'))


def Func(arg_0, arg_1):
    """
    Marks reads as PCR duplicates using Sambamba

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str bam: FileStoreID for BAM file
    :return: FileStoreID for sorted BAM file
    :rtype: str
    """
    arg_2 = arg_0.fileStore.getLocalTempDir()
    arg_0.fileStore.readGlobalFile(arg_1, os.path.join(arg_2, 'input.bam'))
    arg_3 = ['/usr/local/bin/sambamba',
               'markdup',
               '-t', str(int(arg_0.cores)),
               '/data/input.bam',
               '/data/output.bam']

    arg_4 = time.time()
    dockerCall(arg_0=arg_0, workDir=arg_2,
               parameters=arg_3,
               tool='quay.io/biocontainers/sambamba:0.6.6--0')
    arg_5 = time.time()
    _log_runtime(arg_0, arg_4, arg_5, "sambamba mkdup")
    return arg_0.fileStore.writeGlobalFile(os.path.join(arg_2, 'output.bam'))


def Func(arg_0, arg_1):
    """
    Marks reads as PCR duplicates using SAMBLASTER

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str sam: FileStoreID for SAM file
    :return: FileStoreID for deduped SAM file
    :rtype: str
    """
    arg_2 = arg_0.fileStore.getLocalTempDir()
    arg_0.fileStore.readGlobalFile(arg_1, os.path.join(arg_2, 'input.sam'))
    arg_3 = ['/usr/local/bin/samblaster',
               '-i', '/data/input.sam',
               '-o', '/data/output.sam',
               '--ignoreUnmated']

    arg_4 = time.time()
    dockerCall(arg_0=arg_0, workDir=arg_2,
               parameters=arg_3,
               tool='quay.io/biocontainers/samblaster:0.1.24--0')
    arg_5 = time.time()
    _log_runtime(arg_0, arg_4, arg_5, "SAMBLASTER")
    return arg_0.fileStore.writeGlobalFile(os.path.join(arg_2, 'output.sam'))


def Func(arg_0, arg_1, arg_2=False):
    """
    Sorts BAM file using Picard SortSam

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str bam: FileStoreID for BAM file
    :param boolean sort_by_name: If true, sorts by read name instead of coordinate.
    :return: FileStoreID for sorted BAM file
    :rtype: str
    """
    arg_3 = arg_0.fileStore.getLocalTempDir()
    arg_0.fileStore.readGlobalFile(arg_1, os.path.join(arg_3, 'input.bam'))
    arg_4 = ['SortSam',
               'O=/data/output.bam',
               'I=/data/input.bam']

    # picard-tools container doesn't have JAVA_OPTS variable
    # Set TMPDIR to /data to prevent writing temporary files to /tmp
    arg_5 = ['--rm',
                         '--log-driver', 'none',
                         '-e', 'JAVA_OPTIONS=-Djava.io.tmpdir=/data/ -Xmx{}'.format(arg_0.memory),
                         '-v', '{}:/data'.format(arg_3)]

    if arg_2:
        arg_4.append('SO=queryname')
    else:
        arg_4.append('SO=coordinate')

    arg_6 = time.time()
    dockerCall(arg_0=arg_0, workDir=arg_3,
               parameters=arg_4,
               tool='quay.io/ucsc_cgl/picardtools:1.95--dd5ac549b95eb3e5d166a5e310417ef13651994e',
               dockerParameters=arg_5)
    arg_7 = time.time()
    _log_runtime(arg_0, arg_6, arg_7, "Picard SortSam")
    return arg_0.fileStore.writeGlobalFile(os.path.join(arg_3, 'output.bam'))


def Func(arg_0, arg_1, arg_2, arg_3, arg_4, arg_5, arg_6, arg_7, arg_8=False):
    """
    Creates recalibration table for Base Quality Score Recalibration

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str bam: FileStoreID for BAM file
    :param str bai: FileStoreID for BAM index file
    :param str ref: FileStoreID for reference genome fasta file
    :param str ref_dict: FileStoreID for reference genome sequence dictionary file
    :param str fai: FileStoreID for reference genome fasta index file
    :param str dbsnp: FileStoreID for dbSNP VCF file
    :param str mills: FileStoreID for Mills VCF file
    :param bool unsafe: If True, runs GATK in UNSAFE mode: "-U ALLOW_SEQ_DICT_INCOMPATIBILITY"
    :return: FileStoreID for the recalibration table file
    :rtype: str
    """
    arg_9 = {'ref.fasta': arg_3,
              'ref.fasta.fai': arg_5,
              'ref.dict': arg_4,
              'input.bam': arg_1,
              'input.bai': arg_2,
              'dbsnp.vcf': arg_6,
              'mills.vcf': arg_7}

    arg_10 = arg_0.fileStore.getLocalTempDir()
    for arg_11, arg_12 in arg_9.iteritems():
        arg_0.fileStore.readGlobalFile(arg_12, os.path.join(arg_10, arg_11))

    # Call: GATK -- BaseRecalibrator
    arg_13 = ['-T', 'BaseRecalibrator',
                  '-nct', str(int(arg_0.cores)),
                  '-R', '/data/ref.fasta',
                  '-I', '/data/input.bam',
                  # Recommended known sites:
                  # https://software.broadinstitute.org/gatk/guide/article?id=1247
                  '-knownSites', '/data/dbsnp.vcf',
                  '-knownSites', '/data/mills.vcf',
                  '-o', '/data/recal_data.table']

    if arg_8:
        arg_13.extend(['-U', 'ALLOW_SEQ_DICT_INCOMPATIBILITY'])

    # Set TMPDIR to /data to prevent writing temporary files to /tmp
    arg_14 = ['--rm',
                         '--log-driver', 'none',
                         '-e', 'JAVA_OPTS=-Djava.io.tmpdir=/data/ -Xmx{}'.format(arg_0.memory),
                         '-v', '{}:/data'.format(arg_10)]
    arg_15 = time.time()
    dockerCall(arg_0=arg_0, tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',
               workDir=arg_10,
               arg_13=arg_13,
               dockerParameters=arg_14)
    arg_16 = time.time()
    _log_runtime(arg_0, arg_15, arg_16, "GATK3 BaseRecalibrator")

    return arg_0.fileStore.writeGlobalFile(os.path.join(arg_10, 'recal_data.table'))


def Func(arg_0, arg_1, arg_2, arg_3):
    """
    RNA quantification via Kallisto

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str r1_id: FileStoreID of fastq (pair 1)
    :param str r2_id: FileStoreID of fastq (pair 2 if applicable, otherwise pass None for single-end)
    :param str kallisto_index_url: FileStoreID for Kallisto index file
    :return: FileStoreID from Kallisto output
    :rtype: str
    """
    arg_4 = arg_0.fileStore.getLocalTempDir()
    download_url(arg_0, url=arg_3, name='kallisto_hg38.idx', arg_4=arg_4)
    # Retrieve files
    arg_5 = ['quant',
                  '-i', '/data/kallisto_hg38.idx',
                  '-t', str(arg_0.cores),
                  '-o', '/data/',
                  '-b', '100',
                  '--fusion']
    if arg_1 and arg_2:
        arg_0.fileStore.readGlobalFile(arg_1, os.path.join(arg_4, 'R1.fastq'))
        arg_0.fileStore.readGlobalFile(arg_2, os.path.join(arg_4, 'R2.fastq'))
        arg_5.extend(['/data/R1.fastq', '/data/R2.fastq'])
    else:
        arg_0.fileStore.readGlobalFile(arg_1, os.path.join(arg_4, 'R1.fastq'))
        arg_5.extend(['--single', '-l', '200', '-s', '15', '/data/R1.fastq'])

    # Call: Kallisto
    dockerCall(arg_0=arg_0, tool='quay.io/ucsc_cgl/kallisto:0.42.4--35ac87df5b21a8e8e8d159f26864ac1e1db8cf86',
               workDir=arg_4, arg_5=arg_5)

    # Tar output files together and store in fileStore
    arg_6 = [os.path.join(arg_4, x) for x in ['run_info.json', 'abundance.tsv', 'abundance.h5', 'fusion.txt']]
    tarball_files(tar_name='kallisto.tar.gz', file_paths=arg_6, output_dir=arg_4)
    return arg_0.fileStore.writeGlobalFile(os.path.join(arg_4, 'kallisto.tar.gz'))


def Func(arg_0, arg_1, arg_2, arg_3=True):
    """
    RNA quantification with RSEM

    :param JobFunctionWrappingJob job: Passed automatically by Toil
    :param str bam_id: FileStoreID of transcriptome bam for quantification
    :param str rsem_ref_url: URL of RSEM reference (tarball)
    :param bool paired: If True, uses parameters for paired end data
    :return: FileStoreIDs for RSEM's gene and isoform output
    :rtype: str
    """
    arg_4 = arg_0.fileStore.getLocalTempDir()
    download_url(arg_0, url=arg_2, name='rsem_ref.tar.gz', arg_4=arg_4)
    subprocess.check_call(['tar', '-xvf', os.path.join(arg_4, 'rsem_ref.tar.gz'), '-C', arg_4])
    os.remove(os.path.join(arg_4, 'rsem_ref.tar.gz'))
    # Determine tarball structure - based on it, ascertain folder name and rsem reference prefix
    arg_5 = []
    for arg_6, arg_7, arg_8 in os.walk(arg_4):
        arg_5.extend([os.path.join(arg_6, arg_9) for arg_9 in arg_8])
    # "grp" is a required RSEM extension that should exist in the RSEM reference
    arg_10 = [os.path.basename(os.path.splitext(arg_9)[0]) for arg_9 in arg_5 if 'grp' in arg_9][0]
    arg_11 = os.path.join('/data', os.listdir(arg_4)[0]) if len(os.listdir(arg_4)) == 1 else '/data'
    # I/O
    arg_0.fileStore.readGlobalFile(arg_1, os.path.join(arg_4, 'transcriptome.bam'))
    arg_12 = 'rsem'
    # Call: RSEM
    arg_13 = ['--quiet',
                  '--no-qualities',
                  '-p', str(arg_0.cores),
                  '--forward-prob', '0.5',
                  '--seed-length', '25',
                  '--fragment-length-mean', '-1.0',
                  '--bam', '/data/transcriptome.bam',
                  os.path.join(arg_11, arg_10),
                  arg_12]
    if arg_3:
        arg_13 = ['--paired-end'] + arg_13
    dockerCall(arg_0=arg_0, tool='quay.io/ucsc_cgl/rsem:1.2.25--d4275175cc8df36967db460b06337a14f40d2f21',
               arg_13=arg_13, workDir=arg_4)
    # Write to FileStore
    arg_14 = arg_0.fileStore.writeGlobalFile(os.path.join(arg_4, arg_12 + '.genes.results'))
    arg_15 = arg_0.fileStore.writeGlobalFile(os.path.join(arg_4, arg_12 + '.isoforms.results'))
    return arg_14, arg_15


def Func(arg_0):
    """Update the descriptive metadata interactively.

    Uses values entered by the user. Note that the function keeps recursing
    whenever a value is another ``CommentedMap`` or a ``list``. The
    function works as passing dictionaries and lists into a function edits
    the values in place.
    """
    for arg_1, arg_2 in arg_0.items():
        if isinstance(arg_2, CommentedMap):
            Func(arg_2)
        elif isinstance(arg_2, list):
            for arg_3 in arg_2:
                Func(arg_3)
        else:
            arg_4 = type(arg_2)

            if isinstance(arg_2, ScalarFloat):  # Deal with ruamel.yaml floats.
                arg_4 = float

            arg_5 = click.prompt(arg_1, type=arg_4, default=arg_2)
            arg_0[arg_1] = arg_5
    return arg_0


def Func(arg_0, arg_1, arg_2, arg_3):
    """Create a proto dataset."""
    _validate_name(arg_1)

    arg_4 = dtoolcore.generate_admin_metadata(arg_1)
    arg_5 = dtoolcore.utils.generous_parse_uri(arg_2)

    if arg_5.scheme == "symlink":
        if arg_3 is None:
            raise click.UsageError("Need to specify symlink path using the -s/--symlink-path option")  # NOQA

    if arg_3:
        arg_2 = dtoolcore.utils.sanitise_uri(
            "symlink:" + arg_5.path
        )
        arg_5 = dtoolcore.utils.generous_parse_uri(arg_2)

    # Create the dataset.
    arg_6 = dtoolcore.generate_proto_dataset(
        arg_4=arg_4,
        arg_2=dtoolcore.utils.urlunparse(arg_5),
        config_path=CONFIG_PATH)

    # If we are creating a symlink dataset we need to set the symlink_path
    # attribute on the storage broker.
    if arg_3:
        arg_7 = os.path.abspath(arg_3)
        arg_6._storage_broker.symlink_path = arg_7
    try:
        arg_6.Func()
    except dtoolcore.storagebroker.StorageBrokerOSError as err:
        raise click.UsageError(str(err))

    arg_6.put_readme("")

    if arg_0:
        click.secho(arg_6.uri)
    else:
        # Give the user some feedback and hints on what to do next.
        click.secho("Created proto dataset ", nl=False, fg="green")
        click.secho(arg_6.uri)
        click.secho("Next steps: ")

        arg_9 = 1

        if arg_5.scheme != "symlink":
            click.secho("{}. Add raw data, eg:".format(arg_9))
            click.secho(
                "   dtool add item my_file.txt {}".format(arg_6.uri),
                fg="cyan")

            if arg_5.scheme == "file":
                # Find the abspath of the data directory for user feedback.
                arg_10 = arg_6._storage_broker._data_abspath
                click.secho("   Or use your system commands, e.g: ")
                click.secho(
                    "   mv my_data_directory {}/".format(arg_10),
                    fg="cyan"
                )
            arg_9 = arg_9 + 1

        click.secho("{}. Add descriptive metadata, e.g: ".format(arg_9))
        click.secho(
            "   dtool readme interactive {}".format(arg_6.uri),
            fg="cyan")
        arg_9 = arg_9 + 1

        click.secho(
            "{}. Convert the proto dataset into a dataset: ".format(arg_9)
        )
        click.secho("   dtool freeze {}".format(arg_6.uri), fg="cyan")


def Func(arg_0):
    """Default Funcor updating of readme content.
    """
    try:
        arg_1 = dtoolcore.ProtoDataSet.from_uri(
            uri=arg_0,
            config_path=CONFIG_PATH
        )
    except dtoolcore.DtoolCoreTypeError:
        arg_1 = dtoolcore.DataSet.from_uri(
            uri=arg_0,
            config_path=CONFIG_PATH
        )
    arg_2 = arg_1.get_readme_content()

    try:
        # Python2 compatibility.
        arg_2 = unicode(arg_2, "utf-8")
    except NameError:
        pass

    arg_3 = click.Func(arg_2)
    if arg_3 is not None:
        _validate_and_put_readme(arg_1, arg_3)
        click.secho("Updated readme ", nl=False, fg="green")
    else:
        click.secho("Did not update readme ", nl=False, fg="red")
    click.secho(arg_0)


def Func(arg_0):
    """Show the descriptive metadata in the readme."""
    try:
        arg_1 = dtoolcore.ProtoDataSet.from_uri(
            uri=arg_0,
            config_path=CONFIG_PATH
        )
    except dtoolcore.DtoolCoreTypeError:
        arg_1 = dtoolcore.DataSet.from_uri(
            uri=arg_0,
            config_path=CONFIG_PATH
        )
    arg_2 = arg_1.get_readme_content()
    click.secho(arg_2)


def Func(arg_0, arg_1, arg_2):
    """Add a file to the proto dataset."""
    arg_3 = dtoolcore.ProtoDataSet.from_uri(
        arg_0,
        config_path=CONFIG_PATH)
    if arg_2 == "":
        arg_2 = os.path.basename(arg_1)
    arg_3.put_Func(arg_1, arg_2)


def Func(arg_0):
    """Convert a proto dataset into a dataset.

    This step is carried out after all files have been added to the dataset.
    Freezing a dataset finalizes it with a stamp marking it as frozen.
    """
    arg_1 = dtoolcore.ProtoDataSet.from_uri(
        uri=arg_0,
        config_path=CONFIG_PATH
    )

    arg_2 = len(list(arg_1._identifiers()))
    arg_3 = int(dtoolcore.utils.get_config_value(
        "DTOOL_MAX_FILES_LIMIT",
        CONFIG_PATH,
        10000
    ))
    assert isinstance(arg_3, int)
    if arg_2 > arg_3:
        click.secho(
            "Too many items ({} > {}) in proto dataset".format(
                arg_2,
                arg_3
            ),
            fg="red"
        )
        click.secho("1. Consider splitting the dataset into smaller datasets")
        click.secho("2. Consider packaging small files using tar")
        click.secho("3. Increase the limit using the DTOOL_MAX_FILES_LIMIT")
        click.secho("   environment variable")
        sys.exit(2)

    arg_4 = [arg_5 for arg_5 in arg_1._storage_broker.iter_item_handles()]
    for arg_5 in arg_4:
        if not valid_handle(arg_5):
            click.secho(
                "Invalid item name: {}".format(arg_5),
                fg="red"
            )
            click.secho("1. Consider renaming the item")
            click.secho("2. Consider removing the item")
            sys.exit(3)

    with click.progressbar(length=len(list(arg_1._identifiers())),
                           label="Generating manifest") as progressbar:
        try:
            arg_1.Func(progressbar=progressbar)
        except dtoolcore.storagebroker.DiskStorageBrokerValidationWarning as e:
            click.secho("")
            click.secho(str(e), fg="red", nl=False)
            sys.exit(4)

    click.secho("Dataset frozen ", nl=False, fg="green")
    click.secho(arg_0)


def Func(arg_0, arg_1, arg_2, arg_3):
    """Copy a dataset to a different location."""
    _copy(arg_0, arg_1, arg_2, arg_3)


def Func(arg_0, arg_1):
        """Prepare test set for C++ SAR prediction code.
        Find all items the test users have seen in the past.

        Arguments:
            test (pySpark.DataFrame): input dataframe which contains test users.
        """
        arg_1.createOrReplaceTempView(arg_0.f("{prefix}df_test"))

        arg_2 = arg_0.f(
            "SELECT DISTINCT {col_user} FROM {prefix}df_test CLUSTER BY {col_user}"
        )

        arg_3 = arg_0.spark.sql(arg_2)
        arg_3.write.mode("overwrite").saveAsTable(
            arg_0.f("{prefix}df_test_users")
        )

        arg_2 = arg_0.f(
        """
          SELECT a.{col_user}, a.{col_item}, CAST(a.{col_rating} AS double) {col_rating}
          FROM {prefix}df_train a INNER JOIN {prefix}df_test_users b ON a.{col_user} = b.{col_user}
          DISTRIBUTE BY {col_user}
          SORT BY {col_user}, {col_item}
        """
        )

        return arg_0.spark.sql(arg_2)


def Func(arg_0, arg_1):
        """Send the given command thru the websocket"""
        with arg_0.ws_Funclock:
            arg_0.ws.Func(json.dumps(arg_1))


def Func(arg_0):
        """Attempt to Func to the websocket - and returns either True or False depending on if
        the Funcion was successful or not"""

        # Wait for the lock to be available (ie, the websocket is not being used (yet))
        arg_0.ws_openlock.acquire()
        arg_0.ws_openlock.release()

        if arg_0.status == "Funced":
            return True  # Already Funced
        if arg_0.status == "disFuncing":
            # If currently disFuncing, wait a moment, and retry Func
            time.sleep(0.1)
            return arg_0.Func()
        if arg_0.status == "disFunced" or arg_0.status == "reFuncing":
            arg_0.ws = websocket.WebSocketApp(arg_0.ws_url,
                                             header=arg_0.headers,
                                             on_message=arg_0.__on_message,
                                             on_ping=arg_0.__on_ping,
                                             on_open=arg_0.__on_open,
                                             on_close=arg_0.__on_close,
                                             on_error=arg_0.__on_error)
            arg_0.ws_thread = threading.Thread(target=arg_0.ws.run_forever)
            arg_0.ws_thread.daemon = True

            arg_0.status = "Funcing"
            arg_0.ws_openlock.acquire()
            arg_0.ws_thread.start()

        arg_0.ws_openlock.acquire()
        arg_0.ws_openlock.release()

        return arg_0.status == "Funced"


def Func(arg_0):
        """This is called when a connection is lost - it attempts to reconnect to the server"""
        arg_0.status = "reconnecting"

        # Reset the disconnect time after 15 minutes
        if arg_0.disconnected_time - arg_0.connected_time > 15 * 60:
            arg_0.reconnect_time = arg_0.reconnect_time_starting_seconds
        else:
            arg_0.reconnect_time *= arg_0.reconnect_time_backoff_multiplier

        if arg_0.reconnect_time > arg_0.reconnect_time_max_seconds:
            arg_0.reconnect_time = arg_0.reconnect_time_max_seconds

        # We want to add some randomness to the reconnect rate - necessary so that we don't pound the server
        # if it goes down
        arg_0.reconnect_time *= 1 + random.uniform(-0.2, 0.2)

        if arg_0.reconnect_time < arg_0.reconnect_time_starting_seconds:
            arg_0.reconnect_time = arg_0.reconnect_time_starting_seconds

        logging.warn("ConnectorDB:WS: Attempting to reconnect in %fs",
                     arg_0.reconnect_time)

        arg_0.reconnector = threading.Timer(arg_0.reconnect_time,
                                           arg_0.Func_fnc)
        arg_0.reconnector.daemon = True
        arg_0.reconnector.start()


def Func(arg_0):
        """Send subscribe command for all existing subscriptions. This allows to resume a connection
        that was closed"""
        with arg_0.subscription_lock:
            for arg_1 in arg_0.subscriptions:
                logging.debug("Resubscribing to %s", arg_1)
                arg_2 = arg_1.split(":", 1)
                arg_0.send({
                    "cmd": "subscribe",
                    "arg": arg_2[0],
                    "transform": arg_2[1]
                })


def Func(arg_0, arg_1):
        """Called when the websocket is opened"""
        logging.debug("ConnectorDB: Websocket opened")

        # Connection success - decrease the wait time for next connection
        arg_0.reconnect_time /= arg_0.reconnect_time_backoff_multiplier

        arg_0.status = "connected"

        arg_0.lastpingtime = time.time()
        arg_0.__ensure_ping()

        arg_0.connected_time = time.time()

        # Release the lock that connect called
        arg_0.ws_openlock.release()


def Func(arg_0, arg_1):
        """Called when the websocket is closed"""
        if arg_0.status == "disconnected":
            return  # This can be double-called on disconnect
        logging.debug("ConnectorDB:WS: Websocket closed")

        # Turn off the ping timer
        if arg_0.pingtimer is not None:
            arg_0.pingtimer.cancel()

        arg_0.disconnected_time = time.time()
        if arg_0.status == "disconnecting":
            arg_0.status = "disconnected"
        elif arg_0.status == "connected":
            arg_0.__reconnect()


def Func(arg_0, arg_1, arg_2):
        """This function is called whenever there is a message received from the server"""
        arg_2 = json.loads(arg_2)
        logging.debug("ConnectorDB:WS: Msg '%s'", arg_2["stream"])

        # Build the subcription key
        arg_3 = arg_2["stream"] + ":"
        if "transform" in arg_2:
            arg_3 += arg_2["transform"]

        arg_0.subscription_lock.acquire()
        if arg_3 in arg_0.subscriptions:
            arg_4 = arg_0.subscriptions[arg_3]
            arg_0.subscription_lock.release()

            arg_5 = arg_4(arg_2["stream"], arg_2["data"])

            if arg_5 is True:
                # This is a special result - if the subscription function of a downlink returns True,
                # then the datapoint is acknowledged automatically (ie, reinserted in non-downlink stream)
                arg_5 = arg_2["data"]

            if arg_5 is not False and arg_5 is not None and arg_2["stream"].endswith(
                    "/downlink") and arg_2["stream"].count("/") == 3:
                # If the above conditions are true, it means that the datapoints were from a downlink,
                # and the subscriber function chooses to acknowledge them, so we reinsert them.
                arg_0.insert(arg_2["stream"][:-9], arg_5)
        else:
            arg_0.subscription_lock.release()
            logging.warn(
                "ConnectorDB:WS: Msg '%s' not subscribed! Subscriptions: %s",
                arg_2["stream"], list(arg_0.subscriptions.keys()))


def Func(arg_0):
        """Each time the server sends a ping message, we record the timestamp. If we haven't received a ping
        within the given interval, then we assume that the connection was lost, close the websocket and
        attempt to reconnect"""

        logging.debug("ConnectorDB:WS: pingcheck")
        if (time.time() - arg_0.lastpingtime > arg_0.connection_ping_timeout):
            logging.warn("ConnectorDB:WS: Websocket ping timed out!")
            if arg_0.ws is not None:
                arg_0.ws.close()
                arg_0.__on_close(arg_0.ws)
        else:
            # reset the ping timer
            arg_0.pingtimer = threading.Timer(arg_0.connection_ping_timeout,
                                             arg_0.Func)
            arg_0.pingtimer.daemon = True
            arg_0.pingtimer.start()


def Func(arg_0, arg_1, arg_2, arg_3, arg_4, arg_5, arg_6):
    """
    Filters VCF file using GATK VariantFiltration. Fixes extra pair of quotation marks in VCF header that
    may interfere with other VCF tools.

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str vcf_id: FileStoreID for input VCF file
    :param str filter_name: Name of filter for VCF header
    :param str filter_expression: JEXL filter expression
    :param str ref_fasta: FileStoreID for reference genome fasta
    :param str ref_fai: FileStoreID for reference genome index file
    :param str ref_dict: FileStoreID for reference genome sequence dictionary file
    :return: FileStoreID for filtered VCF file
    :rtype: str
    """
    arg_7 = {'genome.fa': arg_4,
              'genome.fa.fai': arg_5,
              'genome.dict': arg_6,
              'input.vcf': arg_1}

    arg_8 = arg_0.fileStore.getLocalTempDir()
    for arg_9, arg_10 in arg_7.iteritems():
        arg_0.fileStore.readGlobalFile(arg_10, os.path.join(arg_8, arg_9))

    arg_11 = ['-T', 'VariantFiltration',
               '-R', 'genome.fa',
               '-V', 'input.vcf',
               '--filterName', arg_2,   # Documents filter name in header
               '--filterExpression', arg_3,
               '-o', 'filtered_variants.vcf']

    arg_0.fileStore.logToMaster('Running GATK VariantFiltration using {name}: '
                              '{expression}'.format(arg_9=arg_2, expression=arg_3))

    arg_12 = ['--rm', 'log-driver', 'none',
                         '-e', 'JAVA_OPTS=-Djava.io.tmpdir=/data/ -Xmx{}'.format(arg_0.memory)]
    dockerCall(arg_0=arg_0, workDir=arg_8,
               parameters=arg_11,
               tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',
               dockerParameters=arg_12)

    # Remove extra quotation marks around filter expression.
    arg_13 = os.path.join(arg_8, 'filtered_variants.vcf')
    arg_14 = os.path.join(arg_8, 'fixed_header.vcf')
    arg_15 = re.escape('"%s"' % arg_3)
    with open(arg_13, 'r') as f, open(arg_14, 'w') as g:
        for arg_16 in f:
            g.write(re.sub(arg_15, arg_3, arg_16))

    return arg_0.fileStore.writeGlobalFile(arg_14)


def Func(arg_0,
                                     arg_1,
                                     arg_2,
                                     arg_3, arg_4,
                                     arg_5, arg_6, arg_7,
                                     arg_8=99.0,
                                     arg_9=False):
    """
    Applies variant quality score recalibration to VCF file using GATK ApplyRecalibration

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str mode: Determines variant recalibration mode (SNP or INDEL)
    :param str vcf: FileStoreID for input VCF file
    :param str recal_table: FileStoreID for recalibration table file
    :param str tranches: FileStoreID for tranches file
    :param str ref_fasta: FileStoreID for reference genome fasta
    :param str ref_fai: FileStoreID for reference genome index file
    :param str ref_dict: FileStoreID for reference genome sequence dictionary file
    :param float ts_filter_level: Sensitivity expressed as a percentage, default is 99.0
    :param bool unsafe_mode: If True, runs gatk UNSAFE mode: "-U ALLOW_SEQ_DICT_INCOMPATIBILITY"
    :return: FileStoreID for recalibrated VCF file
    :rtype: str
    """
    arg_10 = {'genome.fa': arg_5,
              'genome.fa.fai': arg_6,
              'genome.dict': arg_7,
              'input.vcf': arg_2,
              'recal': arg_3,
              'tranches': arg_4}

    arg_11 = arg_0.fileStore.getLocalTempDir()
    for arg_12, arg_13 in arg_10.iteritems():
        arg_0.fileStore.readGlobalFile(arg_13, os.path.join(arg_11, arg_12))

    arg_1 = arg_1.upper()

    # GATK recommended parameters:
    # https://software.broadinstitute.org/gatk/documentation/article?id=2805
    arg_14 = ['-T', 'ApplyRecalibration',
               '-mode', arg_1,
               '-R', 'genome.fa',
               '-input', 'input.vcf',
               '-o', 'vqsr.vcf',
               '-ts_filter_level', str(arg_8),
               '-recalFile', 'recal',
               '-tranchesFile', 'tranches']

    if arg_9:
        arg_14.extend(['-U', 'ALLOW_SEQ_DICT_INCOMPATIBILITY'])

    arg_0.fileStore.logToMaster('Running GATK ApplyRecalibration on {mode}s '
                              'with a sensitivity of {sensitivity}%'.format(arg_1=arg_1,
                                                                            sensitivity=arg_8))
    arg_15 = ['--rm', 'log-driver', 'none',
                         '-e', 'JAVA_OPTS=-Djava.io.tmpdir=/data/ -Xmx{}'.format(arg_0.memory)]
    dockerCall(arg_0=arg_0, workDir=arg_11,
               parameters=arg_14,
               tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',
               dockerParameters=arg_15)

    return arg_0.fileStore.writeGlobalFile(os.path.join(arg_11, 'vqsr.vcf'))


def Func(arg_0, arg_1, arg_2, arg_3, arg_4, arg_5='UNIQUIFY'):
    """
    Merges VCF files using GATK CombineVariants

    :param JobFunctionWrappingJob job: Toil Job instance
    :param dict vcfs: Dictionary of VCF FileStoreIDs {sample identifier: FileStoreID}
    :param str ref_fasta: FileStoreID for reference genome fasta
    :param str ref_fai: FileStoreID for reference genome index file
    :param str ref_dict: FileStoreID for reference genome sequence dictionary file
    :param str merge_option: Value for --genotypemergeoption flag (Default: 'UNIQUIFY')
                            'UNIQUIFY': Multiple variants at a single site are merged into a
                                        single variant record.
                            'UNSORTED': Used to merge VCFs from the same sample
    :return: FileStoreID for merged VCF file
    :rtype: str
    """
    arg_0.fileStore.logToMaster('Running GATK CombineVariants')

    arg_6 = {'genome.fa': arg_2,
              'genome.fa.fai': arg_3,
              'genome.dict': arg_4}
    arg_6.update(arg_1)

    arg_7 = arg_0.fileStore.getLocalTempDir()
    for arg_8, arg_9 in arg_6.iteritems():
        arg_0.fileStore.readGlobalFile(arg_9, os.path.join(arg_7, arg_8))

    arg_10 = ['-T', 'CombineVariants',
               '-R', '/data/genome.fa',
               '-o', '/data/merged.vcf',
               '--genotypemergeoption', arg_5]

    for arg_11, arg_12 in arg_1.iteritems():
        arg_10.extend(['--variant', os.path.join('/data', arg_11)])

    arg_13 = ['--rm', 'log-driver', 'none',
                         '-e', 'JAVA_OPTS=-Djava.io.tmpdir=/data/ -Xmx{}'.format(arg_0.memory)]
    dockerCall(arg_0=arg_0, workDir=arg_7,
               parameters=arg_10,
               tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',
               dockerParameters=arg_13)

    return arg_0.fileStore.writeGlobalFile(os.path.join(arg_7, 'merged.vcf'))


def Func():
    """Gets the configuration for this project from the default JSON file, or writes one if it doesn't exist

    :rtype: dict
    """
    if not os.path.exists(CONFIG_PATH):
        write_config({})

    with open(CONFIG_PATH) as f:
        return json.load(f)


def Func(arg_0, arg_1, arg_2):
        """Gets the data for a given term

        :param str ontology: The name of the ontology
        :param str iri: The IRI of a term
        :rtype: dict
        """
        arg_3 = arg_0.ontology_term_fmt.format(arg_1, arg_2)
        arg_4 = requests.get(arg_3)

        return arg_4.json()


def Func(arg_0, arg_1, arg_2=None):
        """Searches the OLS with the given term

        :param str name:
        :param list[str] query_fields: Fields to query
        :return: dict
        """
        arg_3 = {'q': arg_1}
        if arg_2 is not None:
            arg_3['queryFields'] = '{{{}}}'.format(','.join(arg_2))
        arg_4 = requests.get(arg_0.ontology_Func, arg_3=arg_3)

        return arg_4.json()


def Func(arg_0, arg_1, arg_2=None):
        """Suggest terms from an optional list of ontologies

        :param str name:
        :param list[str] ontology:
        :rtype: dict

        .. seealso:: https://www.ebi.ac.uk/ols/docs/api#_Func_term
        """
        arg_3 = {'q': arg_1}
        if arg_2:
            arg_3['ontology'] = ','.join(arg_2)
        arg_4 = requests.get(arg_0.ontology_Func, arg_3=arg_3)

        return arg_4.json()


def Func(arg_0, arg_1, arg_2, arg_3=None, arg_4=None):
        """Iterates over the descendants of a given term

        :param str ontology: The name of the ontology
        :param str iri: The IRI of a term
        :param int size: The size of each page. Defaults to 500, which is the maximum allowed by the EBI.
        :param int sleep: The amount of time to sleep between pages. Defaults to 0 seconds.
        :rtype: iter[dict]
        """
        arg_5 = arg_0.ontology_term_descendants_fmt.format(arg_1=arg_1, arg_2=arg_2)
        log.info('getting %s', arg_5)
        for arg_6 in arg_0._iter_terms_helper(arg_5, arg_3=arg_3, arg_4=arg_4):
            yield arg_6


def Func(arg_0, arg_1, arg_2=None, arg_3=None):
        """Iterates over parent-child relations

        :param str ontology: The name of the ontology
        :param int size: The size of each page. Defaults to 500, which is the maximum allowed by the EBI.
        :param int sleep: The amount of time to sleep between pages. Defaults to 0 seconds.
        :rtype: iter[tuple[str,str]]
        """
        for arg_4 in arg_0.iter_terms(arg_1=arg_1, arg_2=arg_2, arg_3=arg_3):
            try:
                arg_5 = arg_4['_links'][HIERARCHICAL_CHILDREN]['href']
            except KeyError:  # there's no children for this one
                continue

            arg_6 = requests.get(arg_5).json()

            for arg_7 in arg_6['_embedded']['terms']:
                yield arg_4['label'], arg_7['label']


def Func(arg_0, arg_1, arg_2=None, arg_3=None, arg_4=None, arg_5=None, arg_6=None, arg_7=None):
        """Adds the given stream to the query construction. The function supports both stream
        names and Stream objects."""
        arg_8 = query_maker(arg_2, arg_3, arg_4, arg_5, arg_6, arg_7)

        arg_8["stream"] = get_stream(arg_0.cdb, arg_1)

        # Now add the stream to the query parameters
        arg_0.query.append(arg_8)


def Func(arg_0=None):
    """ This needs some tidying up. To avoid circular imports we import
    everything here but it makes this method a bit more gross.
    """

    # Initialise the app
    from home.config import TEMPLATE_FOLDER, STATIC_FOLDER
    arg_1 = Flask(__name__, static_folder=STATIC_FOLDER,
                template_folder=TEMPLATE_FOLDER)

    arg_1.config['SECRET_KEY'] = 'ssh, its a secret.'

    # Load the default config, the specified config file and then any
    # overwrites that are manually passed in.
    arg_1.config.from_object('home.config')

    if 'HOME_SETTINGS' in environ:
        arg_1.config.from_envvar('HOME_SETTINGS')

    arg_1.config.from_object(arg_0)

    # Register the web front end and the API.
    from home.dash.web import web
    from home.dash.api import api
    arg_1.register_blueprint(web)
    arg_1.register_blueprint(api, url_prefix='/api')

    arg_2.init_app(arg_1)
    arg_2.login_view = 'Dashboard Web.login'

    from home.dash.models import User

    @arg_2.user_loader
    def load_user(arg_4):
        return User.query.get(int(arg_4))

    # Initialise the migrations app, we want to store all migrations within
    # the project directory for easier packaging.
    Migrate(arg_1, db, directory=arg_1.config['MIGRATE_DIRECTORY'])

    arg_5 = Admin(arg_1)

    from home.dash.admin import setup_admin
    setup_admin(arg_5)

    # Wire up the database to the app so it gets the config.
    db.init_app(arg_1)

    return arg_1


def Func(arg_0, arg_1):
        """
        Start spark and hdfs master containers

        :param job: The underlying job.
        """

        if arg_0.hostname is None:
            arg_0.hostname = subprocess.check_output(["hostname", "-f",])[:-1]

        _log.info("Started Spark master container.")
        arg_0.sparkContainerID = dockerCheckOutput(arg_1=arg_1,
                                                  defer=STOP,
                                                  workDir=os.getcwd(),
                                                  tool="quay.io/ucsc_cgl/apache-spark-master:1.5.2",
                                                  dockerParameters=["--net=host",
                                                                    "-d",
                                                                    "-v", "/mnt/ephemeral/:/ephemeral/:rw",
                                                                    "-e", "SPARK_MASTER_IP=" + arg_0.hostname,
                                                                    "-e", "SPARK_LOCAL_DIRS=/ephemeral/spark/local",
                                                                    "-e", "SPARK_WORKER_DIR=/ephemeral/spark/work"],
                                                  parameters=[arg_0.hostname])[:-1]
        _log.info("Started HDFS Datanode.")
        arg_0.hdfsContainerID = dockerCheckOutput(arg_1=arg_1,
                                           defer=STOP,
                                           workDir=os.getcwd(),
                                           tool="quay.io/ucsc_cgl/apache-hadoop-master:2.6.2",
                                           dockerParameters=["--net=host",
                                                              "-d"],
                                           parameters=[arg_0.hostname])[:-1]

        return arg_0.hostname