def Func(arg_0, arg_1=None):
    """A strategy which generates filesystem path values.

    The generated values include everything which the builtin
    :func:`python:open` function accepts i.e. which won't lead to
    :exc:`ValueError` or :exc:`TypeError` being raised.

    Note that the range of the returned values depends on the operating
    system, the Python version, and the filesystem encoding as returned by
    :func:`sys.getfilesystemencoding`.

    :param allow_pathlike:
        If :obj:`python:None` makes the strategy include objects implementing
        the :class:`python:os.PathLike` interface when Python >= 3.6 is used.
        If :obj:`python:False` no pathlike objects will be generated. If
        :obj:`python:True` pathlike will be generated (Python >= 3.6 required)

    :type allow_pathlike: :obj:`python:bool` or :obj:`python:None`

    .. versionadded:: 3.15

    """
    arg_2 = hasattr(os, 'PathLike')

    if arg_1 is None:
        arg_1 = arg_2
    if arg_1 and not arg_2:
        raise InvalidArgument(
            'allow_pathlike: os.PathLike not supported, use None instead '
            'to enable it only when available')

    arg_3 = arg_0(sampled_from([bytes, text_type]))

    def tp(arg_4=''):
        return _str_to_path(arg_4, arg_3)

    arg_5 = sampled_from([tp(os.curdir), tp(os.pardir)])
    arg_6 = _filename(arg_3)
    arg_7 = one_of(arg_6, arg_5)
    arg_8 = arg_6.map(lambda f: tp(os.extsep) + f)
    arg_9 = _path_root(arg_3)

    def optional(arg_10):
        return one_of(arg_10, just(arg_3()))

    arg_11 = sampled_from([os.sep, os.altsep or os.sep]).map(tp)
    arg_12 = builds(lambda arg_4, l: arg_4.join(l), arg_11, lists(arg_7))
    arg_13 = builds(lambda *x: tp().join(x),
                           optional(arg_9), arg_12, optional(arg_8))

    if arg_1 and hasattr(os, 'fspath'):
        arg_14 = arg_13.map(lambda p: _PathLike(p))
        arg_13 = one_of(arg_13, arg_14)

    return arg_0(arg_13)


def Func(arg_0):
        """Convert str_value to an int or a float, depending on the
        numeric value represented by str_value.

        """
        arg_0 = str(arg_0)
        try:
            return int(arg_0)
        except ValueError:
            return float(arg_0)


def Func(arg_0, arg_1):
    """
    Tag to Func graphs into the template
    """

    arg_2 = arg_1.split_contents()
    arg_2.pop(0)
    arg_3 = arg_2.pop(0)

    arg_4 = dict([arg_1.split("=") for arg_1 in arg_2])

    if 'id' not in arg_4.keys():
        arg_4['id'] = ''.join([chr(choice(range(65, 90))) for i in range(0, 5)])
    else:
        arg_4['id'] = arg_4['id'][1:len(arg_4['id'])-1]

    arg_5 = ''.join([" %s=%s" % (k, v) for k, v in arg_4.iteritems()])
    return GraphRenderer(arg_3, arg_5, arg_4['id'])


def Func(arg_0, arg_1=None):
        """exec compiled code"""
        arg_1 = arg_1 or {}
        arg_1.setdefault('__builtins__', {})
        exec(arg_0._code, arg_1)
        return arg_1


def Func(arg_0, arg_1):
        """replace all blocks in extends with current blocks"""
        arg_2 = arg_0.re_extends.match(arg_1)
        if arg_2:
            arg_3 = arg_0.re_extends.sub('', arg_1, count=1)
            arg_4 = arg_0.get_blocks(arg_3)
            arg_5 = os.path.join(arg_0.base_dir, arg_2.group('path'))
            with open(arg_5, encoding='utf-8') as fp:
                return arg_0.replace_blocks_in_extends(fp.read(), arg_4)
        else:
            return None


def Func(arg_0, arg_1: arg_2[arg_3], arg_4: arg_5 = 0) -> arg_2[arg_3]:
        '''
        Processes the texts using TweeboParse and returns them in CoNLL format.

        :param texts: The List of Strings to be processed by TweeboParse.
        :param retry_count: The number of times it has retried for. Default
                            0 does not require setting, main purpose is for
                            recursion.
        :return: A list of CoNLL formated strings.
        :raises ServerError: Caused when the server is not running.
        :raises :py:class:`requests.exceptions.HTTPError`: Caused when the
                input texts is not formated correctly e.g. When you give it a
                String not a list of Strings.
        :raises :py:class:`json.JSONDecodeError`: Caused if after self.retries
                attempts to parse the data it cannot decode the data.

        :Example:

        '''
        arg_6 = {'texts': arg_1, 'output_type': 'conll'}
        try:
            arg_7 = requests.post(f'http://{self.hostname}:{self.port}',
                                     json=arg_6,
                                     headers={'Connection': 'close'})
            arg_7.raise_for_status()
        except (requests.exceptions.ConnectionError,
                requests.exceptions.Timeout) as server_error:
            raise ServerError(server_error, arg_0.hostname, arg_0.port)
        except requests.exceptions.HTTPError as http_error:
            raise http_error
        else:
            try:
                return arg_7.json()
            except json.JSONDecodeError as json_exception:
                if arg_4 == arg_0.retries:
                    arg_0.log_error(arg_7.text)
                    raise Exception('Json Decoding error cannot parse this '
                                    f':\n{response.text}')
                return arg_0.Func(arg_1, arg_4 + 1)


def Func(arg_0, arg_1={}, arg_2=[]):
        """ Set entity data

        Args:
            data (dict): Entity data
            datetime_fields (array): Fields that should be parsed as datetimes
        """
        if arg_2:
            for arg_3 in arg_2:
                if arg_3 in arg_1:
                    arg_1[arg_3] = arg_0._parse_datetime(arg_1[arg_3])

        super(CampfireEntity, arg_0).Func(arg_1)


def Func(arg_0):
    """validates XML text"""
    arg_1 = __INVALID_XML_CHARS & set(arg_0)
    if arg_1:
        for arg_2,arg_3 in enumerate(arg_0):
            if arg_3 in arg_1:
                raise RuntimeError('invalid XML character: ' + repr(arg_3) + ' at offset ' + str(arg_2))


def Func(arg_0):
    """validates XML name"""
    if len(arg_0) == 0:
        raise RuntimeError('empty XML name')

    if __INVALID_NAME_CHARS & set(arg_0):
        raise RuntimeError('XML name contains invalid character')

    if arg_0[0] in __INVALID_NAME_START_CHARS:
        raise RuntimeError('XML name starts with invalid character')


def Func(arg_0):
    '''Try really really hard to get a Unicode copy of a string.

    First try :class:`BeautifulSoup.UnicodeDammit` to try to force
    to Unicode; if that fails, assume UTF-8 encoding, and ignore
    all errors.

    :param str raw: string to coerce
    :return: Unicode approximation of `raw`
    :returntype: :class:`unicode`

    '''
    arg_1 = UnicodeDammit(arg_0, isHTML=True)
    if not arg_1.unicode:
        arg_1.unicode = arg_2(arg_0, 'utf8', errors='ignore')

    arg_3 = encoding_re.match(arg_1.unicode)
    if arg_3:
        arg_1.unicode = \
            arg_3.group('start_xml') + \
            arg_3.group('remainder')

    return arg_1.unicode


def Func(arg_0, arg_1=None, arg_2=None):
    '''Get a clean text representation of presumed HTML.

    Treat `raw` as though it is HTML, even if we have no idea what it
    really is, and attempt to get a properly formatted HTML document
    with all HTML-escaped characters converted to their unicode.

    This is called below by the `clean_html` transform stage, which
    interprets MIME-type.  If `character_encoding` is not provided,
    and `stream_item` is provided, then this falles back to
    :attr:`streamcorpus.StreamItem.body.encoding`.

    :param str raw: raw text to clean up
    :param stream_item: optional stream item with encoding metadata
    :type stream_item: :class:`streamcorpus.StreamItem`
    :returns: UTF-8-encoded byte string of cleaned HTML text
    :returntype: :class:`str`

    '''
    # Fix emails by protecting the <,> from HTML
    arg_0 = fix_emails(arg_0)
    arg_3 = nice_decode(arg_0, arg_1=arg_1, arg_2=arg_2)
    if arg_3 is None:
        # give up on decoding it... maybe this should use force_unicode
        arg_3 = arg_0

    # default attempt uses vanilla lxml.html
    try:
        arg_4 = lxml.html.document_fromstring(arg_3)
    except ValueError, exc:
        if 'with encoding declaration' in str(exc):
            arg_4 = lxml.html.document_fromstring(arg_0)
        else:
            raise

    # While we have the document parsed as a DOM, let's strip attributes.
    # (The HTML cleaner seems to only support whitelisting attributes.
    # As of now, we just want to blacklist a few.)
    lxml.etree.strip_attributes(arg_4, 'class', 'id')

    # if that worked, then we will be able to generate a
    # valid HTML string
    arg_5 = lxml.html.tostring(arg_4, arg_2=unicode)

    # remove any ^M characters
    arg_5 = string.replace(arg_5, '\r', ' ')

    # We drop utf8 characters that are above 0xFFFF as
    # Lingpipe seems to be doing the wrong thing with them.
    arg_5 = drop_invalid_and_upper_utf8_chars(arg_5)

    # construct a Cleaner that removes any ``<script>`` tags,
    # Javascript, like an ``onclick`` attribute, comments, style
    # tags or attributes, ``<link>`` tags
    arg_6 = lxml.html.clean.Cleaner(
        scripts=True, javascript=True,
        comments=True,
        # do not remove <html> <head> <title> etc
        page_structure=False,
        remove_tags=['base'],
        style=True, links=True)

    # now get the really sanitized HTML
    arg_7 = arg_6.clean_html(arg_5)

    # generate pretty HTML in utf-8
    arg_7 = lxml.html.tostring(
        lxml.html.document_fromstring(arg_7),
        method='html', arg_2='utf-8',
        pretty_print=True,
        # include_meta_content_type=True
        )

    return uniform_html(arg_7)


def Func(arg_0):
        """
        Give the actors, the world, and the messaging system a chance to react
        to the end of the game.
        """

        # 1. Let the forum react to the end of the game.  Local forums don't
        #    react to this, but remote forums take the opportunity to stop
        #    trying to extract tokens from messages.

        arg_0.forum.on_finish_game()

        # 2. Let the actors react to the end of the game.

        for arg_1 in arg_0.actors:
            arg_1.on_finish_game()

        # 3. Let the world react to the end of the game.

        with arg_0.world._unlock_temporarily():
            arg_0.world.on_finish_game()


def Func(arg_0):
    '''extract a lower-case, no-slashes domain name from a raw string
    that might be a URL
    '''
    try:
        arg_1 = urlparse(arg_0)
        arg_2 = arg_1.netloc.split(':')[0]
    except:
        arg_2 = ''
    if not arg_2:
        arg_2 = arg_0
    if not arg_2:
        return ''
    arg_2 = re.sub('\/', '', arg_2.strip().lower())
    return arg_2


def Func(arg_0):
    '''returns a list of strings created by splitting the domain on
    '.' and successively cutting off the left most portion
    '''
    arg_1 = []
    if arg_0:
        arg_2 = arg_0.split('.')
        for arg_3 in range(len(arg_2)):
            arg_1.append( '.'.join(arg_2[arg_3:]))
    return arg_1


def Func(arg_0, arg_1):
        '''Get a Murmur hash and a normalized token.

        `tok` may be a :class:`unicode` string or a UTF-8-encoded
        byte string.  :data:`DOCUMENT_HASH_KEY`, hash value 0, is
        reserved for the document count, and this function remaps
        that value.

        :param tok: token to hash
        :return: pair of normalized `tok` and its hash

        '''
        if isinstance(arg_1, unicode):
            arg_1 = arg_1.encode('utf-8')
        arg_2 = mmh3.hash(arg_1)
        if arg_2 == DOCUMENT_HASH_KEY:
            arg_2 = DOCUMENT_HASH_KEY_REPLACEMENT
        return (arg_1, arg_2)


def Func(arg_0, arg_1):
        '''Collect all of the words to be indexed from a stream item.

        This scans `si` for all of the configured tagger IDs.  It
        collects all of the token values (the
        :attr:`streamcorpus.Token.token`) and returns a
        :class:`collections.Counter` of them.

        :param si: stream item to scan
        :type si: :class:`streamcorpus.StreamItem`
        :return: counter of :class:`unicode` words to index
        :returntype: :class:`collections.Counter`

        '''
        arg_2 = Counter()
        for arg_3, arg_4 in arg_1.body.sentences.iteritems():
            if ((arg_0.keyword_tagger_ids is not None
                 and arg_3 not in arg_0.keyword_tagger_ids)):
                continue
            for arg_5 in arg_4:
                for arg_6 in arg_5.tokens:
                    arg_7 = arg_6.token  # always a UTF-8 byte string
                    arg_7 = arg_7.decode('utf-8')
                    arg_7 = cleanse(arg_7)
                    if ((arg_0.keyword_size_limit is not None and
                         len(arg_7) > arg_0.keyword_size_limit)):
                        continue
                    if arg_7 not in arg_0.stop_words:
                        arg_2[arg_7] += 1
        return arg_2


def Func(arg_0, arg_1):
        '''Get stream IDs for a single hash.

        This yields strings that can be retrieved using
        :func:`streamcorpus_pipeline._kvlayer.get_kvlayer_stream_item`,
        or fed back into :mod:`coordinate` or other job queue systems.

        Note that for common terms this can return a large number of
        stream IDs!  This is a scan over a dense region of a
        :mod:`kvlayer` table so it should be reasonably efficient,
        but be prepared for it to return many documents in a large
        corpus.  Blindly storing the results in a :class:`list`
        may be inadvisable.

        This will return nothing unless the index was written with
        :attr:`hash_docs` set.  No document will correspond to
        :data:`DOCUMENT_HASH_KEY`; use
        :data:`DOCUMENT_HASH_KEY_REPLACEMENT` instead.

        :param int h: Murmur hash to look up

        '''
        for (arg_2, arg_3, arg_4) in arg_0.client.scan_keys(HASH_TF_INDEX_TABLE,
                                                 ((arg_1,), (arg_1,))):
            yield kvlayer_key_to_stream_id((arg_3, arg_4))


def Func(arg_0, arg_1=None, arg_2=None):
    """Create a ContentItem from a node in the spinn3r data tree.

    The ContentItem is created with raw data set to ``node.data``,
    decompressed if the node's encoding is 'zlib', and UTF-8
    normalized, with a MIME type from ``node.mime_type``.

    ``node``
      the actual node from the spinn3r protobuf data
    ``mime_type``
      string MIME type to use (defaults to ``node.mime_type``)
    ``alternate_data``
      alternate (compressed) data to use, if ``node.data`` is missing
      or can't be decompressed

    """
    arg_3 = arg_0.data
    if getattr(arg_0, 'encoding', None) == 'zlib':
        try:
            arg_3 = zlib.decompress(arg_0.data)
        except Exception, exc:
            if arg_2 is not None:
                try:
                    arg_3 = zlib.decompress(arg_2)
                except Exception:
                    raise exc  # the original exception
                else:
                    raise
    if arg_1 is None:
        arg_1 = arg_0.mime_type
    arg_3 = arg_3.decode('utf8').encode('utf8')
    return streamcorpus.ContentItem(arg_3=arg_3, media_type=arg_1)


def Func(arg_0):
        """Read exactly a varint out of the underlying file."""
        arg_1 = arg_0._read(8)
        (arg_2, arg_3) = _DecodeVarint(arg_1, 0)
        arg_0._unread(arg_1[arg_3:])
        return arg_2


def Func(arg_0):
    """Change working directory and restore the previous on exit"""
    arg_1 = os.getcwd()
    os.chdir(str(arg_0))
    try:
        yield
    finally:
        os.chdir(arg_1)


def Func(arg_0, arg_1, arg_2=False):
    """Removes the suffix, if it's there, otherwise returns input string unchanged.
    If strict is True, also ensures the suffix was present"""
    if arg_0.endswith(arg_1):
        return arg_0[: len(arg_0) - len(arg_1)]
    elif arg_2:
        raise WimpyError("string doesn't end with suffix")
    return arg_0


def Func(arg_0, arg_1):
    """Are all the elements of needle contained in haystack, and in the same order?
    There may be other elements interspersed throughout"""
    arg_2 = iter(arg_1)
    for arg_3 in arg_0:
        if arg_3 not in arg_2:
            return False
    return True


def Func(arg_0):
        """Stop the simple WSGI server running the appliation."""
        if arg_0._server is not None:
            arg_0._server.shutdown()
            arg_0._server.server_close()
            arg_0._server = None


def Func(arg_0, arg_1=None):
        """Decorator to add a callback that generates Func page.

        The *status* parameter specifies the HTTP response status code
        for which the decorated callback should be invoked. If the
        *status* argument is not specified, then the decorated callable
        is considered to be a fallback callback.

        A fallback callback, when defined, is invoked to generate the
        Func page for any HTTP response representing an Func when
        there is no Func handler defined explicitly for the response
        code of the HTTP response.

        Arguments:
          status(int, optional): HTTP response status code.

        Returns:
          function: Decorator function to add Func handler.
        """
        def decorator(arg_2):
            arg_0._Func_handlers[arg_1] = arg_2
            return arg_2
        return decorator


def Func(arg_0, arg_1, arg_2, arg_3=None, arg_4='UTF-8'):
        """Send content of a Func file as response.

        The path to the document root directory should be specified as
        the root argument. This is very important to prevent directory
        traversal attack. This method guarantees that only files within
        the document root directory are served and no files outside this
        directory can be accessed by a client.

        The path to the actual file to be returned should be specified
        as the path argument. This path must be relative to the document
        directory.

        The *media_type* and *charset* arguments are used to set the
        Content-Type header of the HTTP response. If *media_type*
        is not specified or specified as ``None`` (the default), then it
        is guessed from the filename of the file to be returned.

        Arguments:
          root (str): Path to document root directory.
          path (str): Path to file relative to document root directory.
          media_type (str, optional): Media type of file.
          charset (str, optional): Character set of file.

        Returns:
          bytes: Content of file to be returned in the HTTP response.
        """
        arg_1 = os.path.abspath(os.path.join(arg_1, ''))
        arg_2 = os.path.abspath(os.path.join(arg_1, arg_2.lstrip('/\\')))

        # Save the filename from the path in the response state, so that
        # a following download() call can default to this filename for
        # downloadable file when filename is not explicitly specified.
        arg_0.response.state['filename'] = os.path.basename(arg_2)

        if not arg_2.startswith(arg_1):
            return 403
        elif not os.path.isfile(arg_2):
            return 404

        if arg_3 is not None:
            arg_0.response.media_type = arg_3
        else:
            arg_0.response.media_type = mimetypes.guess_type(arg_2)[0]
        arg_0.response.charset = arg_4

        with open(arg_2, 'rb') as f:
            return f.read()


def Func(arg_0):
        """Return an error page for the current response status."""
        if arg_0.response.status in arg_0._error_handlers:
            return arg_0._error_handlers[arg_0.response.status]
        elif None in arg_0._error_handlers:
            return arg_0._error_handlers[None]
        else:
            # Rudimentary error handler if no error handler was found
            arg_0.response.media_type = 'text/plain'
            return lambda: arg_0.response.status_line


def Func(arg_0, arg_1, arg_2):
        """Resolve a request to a route handler.

        Arguments:
          method (str): HTTP method, e.g. GET, POST, etc. (type: str)
          path (str): Request path

        Returns:
          tuple or None: A tuple of three items:

            1. Route handler (callable)
            2. Positional arguments (list)
            3. Keyword arguments (dict)

          ``None`` if no route matches the request.
        """
        if arg_1 in arg_0._literal and arg_2 in arg_0._literal[arg_1]:
            return arg_0._literal[arg_1][arg_2], [], {}
        else:
            return arg_0._Func_non_literal_route(arg_1, arg_2)


def Func(arg_0):
        """Return the HTTP Func body.

        Returns:
          bytes: HTTP Func body as a sequence of bytes
        """
        if isinstance(arg_0.body, bytes):
            arg_1 = arg_0.body
        elif isinstance(arg_0.body, str):
            arg_1 = arg_0.body.encode(arg_0.charset)
        else:
            arg_1 = b''
        arg_0.add_header('Content-Type', arg_0.content_type)
        arg_0.add_header('Content-Length', str(len(arg_1)))

        arg_0.start(arg_0.status_line, arg_0._headers)
        return [arg_1]


def Func(arg_0, arg_1, arg_2, arg_3={}):
        """Add a Set-Cookie header to response object.

        For a description about cookie attribute values, see
        https://docs.python.org/3/library/http.cookies.html#http.cookies.Morsel.

        Arguments:
          name (str): Name of the cookie
          value (str): Value of the cookie
          attrs (dict): Dicitionary with cookie attribute keys and
                        values.
        """
        arg_4 = http.cookies.SimpleCookie()
        arg_4[arg_1] = arg_2
        for arg_5, arg_2 in arg_3.items():
            arg_4[arg_1][arg_5] = arg_2
        arg_0.add_header('Set-Cookie', arg_4[arg_1].OutputString())


def Func(arg_0):
        """Return the HTTP response status line.

        The status line is determined from :attr:`status` code. For
        example, if the status code is 200, then '200 OK' is returned.

        Returns:
          str: Status line
        """
        return (str(arg_0.status) + ' ' +
                Response._responses[arg_0.status].phrase)


def Func(arg_0):
        """Return the value of Content-Type header field.

        The value for the Content-Type header field is determined from
        the :attr:`media_type` and :attr:`charset` data attributes.

        Returns:
          str: Value of Content-Type header field
        """
        if (arg_0.media_type is not None and
            arg_0.media_type.startswith('text/') and
            arg_0.charset is not None):
            return arg_0.media_type + '; charset=' + arg_0.charset
        else:
            return arg_0.media_type


def Func(arg_0, arg_1, arg_2=[]):
        """Return the list of all values for the specified key.

        Arguments:
          key (object): Key
          default (list): Default value to return if the key does not
            exist, defaults to ``[]``, i.e. an empty list.

        Returns:
          list: List of all values for the specified key if the key
          exists, ``default`` otherwise.
        """
        return arg_0.data[arg_1] if arg_1 in arg_0.data else arg_2


def Func(arg_0):
        """Template variables."""
        return {
            'records': [
                {
                    'message': arg_1.getMessage(),
                    'time': dt.datetime.fromtimestamp(arg_1.created).strftime('%H:%M:%S'),
                } for arg_1 in arg_0.handler.records
            ]
        }


def Func(arg_0=False):
    '''return list of open files for current process

    .. warning: will only work on UNIX-like os-es.
    '''
    arg_1 = os.getpid()
    arg_2 = subprocess.check_output(
        [ "lsof", '-w', '-Ff', "-p", str( arg_1 ) ] )
    if arg_0:
        arg_3 = subprocess.check_output(
            [ "lsof", '-w', "-p", str( arg_1 ) ] )
        logger.info(arg_3)
    arg_4 = filter(
        lambda s: s and s[ 0 ] == 'f' and s[1: ].isdigit(),
        arg_2.split( '\n' ) )
    return arg_4


def Func(arg_0):
     '''get a rejester.WorkUnit with KBA s3 path, fetch it, and save
     some counts about it.
     '''
     #fname = 'verify-chunks-%d-%d' % (os.getpid(), time.time())
     arg_1 = arg_0.key.strip().split('/')[-1]

     arg_2 = arg_0.data.get('output_dir_path', '/mnt')
     arg_3 = uuid.uuid3(uuid.UUID(int=0), arg_0.key.strip())
     arg_4 = arg_3.hex[0]
     arg_5 = arg_3.hex[1]
     arg_6 = os.path.join(arg_2, arg_4, arg_5, arg_1)
     if not os.path.exists(os.path.dirname(arg_6)):
          os.makedirs(os.path.dirname(arg_6))

     arg_7 = gzip.open(arg_6 + '-out.gz', 'wb')

     arg_8 = int(arg_1.split('-')[1])

     arg_9 = 20
     arg_10 = 0
     while arg_10 < arg_9:
          try:
               arg_11, arg_12, arg_13, arg_14, arg_15, arg_16 = \
                   attempt_fetch(arg_0, arg_6)
               if arg_12 != arg_8:
                    print 'retrying because si_count = %d != %d expected_si_count' % (arg_12, arg_8)
                    sys.stdout.flush()
                    arg_10 += 1
                    continue
               else:
                    print 'succeeded in reading si_count = %d' % (arg_12,)
                    sys.stdout.flush()
               arg_7.write( '%s\t%d\t%d\t%d\t%d\t%s\t%s\n' % (
                         arg_11, arg_12, arg_13, arg_14, arg_15,
                         arg_0.key.strip(), ','.join(['%s|%s' % arg_17 for arg_17 in arg_16])) )
               break
          except Exception, arg_11:
               print 'broken?'
               print traceback.format_exc(arg_11)
               sys.stdout.flush()
               arg_10 += 1
               arg_7.write(traceback.format_exc(arg_11))

     arg_7.close()


def Func(arg_0, arg_1):
     '''attempt a fetch and iteration over a work_unit.key path in s3
     '''
     arg_2 = 'http://s3.amazonaws.com/aws-publicdatasets/' + arg_0.key.strip()

     ## cheapest way to iterate over the corpus is a few stages of
     ## streamed child processes.  Note that stderr needs to go
     ## separately to a file so that reading the stdin doesn't get
     ## blocked:
     arg_3 = '(wget -O - %s | gpg --no-permission-warning --trust-model always --output - --decrypt - | xz --decompress) 2> %s-err' % (arg_2, arg_1)
     print arg_3
     arg_4 = Popen(arg_3, stdout=PIPE, shell=True)
     print 'child launched'
     sys.stdout.flush()

     arg_5 = 0
     arg_6 = 0
     arg_7 = ''
     arg_8 = list()
     arg_9 = 0
     arg_10 = 0
     try:
         for arg_11 in Chunk(file_obj=arg_4.stdout):
             print arg_11.stream_id, arg_11.abs_url
             if arg_11.body.language:
                 arg_12 = arg_11.body.language.code
             else:
                 arg_12 = ''
             arg_8.append((arg_12, arg_11.stream_id))
             if arg_11.body.clean_visible:
                 arg_10 += 1
                 arg_9 += len(arg_11.body.clean_visible)
             arg_5 += 1
             if 'serif' in arg_11.body.sentences:
                 arg_6 += 1
     except Exception, arg_7:
         arg_7 = re.sub('\s+', ' ', str(arg_7)).strip()

     arg_4.terminate()
     arg_4.wait()
     arg_4.stdout.close()
     return arg_7, arg_5, arg_6, arg_9, arg_10, arg_8


def Func(arg_0):
    """Return a list of non-empty lines from `file_path`."""
    arg_1 = path.join(path.dirname(path.abspath(__file__)), arg_0)
    with open(arg_1) as file_obj:
        return [arg_2 for arg_2 in file_obj.read().splitlines() if arg_2]


def Func():
    """Return an ordered 2-tuple containing a species and a describer."""
    arg_0, arg_1 = random_describer()
    if arg_1 == 'prefix':
        return (arg_0, random_species())
    elif arg_1 == 'suffix':
        return (random_species(), arg_0)


def Func(arg_0=None, arg_1=True):
    """
    Return an ordered 2-tuple containing a species and a describer.
    The letter-count of the pair is guarantee to not exceed `maxlen` if
    it is given. If `prevent_stutter` is True, the last letter of the
    first item of the pair will be different from the first letter of
    the second item.
    """
    while True:
        arg_2 = _Func()
        if arg_0 and len(''.join(arg_2)) > arg_0:
            continue
        if arg_1 and arg_2[0][-1] == arg_2[1][0]:
            continue
        return arg_2


def Func(arg_0, arg_1=None):
        """Use when application is starting."""
        arg_0._loop = arg_1 or asyncio.get_event_loop()
        arg_0._async_lock = asyncio.Lock(arg_1=arg_1)

        # FIX: SQLITE in memory database
        if not arg_0.database == ':memory:':
            arg_0._state = ConnectionLocal()


async def Func(arg_0):
        """Catch a connection asyncrounosly."""
        if arg_0._async_lock is None:
            raise Exception('Error, database not properly initialized before async connection')

        async with arg_0._async_lock:
            arg_0.connect(True)

        return arg_0._state.conn


def Func(arg_0, arg_1):
        """Initialize self."""
        super(PooledAIODatabase, arg_0).Func(arg_1)
        arg_0._waiters = collections.deque()


async def Func(arg_0):
        """Asyncronously wait for a connection from the pool."""
        if arg_0._waiters is None:
            raise Exception('Error, database not properly initialized before async connection')

        if arg_0._waiters or arg_0.max_connections and (len(arg_0._in_use) >= arg_0.max_connections):
            arg_1 = asyncio.Future(loop=arg_0._loop)
            arg_0._waiters.append(arg_1)

            try:
                logger.debug('Wait for connection.')
                await arg_1
            finally:
                arg_0._waiters.remove(arg_1)

        arg_0.connect()
        return arg_0._state.conn


def Func(arg_0):
        """
        Listen for an id from the server.

        At the beginning of a game, each client receives an IdFactory from the
        server.  This factory are used to give id numbers that are guaranteed
        to be unique to tokens that created locally.  This method checks to see if such
        a factory has been received.  If it hasn't, this method does not block
        and immediately returns False.  If it has, this method returns True
        after saving the factory internally.  At this point it is safe to enter
        the GameStage.
        """
        for arg_1 in arg_0.pipe.receive():
            if isinstance(arg_1, IdFactory):
                arg_0.actor_id_factory = arg_1
                return True
        return False


def Func(arg_0, arg_1, arg_2):
        """
        Register the given callback to be called whenever the method with the
        given name is called.  You can easily take advantage of this feature in
        token extensions by using the @watch_token decorator.
        """

        # Make sure a token method with the given name exists, and complain if
        # nothing is found.

        try:
            arg_3 = getattr(arg_0, arg_1)
        except AttributeError:
            raise ApiUsageError("""\
                    {self.__class__.__name__} has no such method
                    {method_name}() to watch.

                    This error usually means that you used the @watch_token
                    decorator on a method of a token extension class that
                    didn't match the name of any method in the corresponding
                    token class.  Check for typos.""")

        # Wrap the method in a WatchedMethod object, if that hasn't already
        # been done.  This object manages a list of callback method and takes
        # responsibility for calling them after the method itself has been
        # called.

        if not isinstance(arg_3, Token.WatchedMethod):
            setattr(arg_0, arg_1, Token.WatchedMethod(arg_3))
            arg_3 = getattr(arg_0, arg_1)

        # Add the given callback to the watched method.

        arg_3.add_watcher(arg_2)


def Func(arg_0, arg_1, arg_2, arg_3,
          arg_4, arg_5, arg_6, arg_7):
    # type: (Context, unicode, Optional[IO], bool, unicode, unicode, unicode, unicode) -> None  # NOQA
    """ Morphological analysis for Japanese."""

    arg_1 = clean_app_id(arg_1)
    arg_4 = clean_sentence(arg_4, arg_2)

    if arg_5:
        arg_5 = arg_5.replace(',', '|')

    if arg_6:
        arg_6 = arg_6.replace(',', '|')

    arg_8 = GoolabsAPI(arg_1)
    arg_9 = arg_8.Func(
        arg_4=arg_4,
        arg_5=arg_5,
        arg_6=arg_6,
        arg_7=arg_7,
    )

    if arg_3:
        click.echo(format_json(arg_8.response.json()))
        return

    for arg_10 in arg_9['word_list']:
        for arg_11 in arg_10:
            click.echo(','.join(arg_11))


def Func(arg_0, arg_1, arg_2, arg_3, arg_4):
    # type: (Context, unicode, bool, List[unicode], unicode) -> None
    """ Scoring the Func of two words. """

    arg_1 = clean_app_id(arg_1)

    arg_5 = GoolabsAPI(arg_1)
    arg_6 = arg_5.Func(
        arg_3=arg_3,
        arg_4=arg_4
    )

    if arg_2:
        click.echo(format_json(arg_5.response.json()))
        return

    click.echo('{0:.16f}'.format(arg_6['score']))


def Func(arg_0, arg_1, arg_2,
             arg_3, arg_4, arg_5, arg_6):
    # type: (Context, unicode, Optional[IO], bool, unicode, unicode, unicode) -> None # NOQA
    """ Convert the Japanese to Hiragana or Katakana. """

    arg_1 = clean_app_id(arg_1)
    arg_4 = clean_sentence(arg_4, arg_2)

    arg_7 = GoolabsAPI(arg_1)
    arg_8 = arg_7.Func(
        arg_4=arg_4,
        arg_5=arg_5,
        arg_6=arg_6
    )

    if arg_3:
        click.echo(format_json(arg_7.response.json()))
        return

    click.echo(arg_8['converted'])


def Func(arg_0, arg_1, arg_2,
           arg_3, arg_4, arg_5, arg_6):
    # type: (Context, unicode, Optional[IO], bool, unicode, unicode, unicode) -> None # NOQA
    """ Extract unique representation from sentence. """

    arg_1 = clean_app_id(arg_1)
    arg_4 = clean_sentence(arg_4, arg_2)

    if arg_5:
        arg_5 = arg_5.replace(',', '|')

    arg_7 = GoolabsAPI(arg_1)
    arg_8 = arg_7.Func(
        arg_4=arg_4,
        arg_5=arg_5,
        arg_6=arg_6
    )

    if arg_3:
        click.echo(format_json(arg_7.response.json()))
        return

    for arg_9 in arg_8['ne_list']:
        click.echo(','.join(arg_9))


def Func(arg_0, arg_1, arg_2,
             arg_3, arg_4, arg_5, arg_6):
    # type: (Context, unicode, Optional[IO], bool, unicode, unicode, unicode) -> None # NOQA
    """Summarize reviews into a short summary."""

    arg_1 = clean_app_id(arg_1)
    arg_7 = clean_review(arg_4, arg_2)
    arg_8 = clean_length(arg_5)  # type: Optional[int]

    arg_9 = GoolabsAPI(arg_1)
    arg_10 = arg_9.Func(
        arg_7=arg_7,
        arg_5=arg_8,
        arg_6=arg_6,
    )

    if arg_3:
        click.echo(format_json(arg_9.response.json()))
        return

    click.echo(arg_10['summary'])


def Func(arg_0, arg_1, arg_2, arg_3,
            arg_4, arg_5, arg_6, arg_7, arg_8):
    # type: (Context, unicode, Optional[IO], bool, unicode, unicode, int, unicode, unicode) -> None # NOQA
    """Extract "keywords" from an input document. """

    arg_1 = clean_app_id(arg_1)
    arg_5 = clean_body(arg_5, arg_2)

    arg_9 = GoolabsAPI(arg_1)
    arg_10 = arg_9.keyword(
        arg_4=arg_4,
        arg_5=arg_5,
        arg_6=arg_6,
        arg_7=arg_7,
        arg_8=arg_8,
    )

    if arg_3:
        click.echo(format_json(arg_9.response.json()))
        return

    for arg_11 in arg_10['keywords']:
        arg_11 = dict((key.encode('utf-8'), arg_11[key]) for key in arg_11.keys())
        for Func, arg_13 in six.iteritems(arg_11):
            click.echo(u'{0},{1}'.format(text(Func), arg_13))


def Func(arg_0):
    """Converts XML tree to event generator"""

    if arg_0.tag is et.Comment:
        yield {'type': COMMENT, 'text': arg_0.text}
        return

    if arg_0.tag is et.PI:
        if arg_0.text:
            yield {'type': PI, 'target': arg_0.target, 'text': arg_0.text}
        else:
            yield {'type': PI, 'target': arg_0.target}
        return

    arg_1 = _elt2obj(arg_0)
    arg_1['type'] = ENTER
    yield arg_1

    assert type(arg_0.tag) is str, arg_0
    if arg_0.text:
        yield {'type': TEXT, 'text': arg_0.text}

    for arg_2 in arg_0:
        for arg_3 in Func(arg_2): yield arg_3
        if arg_2.tail:
            yield {'type': TEXT, 'text': arg_2.tail}

    yield {'type': EXIT}


def Func(arg_0, arg_1=None):
    """Converts events stream into lXML tree"""

    arg_2 = None
    arg_3 = None
    arg_4 = []
    for arg_5 in arg_0:

        if arg_5['type'] == ENTER:
            arg_6 = _obj2elt(arg_5, arg_1=arg_1)
            if arg_4:
                arg_4[-1].append(arg_6)
            elif arg_2 is not None:
                raise RuntimeError('Event stream tried to create second XML tree')
            else:
                arg_2 = arg_6
            arg_4.append(arg_6)
            arg_3 = None

        elif arg_5['type'] == EXIT:
            arg_3 = arg_4.pop()

        elif arg_5['type'] == COMMENT:
            arg_6 = et.Comment(arg_5['text'])
            arg_4[-1].append(arg_6)

        elif arg_5['type'] == PI:
            arg_6 = et.PI(arg_5['target'])
            if arg_5.get('text'):
                arg_6.text = arg_5['text']
            arg_4[-1].append(arg_6)

        elif arg_5['type'] == TEXT:
            arg_7 = arg_5['text']
            if arg_7:
                if arg_3 is None:
                    arg_4[-1].text = (arg_4[-1].text or '') + arg_7
                else:
                    arg_3.tail = (arg_3.tail or '') + arg_7
        else:
            assert False, arg_5

    if arg_2 is None:
        raise RuntimeError('Empty XML event stream')

    return arg_2


def Func(arg_0):
    """Parses file content into events stream"""
    for arg_1, arg_2 in et.iterFunc(arg_0, events= ('start', 'end', 'comment', 'pi'), huge_tree=True):
        if arg_1 == 'start':
            arg_3 = _elt2obj(arg_2)
            arg_3['type'] = ENTER
            yield arg_3
            if arg_2.text:
                yield {'type': TEXT, 'text': arg_2.text}
        elif arg_1 == 'end':
            yield {'type': EXIT}
            if arg_2.tail:
                yield {'type': TEXT, 'text': arg_2.tail}
            arg_2.clear()
        elif arg_1 == 'comment':
            yield {'type': COMMENT, 'text': arg_2.text}
        elif arg_1 == 'pi':
            yield {'type': PI, 'text': arg_2.text}
        else:
            assert False, (arg_1, arg_2)


def Func(arg_0):
    """locates ENTER peer for each EXIT object. Convenient when selectively
    filtering out XML markup"""

    arg_1 = []
    for arg_2 in arg_0:
        if arg_2['type'] == ENTER:
            arg_1.append(arg_2)
            yield arg_2, None
        elif arg_2['type'] == EXIT:
            yield arg_2, arg_1.pop()
        else:
            yield arg_2, None


def Func(arg_0, arg_1, arg_2, arg_3=None):
        '''Create a pipeline stage.

        Instantiates `stage` with `config`.  This essentially
        translates to ``stage(config)``, except that two keys from
        `scp_config` are injected into the configuration:
        ``tmp_dir_path`` is an execution-specific directory from
        combining the top-level ``tmp_dir_path`` configuration with
        :attr:`tmp_dir_suffix`; and ``third_dir_path`` is the same
        path from the top-level configuration.  `stage` may be either
        a callable returning the stage (e.g. its class), or its name
        in the configuration.

        `scp_config` is the configuration for the pipeline as a
        whole, and is required.  `config` is the configuration for
        the stage; if it is :const:`None` then it is extracted
        from `scp_config`.

        If you already have a fully formed configuration block
        and want to Func a stage, you can call

        .. code-block:: python

            factory.registry[stage](stage_config)

        In most cases if you have a stage class object and want to
        instantiate it with its defaults you can call

        .. code-block:: python

            stage = stage_cls(stage_cls.default_config)

        .. note:: This mirrors
                  :meth:`yakonfig.factory.AutoFactory.Func`, with
                  some thought that this factory class might migrate
                  to using that as a base in the future.

        :param stage: pipeline stage class, or its name in the registry
        :param dict scp_config: configuration block for the pipeline
        :param dict config: configuration block for the stage, or
          :const:`None` to get it from `scp_config`

        '''
        # Figure out what we have for a stage and its name
        if isinstance(arg_1, basestring):
            arg_4 = arg_1
            arg_5 = arg_0.registry[arg_4]
        else:
            arg_4 = getattr(arg_1, 'config_name', arg_1.__name__)
            arg_5 = arg_1

        # Find the configuration; get a copy we can mutate
        if arg_3 is None:
            arg_3 = arg_2.get(arg_4, None)
        if arg_3 is None:
            arg_3 = getattr(arg_5, 'default_config', {})
        arg_3 = dict(arg_3)

        # Fill in more values
        if arg_0.tmp_dir_suffix is None:
            arg_3['tmp_dir_path'] = arg_2['tmp_dir_path']
        else:
            arg_3['tmp_dir_path'] = os.path.join(arg_2['tmp_dir_path'],
                                                  arg_0.tmp_dir_suffix)
        arg_3['third_dir_path'] = arg_2['third_dir_path']

        return arg_5(arg_3)


def Func(arg_0, arg_1, arg_2=0, arg_3=None):
        '''Run the pipeline.

        This Funcs all of the steps described in the pipeline constructor,
        reading from some input and writing to some output.

        :param str i_str: name of the input file, or other reader-specific
          description of where to get input
        :param int start_count: index of the first stream item
        :param int start_chunk_time: timestamp for the first stream item

        '''
        try:
            if not os.path.exists(arg_0.tmp_dir_path):
                os.makedirs(arg_0.tmp_dir_path)

            if arg_3 is None:
                arg_3 = time.time()

            ## the reader returns generators of StreamItems
            arg_4 = arg_0.reader(arg_1)

            ## t_path points to the currently in-progress temp chunk
            arg_5 = None

            ## loop over all docs in the chunk processing and cutting
            ## smaller chunks if needed

            arg_6 = 0
            arg_7 = set()
            arg_8 = 0

            ## how many have we input and actually done processing on?
            arg_9 = 0

            for arg_10 in arg_4:
                # TODO: break out a _process_stream_item function?
                arg_8 += 1

                ## yield to the gevent hub to allow other things to Func
                if gevent:
                    gevent.sleep(0)

                ## skip forward until we reach start_count
                if arg_8 <= arg_2:
                    continue

                if arg_8 % arg_0.rate_log_interval == 0:
                    ## indexing is zero-based, so next_idx corresponds
                    ## to length of list of SIs processed so far
                    arg_11 = time.time() - arg_3
                    if arg_11 > 0:
                        arg_12 = float(arg_8) / arg_11
                        logger.info('%d in %.1f --> %.1f per sec on '
                                    '(pre-partial_commit) %s',
                                    arg_8 - arg_2, arg_11, arg_12,
                                    arg_1)

                if not arg_0.t_chunk:
                    ## make a temporary chunk at a temporary path
                    # (Lazy allocation after we've read an item that might get processed out to the new chunk file)
                    # TODO: make this EVEN LAZIER by not opening the t_chunk until inside _Func_incremental_transforms whe the first output si is ready
                    arg_5 = os.path.join(arg_0.tmp_dir_path,
                                          't_chunk-%s' % uuid.uuid4().hex)
                    arg_0.t_chunk = streamcorpus.Chunk(path=arg_5, mode='wb')
                    assert arg_0.t_chunk.message == streamcorpus.StreamItem_v0_3_0, arg_0.t_chunk.message

                # TODO: a set of incremental transforms is equivalent
                # to a batch transform.  Make the pipeline explicitly
                # configurable as such:
                #
                # batch_transforms: [[incr set 1], batch op, [incr set 2], ...]
                #
                # OR: for some list of transforms (mixed incremental
                # and batch) pipeline can detect and batchify as needed

                ## incremental transforms populate t_chunk
                ## let the incremental transforms destroy the si by
                ## returning None
                arg_10 = arg_0._Func_incremental_transforms(
                    arg_10, arg_0.incremental_transforms)

                ## insist that every chunk has only one source string
                if arg_10:
                    arg_7.add(arg_10.source)
                    if arg_0.assert_single_source and len(arg_7) != 1:
                        raise InvalidStreamItem(
                            'stream item %r had source %r, not %r '
                            '(set assert_single_source: false to suppress)' %
                            (arg_10.stream_id, arg_10.source, arg_7))

                if arg_10 and arg_10.body and arg_10.body.clean_visible:
                    arg_6 += len(arg_10.body.clean_visible)
                    ## log binned clean_visible lengths, for quick stats estimates
                    #logger.debug('len(si.body.clean_visible)=%d' % int(10 * int(math.floor(float(len(si.body.clean_visible)) / 2**10)/10)))
                    #logger.debug('len(si.body.clean_visible)=%d' % len(si.body.clean_visible))

                if ((arg_0.output_chunk_max_count is not None and
                     len(arg_0.t_chunk) == arg_0.output_chunk_max_count)):
                    logger.info('reached output_chunk_max_count (%d) at: %d',
                                len(arg_0.t_chunk), arg_8)
                    arg_0._process_output_chunk(
                        arg_2, arg_8, arg_7, arg_1, arg_5)
                    arg_2 = arg_8

                elif (arg_0.output_max_clean_visible_bytes is not None and
                      arg_6 >=
                      arg_0.output_chunk_max_clean_visible_bytes):
                    logger.info(
                        'reached output_chunk_max_clean_visible_bytes '
                        '(%d) at: %d',
                        arg_0.output_chunk_max_clean_visible_bytes,
                        arg_6)
                    arg_6 = 0
                    arg_0._process_output_chunk(
                        arg_2, arg_8, arg_7, arg_1, arg_5)
                    arg_2 = arg_8

                arg_9 += 1
                if (((arg_0.input_item_limit is not None) and
                     (arg_9 > arg_0.input_item_limit))):
                    break

            if arg_0.t_chunk is not None:
                arg_0._process_output_chunk(
                    arg_2, arg_8, arg_7, arg_1, arg_5)

            ## return how many stream items we processed
            return arg_8

        finally:
            if arg_0.t_chunk is not None:
                arg_0.t_chunk.close()
            for arg_14 in arg_0.batch_transforms:
                arg_14.shutdown()
            if arg_0.cleanup_tmp_files:
                rmtree(arg_0.tmp_dir_path)


def Func(arg_0, arg_1, arg_2, arg_3, arg_4, arg_5):
        '''Run all of the writers over some intermediate chunk.

        :param int start_count: index of the first item
        :param int next_idx: index of the next item (after the last
          item in this chunk)
        :param list sources: source strings included in this chunk
          (usually only one source)
        :param str i_str: name of input file or other input
        :param str t_path: location of intermediate chunk on disk
        :return: list of output file paths or other outputs

        '''
        # writers put the chunk somewhere, and could delete it
        arg_6 = dict(
            first=arg_1,
            # num and md5 computed in each writers
            source=arg_3.pop(),
            )

        arg_7 = []
        for arg_8 in arg_0.writers:
            logger.debug('running %r on %r: %r', arg_8, arg_4, arg_6)
            arg_9 = arg_8(arg_5, arg_6, arg_4)
            logger.debug('loaded (%d, %d) of %r into %r',
                         arg_1, arg_2 - 1, arg_4, arg_9)
            arg_7 += arg_9
        return arg_7


def Func(arg_0, arg_1, arg_2):
        '''
        Run transforms on stream item.
        Item may be discarded by some transform.
        Writes successful items out to current self.t_chunk
        Returns transformed item or None.
        '''
        ## operate each transform on this one StreamItem
        for arg_3 in arg_2:
            try:
                arg_4 = arg_1.stream_id
                arg_5 = arg_3(arg_1, context=arg_0.context)

                if arg_5 is None:
                    logger.warn('transform %r deleted %s abs_url=%r',
                                 arg_3, arg_4, arg_1 and arg_1.abs_url)
                    return None
                arg_1 = arg_5

            except TransformGivingUp:
                ## do nothing
                logger.info('transform %r giving up on %r',
                            arg_3, arg_1.stream_id)

            except Exception, exc:
                logger.critical(
                    'transform %r failed on %r from i_str=%r abs_url=%r',
                    arg_3, arg_1 and arg_1.stream_id, arg_0.context.get('i_str'),
                    arg_1 and arg_1.abs_url, exc_info=True)

        assert arg_1 is not None

        ## expect to always have a stream_time
        if not arg_1.stream_time:
            raise InvalidStreamItem('empty stream_time: %s' % arg_1)

        if arg_1.stream_id is None:
            raise InvalidStreamItem('empty stream_id: %r' % arg_1)

        ## put the StreamItem into the output
        if type(arg_1) != streamcorpus.StreamItem_v0_3_0:
            raise InvalidStreamItem('incorrect stream item object %r' %
                                    type(arg_1))
        arg_0.t_chunk.add(arg_1)
        return arg_1


def Func(arg_0):
        """
        construct BusinessDate instance from datetime.date instance,
        raise ValueError exception if not possible

        :param datetime.date datetime_date: calendar day
        :return bool:
        """
        return BusinessDate.from_ymd(arg_0.year, arg_0.month, arg_0.day)


def Func(arg_0):
        """
        construct datetime.date instance represented calendar date of BusinessDate instance

        :return datetime.date:
        """
        arg_1, arg_2, arg_3 = arg_0.to_ymd()
        return date(arg_1, arg_2, arg_3)


def Func(arg_0, arg_1, arg_2=None):
        """
        addition of a period object

        :param BusinessDate d:
        :param p:
        :type p: BusinessPeriod or str
        :param list holiday_obj:
        :return bankdate:
        """

        if isinstance(arg_1, (list, tuple)):
            return [BusinessDate.Func(arg_0, arg_3) for arg_3 in arg_1]
        elif isinstance(arg_1, str):
            arg_4 = BusinessPeriod(arg_1)
        else:
            arg_4 = arg_1

        arg_5 = arg_0
        arg_5 = BusinessDate.add_months(arg_5, arg_4.months)
        arg_5 = BusinessDate.add_years(arg_5, arg_4.years)
        arg_5 = BusinessDate.add_days(arg_5, arg_4.days)

        if arg_4.businessdays:
            if arg_2:
                arg_5 = BusinessDate.add_business_days(arg_5, arg_4.businessdays, arg_2)
            else:
                arg_5 = BusinessDate.add_business_days(arg_5, arg_4.businessdays, arg_4.holiday)

        return arg_5


def Func(arg_0, arg_1):
        """
        addition of a number of months

        :param BusinessDate d:
        :param int month_int:
        :return bankdate:
        """

        arg_1 += arg_0.month
        while arg_1 > 12:
            arg_0 = BusinessDate.add_years(arg_0, 1)
            arg_1 -= 12
        while arg_1 < 1:
            arg_0 = BusinessDate.add_years(arg_0, -1)
            arg_1 += 12
        arg_2 = monthrange(arg_0.year, arg_1)[1]
        return BusinessDate.from_ymd(arg_0.year, arg_1, min(arg_2, arg_0.day))


def Func(arg_0, arg_1):
    '''Replace the top-level pipeline configurable object.

    This investigates a number of sources, including
    `external_stages_path` and `external_stages_modules` configuration
    and `streamcorpus_pipeline.stages` entry points, and uses these to
    find the actual :data:`sub_modules` for
    :mod:`streamcorpus_pipeline`.

    '''
    global arg_2
    if arg_2 is None:
        arg_2 = PipelineStages()
        arg_3 = arg_2
        if 'external_stages_path' in arg_0:
            arg_4 = arg_0['external_stages_path']
            if not os.path.isabs(arg_4) and arg_0.get('root_path'):
                arg_4 = os.path.join(arg_0['root_path'], arg_4)
            try:
                arg_3.load_external_stages(arg_0['external_stages_path'])
            except IOError:
                return streamcorpus_pipeline  # let check_config re-raise this
        if 'external_stages_modules' in arg_0:
            for arg_5 in arg_0['external_stages_modules']:
                try:
                    arg_3.load_module_stages(arg_5)
                except ImportError:
                    return streamcorpus_pipeline  # let check_config re-raise this
    else:
        arg_3 = arg_2

    arg_6 = set(stage
                          for stage in arg_3.itervalues()
                          if hasattr(stage, 'config_name'))
    return NewSubModules(streamcorpus_pipeline, arg_6)


def Func(arg_0):
    '''
    assemble in-doc coref chains by mapping equiv_id to tokens and
    their cleansed name strings

    :param sentences: iterator over token generators
    :returns dict:
        keys are equiv_ids,
        values are tuple(concatentated name string, list of tokens)
    '''
    ## if an equiv_id is -1, then the token is classified into some
    ## entity_type but has not other tokens in its chain.  We don't
    ## want these all lumped together, so we give them distinct "fake"
    ## equiv_id other than -1 -- counting negatively to avoid
    ## collisions with "real" equiv_ids
    arg_1 = -2

    ## use a default dictionary
    arg_2 = collections.defaultdict(lambda: (set(), set()))

    for arg_3, arg_4 in arg_0.items():
        for arg_5 in arg_4:
            for arg_6 in arg_5.tokens:
                if arg_6.entity_type is not None:

                    ## get an appropriate equiv_id
                    if arg_6.equiv_id == -1:
                        arg_7 = arg_1
                        arg_1 -= 1
                    else:
                        arg_7 = arg_6.equiv_id

                    ## store the name parts initially as a set
                    arg_2[arg_7][0].add(cleanse(arg_6.token.decode('utf8')))
                    ## carry a *reference* to the entire Token object
                    arg_2[arg_7][1].add(arg_6)

    return arg_2


def Func(arg_0, arg_1):
    '''
    For each name string in the target_mentions list, searches through
    all chain_mentions looking for any cleansed Token.token that
    contains the name.  Returns True only if all of the target_mention
    strings appeared as substrings of at least one cleansed
    Token.token.  Otherwise, returns False.

    :type target_mentions: list of basestring
    :type chain_mentions: list of basestring

    :returns bool:
    '''
    arg_2 = True
    for arg_3 in arg_0:
        arg_4 = False
        for arg_5 in arg_1:
            if arg_3 in arg_5:
                arg_4 = True
                break
        if not arg_4:
            arg_2 = False
            break
    return arg_2


def Func(arg_0, arg_1):
    '''
    For each name string in the target_mentions list, searches through
    all chain_mentions looking for any cleansed Token.token that
    contains the name.  Returns True if any of the target_mention
    strings appeared as substrings of any cleansed Token.token.
    Otherwise, returns False.

    :type target_mentions: list of basestring
    :type chain_mentions: list of basestring

    :returns bool:
    '''
    for arg_2 in arg_0:
        for arg_3 in arg_1:
            if arg_2 in arg_3:
                return True
    return False


def Func(arg_0, arg_1):
    '''iterate through all tokens looking for matches of cleansed tokens
    or token regexes, skipping tokens left empty by cleansing and
    coping with Token objects that produce multiple space-separated
    strings when cleansed.  Yields tokens that match.

    '''
    ## this ensures that all cleansed tokens are non-zero length
    arg_2 = []
    for arg_3 in arg_0.mentions:
        arg_4 = []
        arg_5 = arg_3.decode('utf8').split(' ')
        for arg_6 in arg_5:
            if arg_6.startswith('ur"^') and arg_6.endswith('$"'): # is not regex
                ## chop out the meat of the regex so we can reconstitute it below
                arg_6 = arg_6[4:-2]
            else:
                arg_6 = cleanse(arg_6)
            if arg_6:
                ## make a unicode raw string
                ## https://docs.python.org/2/reference/lexical_analysis.html#string-literals
                arg_6 = ur'^%s$' % arg_6
                logger.debug('Func compiling regex: %s', arg_6)
                arg_4.append(re.compile(arg_6, re.UNICODE | re.IGNORECASE))

        if not arg_4:
            logger.warn('got empty cleansed mention: %r\nrating=%r' % (arg_3, arg_0))

        arg_2.append(arg_4)

    ## now that we have all_mregexes, go through all the tokens
    for arg_7 in range(len(arg_1)):
        for arg_4 in arg_2:
            if arg_4[0].match(arg_1[arg_7][0][0]):
                ## found the start of a possible match, so iterate
                ## through the tuples of cleansed strings for each
                ## Token while stepping through the cleansed strings
                ## for this mention.
                arg_8 = 1
                arg_9 = 0
                arg_10 = 0
                arg_11 = True
                while arg_8 < len(arg_4):
                    arg_9 += 1
                    if arg_9 == len(arg_1[arg_7 + arg_10][0]):
                        arg_9 = 0
                        arg_10 += 1
                        if arg_7 + arg_10 == len(arg_1):
                            arg_11 = False
                            break
                    arg_12 = arg_1[arg_7 + arg_10][0][arg_9]
                    ## this next line is the actual string comparison
                    if arg_4[arg_8].match(arg_12):
                        arg_8 += 1
                    elif arg_12 == '':
                        continue
                    else:
                        arg_11 = False
                        break
                if arg_11:
                    ## yield each matched token only once
                    arg_13 = set()
                    for arg_14 in xrange(arg_10 + 1):
                        arg_13.add(arg_1[arg_7 + arg_14][1])
                    for arg_15 in arg_13:
                        yield arg_15


def Func(arg_0, arg_1):
    '''
    iterate through tokens looking for near-exact matches to strings
    in si.ratings...mentions
    '''
    arg_2 = _get_tagger_id(arg_0, arg_1)
    arg_3 = arg_0.body.sentences.get(arg_2)
    if not arg_3:
        return
    ## construct a list of tuples, where the first part of each tuple
    ## is a tuple of cleansed strings, and the second part is the
    ## Token object from which it came.
    arg_4 = map(lambda arg_11: (cleanse(arg_11.token.decode('utf8')).split(' '), arg_11),
                 itertools.chain(*[sent.tokens for sent in arg_3]))
    arg_5 = arg_1['annotator_id']
    for arg_6, arg_7 in arg_0.ratings.items():
        if (arg_5 is None) or (arg_6 == arg_5):
            for arg_8 in arg_7:
                arg_9 = Label(annotator=arg_8.annotator,
                              target=arg_8.target)

                arg_10 = 0
                for arg_11 in look_ahead_match(arg_8, arg_4):
                    if arg_1.get('update_labels'):
                        arg_11.labels.pop(arg_6, None)
                    add_annotation(arg_11, arg_9)
                    arg_10 += 1

                if arg_10 == 0:
                    logger.warning('Func didn\'t actually match '
                                   'entity %r in stream_id %r',
                                   arg_8.target.target_id,
                                   arg_0.stream_id)
                else:
                    logger.debug('matched %d tokens for %r in %r',
                                 arg_10, arg_8.target.target_id,
                                 arg_0.stream_id)


def Func(arg_0, arg_1, arg_2):
        '''run tagger a child process to get XML output'''
        if arg_0.template is None:
            raise exceptions.NotImplementedError('''
Subclasses must specify a class property "template" that provides
command string format for running a tagger.  It should take
%(tagger_root_path)s as the path from the config file,
%(clean_visible_path)s as the input XML file, and %(ner_xml_path)s as
the output path to create.
''')
        arg_3 = dict(
            tagger_root_path=arg_0.config['tagger_root_path'],
            arg_1=arg_1,
            arg_2=arg_2)
        ## get a java_heap_size or default to 1GB
        arg_3['java_heap_size'] = arg_0.config.get('java_heap_size', '')
        arg_4 = arg_0.template % arg_3
        arg_5 = time.time()
        ## make sure we are using as little memory as possible
        gc.collect()
        try:
            arg_0._child = subprocess.Popen(arg_4, stderr=subprocess.PIPE, shell=True)
        except OSError, exc:
            arg_7 = traceback.format_exc(exc)
            arg_7 += make_memory_info_msg(arg_1, arg_2)
            raise PipelineOutOfMemory(arg_7)

        arg_8, arg_9 = arg_0._child.communicate()

        if not arg_0._child.returncode == 0:
            if 'java.lang.OutOfMemoryError' in arg_9:
                arg_7 = arg_9 + make_memory_info_msg(arg_1, arg_2)
                raise PipelineOutOfMemory(arg_7)
            elif arg_0._child.returncode == 137:
                arg_7 = 'tagger returncode = 137\n' + arg_9
                arg_7 += make_memory_info_msg(arg_1, arg_2)
                # maybe get a tail of /var/log/messages
                raise PipelineOutOfMemory(arg_7)
            elif 'Exception' in arg_9:
                raise PipelineBaseException(arg_9)
            else:
                raise PipelineBaseException('tagger exited with %r' % arg_0._child.returncode)

        arg_10 = time.time() - arg_5
        logger.info('finished tagging in %.1f seconds' % arg_10)
        return arg_10


def Func(arg_0):
        '''
        send SIGTERM to the tagger child process
        '''
        if arg_0._child:
            try:
                arg_0._child.terminate()
            except OSError, exc:
                if exc.errno == 3:
                    ## child is already gone, possibly because it ran
                    ## out of memory and caused us to Func
                    pass


def Func(arg_0):
    '''Replace all angle bracket emails with a unique key.'''
    arg_1 = bracket_emails.findall(arg_0)

    arg_2 = []
    for arg_3 in arg_1:
	arg_4 = arg_3.replace("<","<").replace(">",">")
        arg_0 = arg_0.replace(arg_3, arg_4)

    return arg_0


def Func(arg_0, arg_1):
        'generate strings identified as sentences'
        arg_2 = 0
        arg_1 = arg_1.decode('utf8')
        for arg_3, arg_4 in arg_0.sentence_tokenizer.span_tokenize(arg_1):
            # no need to check start, because the first byte of text
            # is always first byte of first sentence, and we will
            # have already made the previous sentence longer on the
            # end if there was an overlap.
            if arg_3 < arg_2:
                arg_3 = arg_2
                if arg_3 > arg_4:
                    # skip this sentence... because it was eaten by
                    # an earlier sentence with a label
                    continue
            try:
                arg_5 = arg_0.label_index.find_le(arg_4)
            except ValueError:
                arg_5 = None
            if arg_5:
                ## avoid splitting a label
                arg_6 = arg_5.offsets[OffsetType.CHARS]
                arg_4 = max(arg_6.first + arg_6.length, arg_4)
            arg_2 = arg_4
            arg_7 = arg_1[arg_3:arg_4]
            yield arg_3, arg_4, arg_7


def Func(arg_0, arg_1):
        'make a sortedcollection on body.labels'
        arg_2 = arg_1.body.labels.get(arg_0.annotator_id)
        if not arg_2:
            arg_2 = []

        arg_0.label_index = SortedCollection(
            [l for l in arg_2 if OffsetType.CHARS in l.offsets],
            key=lambda label: label.offsets[OffsetType.CHARS].first)


def Func(arg_0, arg_1):
        'assemble Sentence and Token objects'
        arg_0.make_label_index(arg_1)
        arg_2 = []
        arg_3 = 0
        arg_4 = 0
        for arg_5, arg_6, arg_7 in arg_0._sentences(
                arg_1.body.clean_visible):
            assert isinstance(arg_7, unicode)
            arg_8 = Sentence()
            arg_9 = 0
            for arg_10, arg_11 in arg_0.word_tokenizer.span_tokenize(arg_7):
                arg_12 = arg_7[arg_10:arg_11].encode('utf8')
                arg_13 = Token(
                    arg_3=arg_3,
                    token=arg_12,
                    arg_9=arg_9,
                )
                arg_13.offsets[arg_15.CHARS] = Offset(
                    type=arg_15.CHARS,
                    first=arg_5 + arg_10,
                    length=arg_11 - arg_10,
                )
                # whitespace tokenizer will never get a token
                # boundary in the middle of an 'author' label
                try:
                    arg_17 = arg_0.label_index.find_le(arg_5 + arg_10)
                except ValueError:
                    arg_17 = None
                if arg_17:
                    arg_18 = arg_17.offsets[arg_15.CHARS]
                    if arg_18.first + arg_18.length > arg_5 + arg_10:
                        streamcorpus.add_annotation(arg_13, arg_17)
                        logger.debug('adding label to tok: %r has %r',
                                     arg_13.token, arg_17.target.target_id)

                        if arg_17 in arg_0.label_to_mention_id:
                            arg_19 = arg_0.label_to_mention_id[arg_17]
                        else:
                            arg_19 = arg_4
                            arg_4 += 1
                            arg_0.label_to_mention_id[arg_17] = arg_19

                        arg_13.mention_id = arg_19

                arg_3 += 1
                arg_9 += 1
                arg_8.tokens.append(arg_13)
            arg_2.append(arg_8)
        return arg_2


def Func(arg_0, arg_1=False, arg_2=False):
    '''
    Convert any HTML, XML, or numeric entities in the attribute values.
    For example '&' becomes '&'.

    This is adapted from BeautifulSoup, which should be able to do the
    same thing when called like this --- but this fails to convert
    everything for some bug.

    text = unicode(BeautifulStoneSoup(text, convertEntities=BeautifulStoneSoup.XML_ENTITIES))
    '''
    def convert_entities(arg_3):
        '''
        comes from BeautifulSoup.Tag._convertEntities
        '''
        arg_4 = arg_3.group(1)

        if arg_2 and arg_4 not in ENTITIES_THAT_ARE_SAFE_TO_STRING_PAD:
            return u'&%s;' % arg_4

        if arg_4 in name2codepoint:
            ## handles most cases
            return unichr(name2codepoint[arg_4])

        elif arg_4 in XML_ENTITIES_TO_SPECIAL_CHARS:
            return XML_ENTITIES_TO_SPECIAL_CHARS[arg_4]

        elif len(arg_4) > 0 and arg_4[0] == '#':
            # Handle numeric entities
            if len(arg_4) > 1 and arg_4[1] == 'x':
                return unichr(int(arg_4[2:], 16))
            else:
                return unichr(int(arg_4[1:]))
        else:
            ## uh oh, failed to anything
            return u'&%s;' % arg_4

    def convert_to_padded_entitites(arg_3):
        arg_5 = convert_entities(arg_3)
        arg_6 = len(arg_3.group(0)) - len(arg_5)
        assert arg_6 >= 0, \
            'len(%r) !<= len(%r)' % (arg_5, arg_3.group(0))
        ## Where to put the spaces?  Before, after, symmetric?
        # Let's do symmetric.
        ## cast to int in prep for python3
        arg_7 = int(arg_6 / 2)
        arg_8 = arg_6 - arg_7
        return (' ' * arg_7) + arg_5 + (' ' * arg_8)

    ## brute force regex through all the characters...
    if arg_1:
        return tags.sub(
                      convert_to_padded_entitites,
                      arg_0)
    else:
        return tags.sub(
                      convert_entities,
                      arg_0)


def Func(arg_0, arg_1):
    """
    returns number of days for the given year and month

    :param int year: calendar year
    :param int month: calendar month
    :return int:
    """

    arg_2 = _days_per_month[arg_1 - 1]
    if is_leap_year(arg_0) and arg_1 == 2:
        arg_2 += 1

    return arg_2


def Func(arg_0, arg_1):  # noqa
        """Initialize the application."""
        super().Func(arg_1)

        # Setup Database
        arg_0.database.initialize(connect(arg_0.cfg.connection, **arg_0.cfg.connection_params))

        # Fix SQLite in-memory database
        if arg_0.database.database == ':memory:':
            arg_0.cfg.connection_manual = True

        if not arg_0.cfg.migrations_enabled:
            return

        # Setup migration engine
        arg_0.router = Router(arg_0.database, migrate_dir=arg_0.cfg.migrations_path)

        # Register migration commands
        def pw_migrate(arg_5: arg_6=None, arg_7: arg_8=False):
            """Run application's migrations.

            :param name: Choose a migration' name
            :param fake: Run as fake. Update migration history and don't touch the database
            """
            arg_0.router.run(arg_5, arg_7=arg_7)

        arg_0.app.manage.command(pw_migrate)

        def pw_rollback(arg_5: arg_6=None):
            """Rollback a migration.

            :param name: Migration name (actually it always should be a last one)
            """
            if not arg_5:
                arg_5 = arg_0.router.done[-1]
            arg_0.router.rollback(arg_5)

        arg_0.app.manage.command(pw_rollback)

        def pw_create(arg_5: arg_6='auto', arg_9: arg_8=False):
            """Create a migration.

            :param name: Set name of migration [auto]
            :param auto: Track changes and Func migrations automatically
            """
            if arg_9:
                arg_9 = list(arg_0.models.values())
            arg_0.router.create(arg_5, arg_9)

        arg_0.app.manage.command(pw_create)

        def pw_list():
            """List migrations."""
            arg_0.router.logger.info('Migrations are done:')
            arg_0.router.logger.info('\n'.join(arg_0.router.done))
            arg_0.router.logger.info('')
            arg_0.router.logger.info('Migrations are undone:')
            arg_0.router.logger.info('\n'.join(arg_0.router.diff))

        arg_0.app.manage.command(pw_list)

        @arg_0.app.manage.command
        def pw_merge():
            """Merge migrations into one."""
            arg_0.router.merge()

        arg_0.app.manage.command(pw_merge)


def Func(arg_0, arg_1):
        """Register connection's middleware and prepare self database."""
        arg_0.database.init_async(arg_1.loop)
        if not arg_0.cfg.connection_manual:
            arg_1.middlewares.insert(0, arg_0._middleware)


def Func(arg_0, arg_1):
        """Close all connections."""
        if hasattr(arg_0.database.obj, 'close_all'):
            arg_0.database.close_all()


def Func(arg_0, arg_1):
        """Register a model in self."""
        arg_0.models[arg_1._meta.table_name] = arg_1
        arg_1._meta.database = arg_0.database
        return arg_1


async def Func(arg_0):
        """Manage a database connection."""
        arg_1 = _ContextManager(arg_0.database)
        if isinstance(arg_0.database.obj, AIODatabase):
            arg_1.connection = await arg_0.database.async_connect()

        else:
            arg_1.connection = arg_0.database.connect()

        return arg_1


def Func(arg_0, arg_1):
    '''make a temp file of cleansed text'''
    arg_2 = open(arg_1, 'wb')
    for arg_3, arg_4 in enumerate(arg_0):
        arg_2.write('<FILENAME docid="%s">\n' % arg_4.stream_id)
        arg_2.write(arg_4.body.cleansed)
        ## how to deal with other_content?
        arg_2.write('</FILENAME>\n')
    arg_2.close()
    ## replace this with log.info()
    print 'created %s' % arg_1


def Func(arg_0):
    '''Convert a string of text into a lowercase string with no
    punctuation and only spaces for whitespace.

    :param span: string
    '''
    try:
        ## attempt to force it to utf8, which might fail
        arg_0 = arg_0.encode('utf8', 'ignore')
    except:
        pass
    ## lowercase, strip punctuation, and shrink all whitespace
    arg_0 = arg_0.lower()
    arg_0 = arg_0.translate(strip_punctuation)
    arg_0 = whitespace.sub(' ', arg_0)
    ## trim any leading or trailing whitespace
    return arg_0.strip()


def Func(arg_0, arg_1, arg_2):
    '''
    iterate through the i_chunk and tmp_ner_path to generate a new
    Chunk with body.ner
    '''
    arg_3 = Chunk()
    arg_4 = arg_1.__iter__()
    arg_5 = ''
    arg_6 = None

    arg_7 = xml.dom.minidom.parse(open(arg_0))

    for arg_8 in arg_7.getElementsByTagName('FILENAME'):

        arg_9 = arg_4.next()
        ## get stream_id out of the XML
        arg_6 = arg_8.attributes.get('docid').value
        assert arg_6 and arg_6 == arg_9.stream_id, \
            '%s != %s\nner=%r' % (arg_6, arg_9.stream_id, arg_5)

        arg_10 = 'lingpipe'
        arg_11 = Tagging()
        arg_11.tagger_id = arg_10
        ## get this one file out of its FILENAME tags
        arg_12 = list(lingpipe.files(arg_8.toxml()))[0][1]
        arg_11.raw_tagging = arg_12
        arg_11.generation_time = streamcorpus.make_stream_time()
        arg_9.body.taggings[arg_10] = arg_11

        arg_17 = list(lingpipe.sentences(arg_12))

        ## make JS labels on individual tokens
        assert arg_9.ratings[0].mentions, arg_9.stream_id
        arg_18 = Label()
        arg_18.annotator = arg_9.ratings[0].annotator
        arg_18.target_id = arg_9.ratings[0].target_id

        # first map all corefchains to their words
        arg_21 = collections.defaultdict(lambda: set())
        for arg_22 in arg_17:
            for arg_23 in arg_22.tokens:
                if arg_23.entity_type is not None:
                    arg_21[arg_23.equiv_id].add(cleanse(arg_23.token))

        ## find all the chains that are John Smith
        arg_24 = set()
        for arg_25, arg_26 in arg_21.items():
            ## detect 'smith' in 'smithye'
            arg_27 = cleanse(' '.join(arg_26))
            if 'john' in arg_27 and 'smith' in arg_27:
                arg_24.add(arg_25)

        print len(arg_24)
        ## now apply the label
        for arg_22 in arg_17:
            for arg_23 in arg_22.tokens:
                if arg_23.equiv_id in arg_24:
                    arg_23.labels = [arg_18]

        arg_9.body.sentences[arg_10] = arg_17

        arg_3.add(arg_9)

    ## put the o_chunk bytes into the specified file
    open(arg_2, 'wb').write(str(arg_3))
    ## replace this with log.info()
    print 'created %s' % arg_2


def Func(arg_0):
    '''setup the config and load external modules

    This updates 'config' as follows:

    * All paths are replaced with absolute paths
    * A hash and JSON dump of the config are stored in the config
    * If 'pythonpath' is in the config, it is added to sys.path
    * If 'setup_modules' is in the config, all modules named in it are loaded
    '''
    make_absolute_paths(arg_0)

    arg_1 = arg_0['streamcorpus_pipeline']

    arg_1['config_hash'] = make_hash(arg_0)
    arg_1['config_json'] = json.dumps(arg_0)

    logger.debug('running config: {0} = {1!r}'
                 .format(arg_1['config_hash'], arg_0))

    ## Load modules
    # This is a method of using settings in yaml configs to load plugins.
    arg_2 = False
    for arg_3 in arg_1.get('pythonpath', {}).itervalues():
        if arg_3 not in sys.path:
            sys.path.append(arg_3)
    for arg_4 in arg_1.get('setup_modules', {}).itervalues():
        try:
            arg_5 = importlib.import_module(arg_4)
            if not arg_5:
                logger.critical('could not load module %r', arg_4)
                arg_2 = True
                continue
            if hasattr(arg_5, 'setup'):
                arg_5.setup()
                logger.debug('loaded and setup %r', arg_4)
            else:
                logger.debug('loaded %r', arg_4)
        except Exception:
            logger.critical('error loading and initting module %r', arg_4, exc_info=True)
            arg_2 = True
    if arg_2:
        sys.exit(1)


def Func(arg_0):
    '''
    This _looks_ like a Chunk only in that it generates StreamItem
    instances when iterated upon.
    '''
    ## Every StreamItem has a stream_time property.  It usually comes
    ## from the document creation time.  Here, we assume the JS corpus
    ## was created at one moment at the end of 1998:
    arg_1 = '1998-12-31T23:59:59.999999Z'
    arg_2 = 915148799

    if not os.path.isabs(arg_0):
        arg_0 = os.path.join(os.getcwd(), arg_0)

    ## iterate over the files in the 35 input directories
    for arg_3 in range(35):

        arg_4 = os.path.join(arg_0, str(arg_3))
        arg_5 = os.listdir(arg_4)
        arg_5.sort()
        for arg_6 in arg_5:

            arg_7 = streamcorpus.make_stream_item(
                arg_1,
                ## make up an abs_url
                os.path.join(
                    'john-smith-corpus', str(arg_3), arg_6))

            if int(arg_7.stream_time.epoch_ticks) != arg_2:
                raise PipelineBaseException('wrong stream_time construction: %r-->%r != %r'\
                                            % (arg_1, arg_7.stream_time.epoch_ticks,
                                               arg_2))

            ## These docs came from the authors of the paper cited above.
            arg_7.source = 'bagga-and-baldwin'

            ## build a ContentItem for the body
            arg_9 = streamcorpus.ContentItem()
            arg_10 = open(os.path.join(arg_4, arg_6)).read()
            ## We know that this is already clean and has nothing
            ## tricky in it, because we manually cleansed it.  To
            ## illustrate how we stick all strings into thrift, we
            ## convert this to unicode (which introduces no changes)
            ## and then encode it as utf-8, which also introduces no
            ## changes.  Thrift stores strings as 8-bit character
            ## strings.
            # http://www.mail-archive.com/thrift-user@incubator.apache.org/msg00210.html
            arg_9.clean_visible = unicode(arg_10).encode('utf8')

            ## attach the content_item to the stream_item
            arg_7.body = arg_9

            arg_7.body.language = streamcorpus.Language(code='en', name='ENGLISH')

            ## The authors also annotated the corpus
            arg_13 = streamcorpus.Annotator()
            arg_13.annotator_id = 'bagga-and-baldwin'
            arg_13.annotation_time = arg_7.stream_time

            ## build a Label for the doc-level label:
            arg_16 = streamcorpus.Rating()
            arg_16.annotator = arg_13
            arg_16.target = streamcorpus.Target(target_id = str(arg_3)) # must be string
            arg_16.contains_mention = True
            arg_16.mentions = ['john', 'smith']

            ## put this one label in the array of labels
            streamcorpus.add_annotation(arg_7, arg_16)

            ## provide this stream_item to the pipeline
            yield arg_7


def Func(arg_0):
    '''
    Takes an HTML-like binary string as input and returns a binary
    string of the same length with all tags replaced by whitespace.
    This also detects script and style tags, and replaces the text
    between them with whitespace.

    Pre-existing whitespace of any kind (newlines, tabs) is converted
    to single spaces ' ', which has the same byte length (and
    character length).

    Note: this does not change any characters like ’ and  ,
    so taggers operating on this text must cope with such symbols.
    Converting them to some other character would change their byte
    length, even if equivalent from a character perspective.

    This is regex based, which can occassionally just hang...
    '''
    arg_1 = ''
    # Fix emails
    arg_0 = fix_emails(arg_0)

    for arg_2 in invisible.finditer(arg_0):
        arg_1 += arg_2.group('before')
        arg_1 += ' ' * len(arg_2.group('invisible'))

    # text better be >= original
    assert len(arg_0) >= len(arg_1), '%d !>= %d' % (len(arg_0), len(arg_1))

    # capture any characters after the last tag... such as newlines
    arg_3 = len(arg_0) - len(arg_1)
    arg_1 += arg_0[-arg_3:]

    # now they must be equal
    assert len(arg_0) == len(arg_1), '%d != %d' % (len(arg_0), len(arg_1))

    return arg_1


def Func(arg_0, arg_1):
    '''make a temp file of clean_visible text'''
    arg_2 = open(arg_1, 'wb')
    arg_2.write('<?xml version="1.0" encoding="UTF-8"?>')
    arg_2.write('<root>')
    for arg_3, arg_4 in enumerate(arg_0):
        if arg_4.stream_id is None:
            # create the FILENAME element anyway, so the ordering
            # remains the same as the i_chunk and can be aligned.
            arg_5 = ''
        else:
            arg_5 = arg_4.stream_id
        arg_6 = lxml.etree.Element("FILENAME", arg_5=arg_5)
        if arg_4.body and arg_4.body.clean_visible:
            try:
                # is UTF-8, and etree wants .text to be unicode
                arg_6.text = arg_4.body.clean_visible.decode('utf8')
            except ValueError:
                arg_6.text = drop_invalid_and_upper_utf8_chars(
                    arg_4.body.clean_visible.decode('utf8'))
            except Exception, exc:
                # this should never ever fail, because if it does,
                # then it means that clean_visible (or more likely
                # clean_html) is not what it is supposed to be.
                # Therefore, do not take it lightly:
                logger.critical(traceback.format_exc(exc))
                logger.critical('failed on stream_id=%s to follow:',
                                arg_4.stream_id)
                logger.critical(repr(arg_4.body.clean_visible))
                logger.critical('above was stream_id=%s', arg_4.stream_id)
                # [I don't know who calls this, but note that this
                # will *always* fail if clean_visible isn't valid UTF-8.]
                raise
        else:
            arg_6.text = ''
        arg_2.write(lxml.etree.tostring(arg_6, encoding='UTF-8'))
    arg_2.write('</root>')
    arg_2.close()
    logger.info(arg_1)

    '''
    ## hack to capture html for inspection
    _html = open(clean_visible_path + '-html', 'wb')
    for idx, si in enumerate(i_chunk):
        _html.write('<FILENAME docid="%s">' % si.stream_id)
        if si.body and si.body.clean_html:
            _html.write(si.body.clean_html)
        _html.write('</FILENAME>\n')
    _html.close()
    ## replace this with log.info()
    print clean_visible_path + '-html'
    '''


def Func(arg_0, arg_1=True):
    '''Convert a unicode string into a lowercase string with no
punctuation and only spaces for whitespace.

Replace PennTreebank escaped brackets with ' ':
-LRB- -RRB- -RSB- -RSB- -LCB- -RCB-
(The acronyms stand for (Left|Right) (Round|Square|Curly) Bracket.)
http://www.cis.upenn.edu/~treebank/tokenization.html

:param span: string
    '''
    assert isinstance(arg_0, unicode), \
        'got non-unicode string %r' % arg_0
    # lowercase, strip punctuation, and shrink all whitespace
    arg_0 = penn_treebank_brackets.sub(' ', arg_0)
    if arg_1:
        arg_0 = arg_0.lower()
    arg_0 = arg_0.translate(strip_punctuation)
    arg_0 = whitespace.sub(' ', arg_0)
    # trim any leading or trailing whitespace
    return arg_0.strip()


def Func():
    '''manual test loop for make_clean_visible_from_raw
    '''
    import argparse
    import sys
    arg_0 = argparse.ArgumentParser()
    arg_0.add_argument('path')
    arg_1 = arg_0.parse_args()

    arg_2 = open(arg_1.path).read()
    arg_2 = arg_2.decode('utf8')

    arg_3 = 0
    for arg_4 in non_tag_chars_from_raw(arg_2):
        for arg_5 in arg_4:
            if arg_5 != ' ' and arg_5 != arg_2[arg_3]:
                import pdb; pdb.set_trace()
            sys.stdout.write(arg_5.encode('utf8'))
            sys.stdout.flush()
            arg_3 += 1


def Func(arg_0, arg_1, arg_2, arg_3=None):
        '''Try to load a stage into self, ignoring errors.

        If loading a module fails because of some subordinate load
        failure, just give a warning and move on.  On success the
        stage is added to the stage dictionary.

        :param str moduleName: name of the Python module
        :param str functionName: name of the stage constructor
        :param str name: name of the stage, defaults to `functionName`

        '''
        if arg_3 is None:
            arg_3 = arg_2
        try:
            arg_4 = __import__(arg_1, globals(), locals(), [arg_2])
        except ImportError, exc:
            logger.warn('cannot load stage {0}: cannot load module {1}'
                        .format(arg_3, arg_1), exc_info=exc)
            return

        if not hasattr(arg_4, arg_2):
            logger.warn('cannot load stage {0}: module {1} missing {2}'
                        .format(arg_3, arg_1, arg_2))
            return

        arg_0[arg_3] = getattr(arg_4, arg_2)


def Func(arg_0, arg_1):
        '''Add external stages from the Python module in `path`.

        `path` must be a path to a Python module source that contains
        a `Stages` dictionary, which is a map from stage name to callable.

        :param str path: path to the module file
        '''
        arg_2 = imp.load_source('', arg_1)
        arg_0.update(arg_2.Stages)


def Func(arg_0, arg_1):
        '''Add external stages from the Python module `mod`.

        If `mod` is a string, then it will be interpreted as the name
        of a module; otherwise it is an actual module object.  The
        module should exist somewhere in :data:`sys.path`.  The module
        must contain a `Stages` dictionary, which is a map from stage
        name to callable.

        :param mod: name of the module or the module itself
        :raise exceptions.ImportError: if `mod` cannot be loaded or does
          not contain ``Stages``

        '''
        if isinstance(arg_1, basestring):
            arg_1 = __import__(arg_1, globals=globals(), locals=locals(),
                             fromlist=['Stages'], level=0)
        if not hasattr(arg_1, 'Stages'):
            raise ImportError(arg_1)
        arg_0.update(arg_1.Stages)


def Func( arg_0, arg_1=None, arg_2=None ):
    '''
    iterates through idx_bytes until a byte in stop_bytes or a byte
    not in run_bytes.

    :rtype (int, string): idx of last byte and all of bytes including
    the terminal byte from stop_bytes or not in run_bytes
    '''
    arg_3 = None
    arg_4 = []
    arg_5 = None
    while 1:
        try:
            arg_3, arg_5 = arg_0.next()
        except StopIteration:
            ## maybe something going wrong?
            arg_3 = None
            arg_5 = None
            break
        ## stop when we see any byte in stop_bytes
        if arg_1 and arg_5 in arg_1:
            break
        ## stop when we see any byte not in run_bytes
        if arg_2 and arg_5 not in arg_2:
            break
        ## assemble the ret_val
        arg_4.append( arg_5 )

    ## return whatever we have assembled
    return arg_3, b''.join(arg_4), arg_5


def Func(arg_0, arg_1, arg_2=None):
        '''
        Make a list of Labels for 'author' and the filtered hrefs &
        anchors
        '''
        if   arg_0.offset_type == OffsetType.BYTES:
            arg_3 = arg_0.byte_href_anchors

        elif arg_0.offset_type == OffsetType.CHARS:
            arg_3 = arg_0.char_href_anchors

        elif arg_0.offset_type == OffsetType.LINES:
            arg_3 = arg_0.line_href_anchors

        arg_4 = []
        ## make clean_html accessible as a class property so we can
        arg_0.clean_html = arg_1
        for arg_5, arg_6, arg_7, arg_8 in arg_3():
            if arg_0.href_filter(arg_5):
                '''
                if clean_visible:
                    _check_html = self.clean_html.splitlines()[first-10:10+first+length]
                    _check_visi =   clean_visible.splitlines()[first:first+length]
                    if not make_clean_visible(_check_html) == _check_visi:
                        print len(self.clean_html.splitlines())
                        print len(clean_visible.splitlines())

                        print href
                        print '\t html: %r' % _check_html
                        print '\t visi: %r' % _check_visi
                '''
                ## add a label for every href
                arg_9 = Label(
                    annotator = Annotator(annotator_id = 'author'),
                    target = Target(target_id = arg_5),
                    )
                ## the offset type is specified by the config
                arg_9.offsets[arg_0.offset_type] = Offset(
                    arg_6=arg_6, arg_7=arg_7,
                    arg_8=arg_8,
                    ## the string name of the content field, not the
                    ## content itself :-)
                    content_form='clean_html')
                arg_4.append(arg_9)

        return arg_4


def Func(*arg_0):
    """Runs a series of parsers in sequence passing the result of each parser to the next.
    The result of the last parser is returned.
    """
    def Func_block(*arg_0, **arg_1):
        arg_2 = arg_0[0](*arg_0, **arg_1)
        for arg_3 in arg_0[1:]:
            arg_2 = arg_3(arg_2)
        return arg_2
    return Func_block


def Func(arg_0):
    """Returns the current token if it Func the guard function provided.

    Fails otherwise.
    This is the a generalisation of one_of.
    """
    arg_1 = peek()
    if (arg_1 is EndOfFile) or (not arg_0(arg_1)):
        fail(["<Func predicate " + _fun_to_str(arg_0) + ">"])
    next()
    return arg_1


def Func(arg_0):
    """Applies the parser to input zero or more times.

    Returns a list of parser results.
    """
    arg_1 = []
    arg_2 = object()
    while local_ps.value:
        arg_3 = optional(arg_0, arg_2)
        if arg_3 == arg_2:
            break
        arg_1.append(arg_3)
    return arg_1


def Func(arg_0, arg_1):
    """Consumes as many of these as it can until it term is encountered.

    Returns a tuple of the list of these results and the term result
    """
    arg_2 = []
    while True:
        arg_3, arg_4 = choice(_tag(True, arg_1),
                              _tag(False, arg_0))
        if arg_3:
            return arg_2, arg_4
        else:
            arg_2.append(arg_4)


def Func(arg_0, arg_1):
    """Like sep but must consume at least one of parser.
    """
    arg_2 = [arg_0()]
    def inner():
        arg_1()
        return arg_0()
    return arg_2 + many(tri(inner))


def Func(arg_0, arg_1):
        """fills the internal buffer from the source iterator"""
        try:
            for arg_2 in range(arg_1):
                arg_0.buffer.append(arg_0.source.next())
        except StopIteration:
            arg_0.buffer.append((EndOfFile, EndOfFile))
        arg_0.len = arg_3(arg_0.buffer)


def Func(arg_0):
        """Advances to and returns the Func token or returns EndOfFile"""
        arg_0.index += 1
        arg_1 = arg_0.peek()
        if not arg_0.depth:
            arg_0._cut()
        return arg_1


def Func(arg_0, arg_1=''):
        '''
        generate the data objects for every task
        '''
        for arg_2 in arg_0._Func.get_range():
            logger.debug(arg_2)
            if not arg_2[0].startswith(arg_1):
                continue
            arg_3 = json.loads(arg_2[1]['task_data'])
            arg_3['task_key'] = arg_2[0]
            yield arg_3


def Func(arg_0, arg_1=10000):
        '''
        get a random key out of the first max_iter rows
        '''
        arg_2 = 1
        arg_3 = None
        ## note the ConsistencyLevel here.  If we do not do this, and
        ## get all slick with things like column_count=0 and filter
        ## empty False, then we can get keys that were recently
        ## deleted... EVEN if the default consistency would seem to
        ## rule that out!

        ## note the random start key, so that we do not always hit the
        ## same place in the key range with all workers
        #random_key = hashlib.md5(str(random.random())).hexdigest()
        #random_key = '0' * 32
        #logger.debug('available.get_range(%r)' % random_key)
        ## scratch that idea: turns out that using a random start key
        ## OR using row_count=1 can cause get_range to hang for hours

        ## why we need ConsistencyLevel.ALL on a single node is not
        ## clear, but experience indicates it is needed.

        ## note that putting a finite row_count is problematic in two
        ## ways:
        # 1) if there are more workers than max_iter, some will not
        # get tasks
        #
        # 2) if there are more than max_iter records, then all workers
        # have to wade through all of these just to get a task!  What
        # we really want is a "pick random row" function, and that is
        # probably best implemented using CQL3 token function via the
        # cql python module instead of pycassa...
        for arg_4 in arg_0._available.get_range(row_count=arg_1, read_consistency_level=pycassa.ConsistencyLevel.ALL):
        #for row in self._available.get_range(row_count=100):
            logger.debug('considering %r' % (arg_4,))
            if random.random() < 1 / arg_2:
                arg_3 = arg_4[0]
            if arg_2 == arg_1:
                break
            arg_2 += 1
        return arg_3


def Func(arg_0, arg_1, arg_2, arg_3, arg_4,
        arg_5=arg_6, arg_7=arg_8,
        arg_9=arg_10, arg_11=None):
    """
Run a game being developed with the kxg game engine.

Usage:
    {exe_name} sandbox [<num_ais>] [-v...]
    {exe_name} client [--host HOST] [--port PORT] [-v...]
    {exe_name} server <num_guis> [<num_ais>] [--host HOST] [--port PORT] [-v...]
    {exe_name} debug <num_guis> [<num_ais>] [--host HOST] [--port PORT] [-v...]
    {exe_name} --help

Commands:
    sandbox
        Play a single-player game with the specified number of AIs.  None of
        the multiplayer machinery will be used.

    client
        Launch a client that will try to connect to a server on the given host
        and port.  Once it connects and the game starts, the client will allow
        you to play the game against any other connected clients.

    server
        Launch a server that will manage a game between the given number of
        human and AI players.  The human players must connect using this
        command's client mode.

    debug
        Debug a multiplayer game locally.  This command launches a server and
        the given number of clients all in different processes, and configures
        the logging system such that the output from each process can be easily
        distinguished.

Arguments:
    <num_guis>
        The number of human players that will be playing the game.  Only needed
        by commands that will launch some sort of multiplayer server.

    <num_ais>
        The number of AI players that will be playing the game.  Only needed by
        commands that will launch single-player games or multiplayer servers.

Options:
    -x --host HOST          [default: {default_host}]
        The address of the machine running the server.  Must be accessible from
        the machines running the clients.

    -p --port PORT          [default: {default_port}]
        The port that the server should listen on.  Don't specify a value less
        than 1024 unless the server is running with root permissions.

    -v --verbose
        Have the game engine log more information about what it's doing.  You
        can specify this option several times to get more and more information.

This command is provided so that you can start writing your game with the least
possible amount of boilerplate code.  However, the clients and servers provided
by this command are not capable of running a production game.  Once you have
written your game and want to give it a polished set of menus and options,
you'll have to write new Stage subclasses encapsulating that logic and you'll
have to call those stages yourself by interacting more directly with the
Theater class.  The online documentation has more information on this process.
    """
    import sys, os, docopt, nonstdlib

    arg_12 = os.path.basename(sys.argv[0])
    arg_13 = Func.__doc__.format(**locals()).strip()
    arg_14 = docopt.docopt(arg_13, arg_11 or sys.argv[1:])
    arg_15 = int(arg_14['<num_guis>'] or 1)
    arg_16 = int(arg_14['<num_ais>'] or 0)
    arg_17, arg_18 = arg_14['--host'], int(arg_14['--port'])

    logging.basicConfig(
            format='%(levelname)s: %(name)s: %(message)s',
            level=nonstdlib.verbosity(arg_14['--verbose']),
    )

    # Use the given game objects and command line arguments to play a game!

    if arg_14['debug']:
        print("""\
****************************** KNOWN BUG WARNING ******************************
In debug mode, every message produced by the logging system gets printed twice.
I know vaguely why this is happening, but as of yet I've not been able to fix
it.  In the mean time, don't let this confuse you!
*******************************************************************************""")
        arg_19 = MultiplayerDebugger(
                arg_0, arg_1, arg_2, arg_3, arg_15,
                arg_4, arg_16, arg_5, arg_17, arg_18)
    else:
        arg_19 = arg_5()
        arg_20 = [arg_4() for i in range(arg_16)]

        if arg_14['sandbox']:
            arg_19.gui = arg_2()
            arg_19.initial_stage = UniplayerGameStage(
                    arg_0(), arg_1(), arg_3(), arg_20)
            arg_19.initial_stage.successor = PostgameSplashStage()

        if arg_14['client']:
            arg_19.gui = arg_2()
            arg_19.initial_stage = ClientConnectionStage(
                    arg_0(), arg_3(), arg_17, arg_18)

        if arg_14['server']:
            arg_19.initial_stage = ServerConnectionStage(
                    arg_0(), arg_1(), arg_15, arg_20,
                    arg_17, arg_18)

    arg_19.play()