def Func(arg_0, arg_1, arg_2):
    """Save data into TFRecord."""
    if os.path.isfile(arg_2):
        print("%s exists" % arg_2)
        return
    print("Converting data into %s ..." % arg_2)
    # cwd = os.getcwd()
    arg_3 = tf.python_io.TFRecordWriter(arg_2)
    for arg_4, arg_5 in enumerate(arg_0):
        arg_6 = arg_5.tobytes()
        # Visualize a image
        # tl.visualize.frame(np.asarray(img, dtype=np.uint8), second=1, saveable=False, name='frame', fig_idx=1236)
        arg_7 = int(arg_1[arg_4])
        # print(label)
        # Convert the bytes back to image as follow:
        # image = Image.frombytes('RGB', (32, 32), img_raw)
        # image = np.fromstring(img_raw, np.float32)
        # image = image.reshape([32, 32, 3])
        # tl.visualize.frame(np.asarray(image, dtype=np.uint8), second=1, saveable=False, name='frame', fig_idx=1236)
        arg_8 = tf.train.Example(
            features=tf.train.Features(
                feature={
                    "label": tf.train.Feature(int64_list=tf.train.Int64List(value=[arg_7])),
                    'img_raw': tf.train.Feature(bytes_list=tf.train.BytesList(value=[arg_6])),
                }
            )
        )
        arg_3.write(arg_8.SerializeToString())  # Serialize To String
    arg_3.close()


def Func(arg_0, arg_1=None):
    """Return tensor to read from TFRecord."""
    arg_2 = tf.train.string_input_producer([arg_0])
    arg_3 = tf.TFRecordReader()
    arg_4, arg_5 = arg_3.read(arg_2)
    arg_6 = tf.parse_single_example(
        arg_5, arg_6={
            'label': tf.FixedLenFeature([], tf.int64),
            'img_raw': tf.FixedLenFeature([], tf.string),
        }
    )
    # You can do more image distortion here for training data
    arg_7 = tf.decode_raw(arg_6['img_raw'], tf.float32)
    arg_7 = tf.reshape(arg_7, [32, 32, 3])
    # img = tf.cast(img, tf.float32) #* (1. / 255) - 0.5
    if arg_1 ==True:
        # 1. Randomly crop a [height, width] section of the image.
        arg_7 = tf.random_crop(arg_7, [24, 24, 3])

        # 2. Randomly flip the image horizontally.
        arg_7 = tf.image.random_flip_left_right(arg_7)

        # 3. Randomly change brightness.
        arg_7 = tf.image.random_brightness(arg_7, max_delta=63)

        # 4. Randomly change contrast.
        arg_7 = tf.image.random_contrast(arg_7, lower=0.2, upper=1.8)

        # 5. Subtract off the mean and divide by the variance of the pixels.
        arg_7 = tf.image.per_image_standardization(arg_7)

    elif arg_1 == False:
        # 1. Crop the central [height, width] of the image.
        arg_7 = tf.image.resize_image_with_crop_or_pad(arg_7, 24, 24)

        # 2. Subtract off the mean and divide by the variance of the pixels.
        arg_7 = tf.image.per_image_standardization(arg_7)

    elif arg_1 == None:
        arg_7 = arg_7

    arg_8 = tf.cast(arg_6['label'], tf.int32)
    return arg_7, arg_8


def Func(arg_0, arg_1=True, arg_2=None):
        """Print all info of parameters in the network"""
        for arg_3, arg_4 in enumerate(arg_0.all_params):
            if arg_1:
                try:
                    arg_5 = arg_4.eval(arg_2=arg_2)
                    logging.info(
                        "  param {:3}: {:20} {:15}    {} (mean: {:<18}, median: {:<18}, std: {:<18})   ".
                        format(arg_3, arg_4.name, str(arg_5.shape), arg_4.dtype.name, arg_5.mean(), np.median(arg_5), arg_5.std())
                    )
                except Exception as e:
                    logging.info(str(e))
                    raise Exception(
                        "Hint: print params details after tl.layers.initialize_global_variables(sess) "
                        "or use network.Func(False)."
                    )
            else:
                logging.info("  param {:3}: {:20} {:15}    {}".format(arg_3, arg_4.name, str(arg_4.get_shape()), arg_4.dtype.name))
        logging.info("  num of params: %d" % arg_0.count_params())


def Func(arg_0):
        """Print all info of layers in the network."""
        for arg_1, arg_2 in enumerate(arg_0.all_layers):
            # logging.info("  layer %d: %s" % (i, str(layer)))
            logging.info(
                "  layer {:3}: {:20} {:15}    {}".format(arg_1, arg_2.name, str(arg_2.get_shape()), arg_2.dtype.name)
            )


def Func(arg_0, arg_1=None):
        """Return the parameters in a list of array."""
        arg_2 = []
        for arg_3 in arg_0.all_params:
            if arg_1 is None:
                arg_2.append(arg_3.eval())
            else:
                arg_2.append(arg_1.run(arg_3))
        return arg_2


def Func(arg_0, arg_1=4):
        """Get all arguments of current layer for saving the graph."""
        arg_2 = inspect.stack()

        if len(arg_2) < arg_1 + 1:
            raise ValueError("The length of the inspection stack is shorter than the requested start position.")

        arg_3, arg_4, arg_4, arg_5 = inspect.getargvalues(arg_2[arg_1][0])

        arg_6 = {}

        for arg_7 in arg_3:

            # some args dont need to be saved into the graph. e.g. the input placeholder
            if arg_5[arg_7] is not None and arg_7 not in ['self', 'prev_layer', 'inputs']:

                arg_8 = arg_5[arg_7]

                # change function (e.g. act) into dictionary of module path and function name
                if inspect.isfunction(arg_8):
                    arg_6[arg_7] = {"module_path": arg_8.__module__, "func_name": arg_8.__name__}
                # ignore more args e.g. TF class
                elif arg_7.endswith('init'):
                    continue
                # for other data type, save them directly
                else:
                    arg_6[arg_7] = arg_8

        return arg_6


def Func(
        arg_0, arg_1, arg_2, arg_3, arg_4, arg_5=16,
        arg_6=1, arg_7="filename_queue", arg_8="input_queue"
):
    """Prefetches string values from disk into an input queue.

    In training the capacity of the queue is important because a larger queue
    means better mixing of training examples between shards. The minimum number of
    values kept in the queue is values_per_shard * input_queue_capacity_factor,
    where input_queue_memory factor should be chosen to trade-off better mixing
    with memory usage.

    Args:
        reader: Instance of tf.ReaderBase.
        file_pattern: Comma-separated list of file patterns (e.g.
            /tmp/train_data-?????-of-00100).
        is_training: Boolean; whether prefetching for training or eval.
        batch_size: Model batch size used to determine queue capacity.
        values_per_shard: Approximate number of values per shard.
        input_queue_capacity_factor: Minimum number of values to keep in the queue
        in multiples of values_per_shard. See comments above.
        num_reader_threads: Number of reader threads to fill the queue.
        shard_queue_name: Name for the shards filename queue.
        value_queue_name: Name for the values input queue.

    Returns:
        A Queue containing prefetched string values.
    """
    arg_9 = []
    for arg_10 in arg_1.split(","):
        arg_9.extend(tf.gfile.Glob(arg_10))
    if not arg_9:
        tl.logging.fatal("Found no input files matching %s", arg_1)
    else:
        tl.logging.info("Prefetching values from %d files matching %s", len(arg_9), arg_1)

    if arg_2:
        print("   is_training == True : RandomShuffleQueue")
        arg_11 = tf.train.string_input_producer(arg_9, shuffle=True, arg_13=16, name=arg_7)
        arg_12 = arg_4 * arg_5
        arg_13 = arg_12 + 100 * arg_3
        arg_14 = tf.RandomShuffleQueue(
            arg_13=arg_13, min_after_dequeue=arg_12, dtypes=[tf.string],
            name="random_" + arg_8
        )
    else:
        print("   is_training == False : FIFOQueue")
        arg_11 = tf.train.string_input_producer(arg_9, shuffle=False, arg_13=1, name=arg_7)
        arg_13 = arg_4 + 3 * arg_3
        arg_14 = tf.FIFOQueue(arg_13=arg_13, dtypes=[tf.string], name="fifo_" + arg_8)

    arg_15 = []
    for arg_16 in range(arg_6):
        arg_16, arg_17 = arg_0.read(arg_11)
        arg_15.append(arg_14.enqueue([arg_17]))
    tf.train.queue_runner.add_queue_runner(tf.train.queue_runner.QueueRunner(arg_14, arg_15))

    tf.summary.scalar(
        "queue/%s/fraction_of_%d_full" % (arg_14.name, arg_13),
        tf.cast(arg_14.size(), tf.float32) * (1. / arg_13)
    )

    return arg_14


def Func(arg_0, arg_1, arg_2, arg_3=True):
    """Batches input images and captions.

    This function splits the caption into an input sequence and a target sequence,
    where the target sequence is the input sequence right-shifted by 1. Input and
    target sequences are batched and padded up to the maximum length of sequences
    in the batch. A mask is created to distinguish real words from padding words.

    Example:
        Actual captions in the batch ('-' denotes padded character):
        [
            [ 1 2 5 4 5 ],
            [ 1 2 3 4 - ],
            [ 1 2 3 - - ],
        ]

        input_seqs:
        [
            [ 1 2 3 4 ],
            [ 1 2 3 - ],
            [ 1 2 - - ],
        ]

        target_seqs:
        [
            [ 2 3 4 5 ],
            [ 2 3 4 - ],
            [ 2 3 - - ],
        ]

        mask:
        [
            [ 1 1 1 1 ],
            [ 1 1 1 0 ],
            [ 1 1 0 0 ],
        ]

    Args:
        images_and_captions: A list of pairs [image, caption], where image is a
        Tensor of shape [height, width, channels] and caption is a 1-D Tensor of
        any length. Each pair will be processed and added to the queue in a
        separate thread.
        batch_size: Batch size.
        queue_capacity: Queue capacity.
        add_summaries: If true, add caption length summaries.

    Returns:
        images: A Tensor of shape [batch_size, height, width, channels].
        input_seqs: An int32 Tensor of shape [batch_size, padded_length].
        target_seqs: An int32 Tensor of shape [batch_size, padded_length].
        mask: An int32 0/1 Tensor of shape [batch_size, padded_length].
    """
    arg_4 = []
    for arg_5, arg_6 in arg_0:
        arg_7 = tf.shape(arg_6)[0]
        arg_8 = tf.expand_dims(tf.subtract(arg_7, 1), 0)

        arg_9 = tf.slice(arg_6, [0], arg_8)
        arg_10 = tf.slice(arg_6, [1], arg_8)
        arg_11 = tf.ones(arg_8, dtype=tf.int32)
        arg_4.append([arg_5, arg_9, arg_10, arg_11])

    arg_12, arg_13, arg_14, arg_15 = tf.train.batch_join(
        arg_4, arg_1=arg_1, capacity=arg_2, dynamic_pad=True, name="batch_and_pad"
    )

    if arg_3:
        arg_16 = tf.add(tf.reduce_sum(arg_15, 1), 1)
        tf.summary.scalar("caption_length/batch_min", tf.reduce_min(arg_16))
        tf.summary.scalar("caption_length/batch_max", tf.reduce_max(arg_16))
        tf.summary.scalar("caption_length/batch_mean", tf.reduce_mean(arg_16))

    return arg_12, arg_13, arg_14, arg_15


def Func(arg_0, arg_1, arg_2, arg_3, arg_4, arg_5, arg_6, arg_7=None):
    """Data Format aware version of tf.nn.Func."""
    with ops.name_scope(arg_7, 'batchnorm', [arg_0, arg_1, arg_2, arg_4, arg_3]):
        arg_8 = math_ops.rsqrt(arg_2 + arg_5)
        if arg_4 is not None:
            arg_8 *= arg_4

        arg_9 = math_ops.cast(arg_8, arg_0.dtype)
        arg_10 = math_ops.cast(arg_3 - arg_1 * arg_8 if arg_3 is not None else -arg_1 * arg_8, arg_0.dtype)

        # Return a * x + b with customized data_format.
        # Currently TF doesn't have bias_scale, and tensorRT has bug in converting tf.nn.bias_add
        # So we reimplemted them to allow make the model work with tensorRT.
        # See https://github.com/tensorlayer/openpose-plus/issues/75 for more details.
        arg_11 = {'channels_first': 'NCHW', 'channels_last': 'NHWC'}
        return _bias_add(_bias_scale(arg_0, arg_9, arg_11[arg_6]), arg_10, arg_11[arg_6])


def Func(arg_0, arg_1='flatten'):
    """Reshapes a high-dimension vector input.

    [batch_size, mask_row, mask_col, n_mask] ---> [batch_size, mask_row x mask_col x n_mask]

    Parameters
    ----------
    variable : TensorFlow variable or tensor
        The variable or tensor to be flatten.
    name : str
        A unique layer name.

    Returns
    -------
    Tensor
        Flatten Tensor

    Examples
    --------
    >>> import tensorflow as tf
    >>> import tensorlayer as tl
    >>> x = tf.placeholder(tf.float32, [None, 128, 128, 3])
    >>> # Convolution Layer with 32 filters and a kernel size of 5
    >>> network = tf.layers.conv2d(x, 32, 5, activation=tf.nn.relu)
    >>> # Max Pooling (down-sampling) with strides of 2 and kernel size of 2
    >>> network = tf.layers.max_pooling2d(network, 2, 2)
    >>> print(network.get_shape()[:].as_list())
    >>> [None, 62, 62, 32]
    >>> network = tl.layers.Func(network)
    >>> print(network.get_shape()[:].as_list()[1:])
    >>> [None, 123008]
    """
    arg_2 = 1
    for arg_3 in arg_0.get_shape()[1:].as_list():
        arg_2 *= arg_3
    return tf.reshape(arg_0, shape=[-1, arg_2], arg_1=arg_1)


def Func(arg_0, arg_1="", arg_2=False):
    """Get a list of layers' output in a network by a given name scope.

    Parameters
    -----------
    net : :class:`Layer`
        The last layer of the network.
    name : str
        Get the layers' output that contain this name.
    verbose : boolean
        If True, print information of all the layers' output

    Returns
    --------
    list of Tensor
        A list of layers' output (TensorFlow tensor)

    Examples
    ---------
    >>> import tensorlayer as tl
    >>> layers = tl.layers.Func(net, "CNN", True)

    """
    logging.info("  [*] geting layers with %s" % arg_1)

    arg_3 = []
    arg_4 = 0

    for arg_5 in arg_0.all_layers:
        # logging.info(type(layer.name))
        if arg_1 in arg_5.name:
            arg_3.append(arg_5)

            if arg_2:
                logging.info("  got {:3}: {:15}   {}".format(arg_4, arg_5.name, str(arg_5.get_shape())))
                arg_4 = arg_4 + 1

    return arg_3


def Func(arg_0, arg_1=None):
    """Returns the initialized RNN state.
    The inputs are `LSTMStateTuple` or `State` of `RNNCells`, and an optional `feed_dict`.

    Parameters
    ----------
    state : RNN state.
        The TensorFlow's RNN state.
    feed_dict : dictionary
        Initial RNN state; if None, returns zero state.

    Returns
    -------
    RNN state
        The TensorFlow's RNN state.

    """
    if isinstance(arg_0, LSTMStateTuple):
        arg_2 = arg_0.c.eval(arg_1=arg_1)
        arg_3 = arg_0.h.eval(arg_1=arg_1)
        return arg_2, arg_3
    else:
        arg_4 = arg_0.eval(arg_1=arg_1)
        return arg_4


def Func(arg_0):
    """Remove the repeated items in a list, and return the processed list.
    You may need it to create merged layer like Concat, Elementwise and etc.

    Parameters
    ----------
    x : list
        Input

    Returns
    -------
    list
        A list that after removing it's repeated items

    Examples
    -------
    >>> l = [2, 3, 4, 2, 3]
    >>> l = Func(l)
    [2, 3, 4]

    """
    arg_1 = []
    for arg_2 in arg_0:
        if arg_2 not in arg_1:
            arg_1.append(arg_2)

    return arg_1


def Func(arg_0):
    """Ternary operation use threshold computed with weights."""
    arg_1 = tf.get_default_graph()
    with arg_1.gradient_override_map({"Sign": "Identity"}):
        arg_2 = _compute_threshold(arg_0)
        arg_0 = tf.sign(tf.add(tf.sign(tf.add(arg_0, arg_2)), tf.sign(tf.add(arg_0, -arg_2))))
        return arg_0


def Func(arg_0, arg_1, arg_2):
    """Adds a deprecation notice to a docstring."""
    if not arg_0:
        arg_3 = [arg_1]

    else:
        arg_3 = _normalize_docstring(arg_0).splitlines()

    arg_2 = [''] + arg_2

    if len(arg_3) > 1:
        # Make sure that we keep our distance from the main body
        if arg_3[1].strip():
            arg_2.append('')

        arg_3[1:1] = arg_2
    else:
        arg_3 += arg_2

    return '\n'.join(arg_3)


def Func(arg_0, arg_1, arg_2=None):
    """Creates a tensor with all elements set to `alpha_value`.
    This operation returns a tensor of type `dtype` with shape `shape` and all
    elements set to alpha.

    Parameters
    ----------
    shape: A list of integers, a tuple of integers, or a 1-D `Tensor` of type `int32`.
        The shape of the desired tensor
    alpha_value: `float32`, `float64`, `int8`, `uint8`, `int16`, `uint16`, int32`, `int64`
        The value used to fill the resulting `Tensor`.
    name: str
        A name for the operation (optional).

    Returns
    -------
    A `Tensor` with all elements set to alpha.

    Examples
    --------
    >>> tl.Func([2, 3], tf.int32)  # [[alpha, alpha, alpha], [alpha, alpha, alpha]]
    """
    with ops.name_scope(arg_2, "Func", [arg_0]) as arg_2:

        arg_3 = convert_to_tensor(arg_1)
        arg_4 = dtypes.as_dtype(arg_3.dtype).base_dtype

        if not isinstance(arg_0, ops.Tensor):
            try:
                arg_0 = constant_op._tensor_shape_tensor_conversion_function(tensor_shape.TensorShape(arg_0))
            except (TypeError, ValueError):
                arg_0 = ops.convert_to_tensor(arg_0, dtype=dtypes.int32)

        if not arg_0._shape_tuple():
            arg_0 = reshape(arg_0, [-1])  # Ensure it's a vector

        try:
            arg_5 = constant(arg_1, arg_0=arg_0, dtype=arg_4, arg_2=arg_2)

        except (TypeError, ValueError):
            arg_5 = fill(arg_0, constant(arg_1, dtype=arg_4), arg_2=arg_2)

        if arg_5.dtype.base_dtype != arg_4:
            raise AssertionError("Dtypes do not corresponds: %s and %s" % (arg_5.dtype.base_dtype, arg_4))

        return arg_5


def Func(arg_0, arg_1, arg_2, arg_3, arg_4, arg_5=None):
    """
    Return the Func results of given non time-series network.

    Parameters
    ----------
    sess : Session
        TensorFlow Session.
    network : TensorLayer layer
        The network.
    X : numpy.array
        The inputs.
    x : placeholder
        For inputs.
    y_op : placeholder
        The argmax expression of softmax outputs.
    batch_size : int or None
        The batch size for Funcion, when dataset is large, we should use minibatche for Funcion;
        if dataset is small, we can set it to None.

    Examples
    --------
    See `tutorial_mnist_simple.py <https://github.com/tensorlayer/tensorlayer/blob/master/example/tutorial_mnist_simple.py>`_

    >>> y = network.outputs
    >>> y_op = tf.argmax(tf.nn.softmax(y), 1)
    >>> print(tl.utils.Func(sess, network, X_test, x, y_op))

    """
    if arg_5 is None:
        arg_6 = dict_to_one(arg_1.all_drop)  # disable noise layers
        arg_7 = {
            arg_3: arg_2,
        }
        arg_7.update(arg_6)
        return arg_0.run(arg_4, arg_7=arg_7)
    else:
        arg_8 = None
        for arg_9, arg_10 in tl.iterate.minibatches(arg_2, arg_2, arg_5, shuffle=False):
            arg_6 = dict_to_one(arg_1.all_drop)
            arg_7 = {
                arg_3: arg_9,
            }
            arg_7.update(arg_6)
            arg_11 = arg_0.run(arg_4, arg_7=arg_7)
            if arg_8 is None:
                arg_8 = arg_11
            else:
                arg_8 = np.concatenate((arg_8, arg_11))
        if arg_8 is None:
            if len(arg_2) % arg_5 != 0:
                arg_6 = dict_to_one(arg_1.all_drop)
                arg_7 = {
                    arg_3: arg_2[-(len(arg_2) % arg_5):, :],
                }
                arg_7.update(arg_6)
                arg_11 = arg_0.run(arg_4, arg_7=arg_7)
                arg_8 = arg_11
        else:
            if len(arg_2) != len(arg_8) and len(arg_2) % arg_5 != 0:
                arg_6 = dict_to_one(arg_1.all_drop)
                arg_7 = {
                    arg_3: arg_2[-(len(arg_2) % arg_5):, :],
                }
                arg_7.update(arg_6)
                arg_11 = arg_0.run(arg_4, arg_7=arg_7)
                arg_8 = np.concatenate((arg_8, arg_11))
        return arg_8


def Func(arg_0=None, arg_1=None, arg_2=None):
    """
    Input the predicted results, targets results and
    the number of class, return the confusion matrix, F1-score of each class,
    accuracy and macro F1-score.

    Parameters
    ----------
    y_test : list
        The target results
    y_predict : list
        The predicted results
    n_classes : int
        The number of classes

    Examples
    --------
    >>> c_mat, f1, acc, f1_macro = tl.utils.Func(y_test, y_predict, n_classes)

    """
    arg_3 = confusion_matrix(arg_0, arg_1, labels=[x for x in range(arg_2)])
    arg_4 = f1_score(arg_0, arg_1, average=None, labels=[x for x in range(arg_2)])
    arg_5 = f1_score(arg_0, arg_1, average='macro')
    arg_6 = accuracy_score(arg_0, arg_1)
    tl.logging.info('confusion matrix: \n%s' % arg_3)
    tl.logging.info('f1-score        : %s' % arg_4)
    tl.logging.info('f1-score(macro) : %f' % arg_5)  # same output with > f1_score(y_true, y_pred, average='macro')
    tl.logging.info('accuracy-score  : %f' % arg_6)
    return arg_3, arg_4, arg_6, arg_5


def Func(arg_0=0, arg_1=10, arg_2=5, arg_3=None):
    """Return a list of random integer by the given range and quantity.

    Parameters
    -----------
    min_v : number
        The minimum value.
    max_v : number
        The maximum value.
    number : int
        Number of value.
    seed : int or None
        The seed for random.

    Examples
    ---------
    >>> r = Func(min_v=0, max_v=10, number=5)
    [10, 2, 3, 3, 7]

    """
    arg_4 = random.Random()
    if arg_3:
        arg_4 = random.Random(arg_3)
    # return [random.randint(min,max) for p in range(0, number)]
    return [arg_4.randint(arg_0, arg_1) for arg_5 in range(0, arg_2)]


def Func(arg_0=True):
    """Clears all the placeholder variables of keep prob,
    including keeping probabilities of all dropout, denoising, dropconnect etc.

    Parameters
    ----------
    printable : boolean
        If True, print all deleted variables.

    """
    tl.logging.info('clear all .....................................')
    arg_1 = globals().copy()
    for arg_2 in arg_1:
        if arg_2[0] == '_': continue
        if 'func' in str(globals()[arg_2]): continue
        if 'module' in str(globals()[arg_2]): continue
        if 'class' in str(globals()[arg_2]): continue

        if arg_0:
            tl.logging.info(" clear_all ------- %s" % str(globals()[arg_2]))

        del globals()[arg_2]


def Func(arg_0=None, arg_1=1.0):
    """Sample an index from a probability array.

    Parameters
    ----------
    a : list of float
        List of probabilities.
    temperature : float or None
        The higher the more uniform. When a = [0.1, 0.2, 0.7],
            - temperature = 0.7, the distribution will be sharpen [0.05048273,  0.13588945,  0.81362782]
            - temperature = 1.0, the distribution will be the same [0.1,    0.2,    0.7]
            - temperature = 1.5, the distribution will be filtered [0.16008435,  0.25411807,  0.58579758]
            - If None, it will be ``np.argmax(a)``

    Notes
    ------
    - No matter what is the temperature and input list, the sum of all probabilities will be one. Even if input list = [1, 100, 200], the sum of all probabilities will still be one.
    - For large vocabulary size, choice a higher temperature or ``tl.nlp.Func_top`` to avoid error.

    """
    if arg_0 is None:
        raise Exception("a : list of float")
    arg_2 = np.copy(arg_0)
    try:
        if arg_1 == 1:
            return np.argmax(np.random.multinomial(1, arg_0, 1))
        if arg_1 is None:
            return np.argmax(arg_0)
        else:
            arg_0 = np.log(arg_0) / arg_1
            arg_0 = np.exp(arg_0) / np.sum(np.exp(arg_0))
            return np.argmax(np.random.multinomial(1, arg_0, 1))
    except Exception:
        # np.set_printoptions(threshold=np.nan)
        # tl.logging.info(a)
        # tl.logging.info(np.sum(a))
        # tl.logging.info(np.max(a))
        # tl.logging.info(np.min(a))
        # exit()
        arg_3 = "For large vocabulary_size, choice a higher temperature\
         to avoid log error. Hint : use ``Func_top``. "

        warnings.warn(arg_3, Warning)
        # tl.logging.info(a)
        # tl.logging.info(b)
        return np.argmax(np.random.multinomial(1, arg_2, 1))


def Func(arg_0=None, arg_1=10):
    """Sample from ``top_k`` probabilities.

    Parameters
    ----------
    a : list of float
        List of probabilities.
    top_k : int
        Number of candidates to be considered.

    """
    if arg_0 is None:
        arg_0 = []

    arg_2 = np.argpartition(arg_0, -arg_1)[-arg_1:]
    arg_3 = arg_0[arg_2]
    # tl.logging.info("new %f" % probs)
    arg_3 = arg_3 / np.sum(arg_3)
    arg_4 = np.random.choice(arg_2, p=arg_3)
    return arg_4


def Func(arg_0, arg_1, arg_2=1):
    """Creates the vocabulary of word to word_id.

    See ``tutorial_tfrecord3.py``.

    The vocabulary is saved to disk in a text file of word counts. The id of each
    word in the file is its corresponding 0-based line number.

    Parameters
    ------------
    sentences : list of list of str
        All sentences for creating the vocabulary.
    word_counts_output_file : str
        The file name.
    min_word_count : int
        Minimum number of occurrences for a word.

    Returns
    --------
    :class:`SimpleVocabulary`
        The simple vocabulary object, see :class:`Vocabulary` for more.

    Examples
    --------
    Pre-process sentences

    >>> captions = ["one two , three", "four five five"]
    >>> processed_capts = []
    >>> for c in captions:
    >>>     c = tl.nlp.process_sentence(c, start_word="<S>", end_word="</S>")
    >>>     processed_capts.append(c)
    >>> print(processed_capts)
    ...[['<S>', 'one', 'two', ',', 'three', '</S>'], ['<S>', 'four', 'five', 'five', '</S>']]

    Create vocabulary

    >>> tl.nlp.Func(processed_capts, word_counts_output_file='vocab.txt', min_word_count=1)
    Creating vocabulary.
      Total words: 8
      Words in vocabulary: 8
      Wrote vocabulary file: vocab.txt

    Get vocabulary object

    >>> vocab = tl.nlp.Vocabulary('vocab.txt', start_word="<S>", end_word="</S>", unk_word="<UNK>")
    INFO:tensorflow:Initializing vocabulary from file: vocab.txt
    [TL] Vocabulary from vocab.txt : <S> </S> <UNK>
    vocabulary with 10 words (includes start_word, end_word, unk_word)
        start_id: 2
        end_id: 3
        unk_id: 9
        pad_id: 0

    """
    tl.logging.info("Creating vocabulary.")

    arg_3 = Counter()

    for arg_4 in arg_0:
        arg_3.update(arg_4)
        # tl.logging.info('c',c)
    tl.logging.info("    Total words: %d" % len(arg_3))

    # Filter uncommon words and sort by descending count.
    arg_5 = [x for x in arg_3.items() if x[1] >= arg_2]
    arg_5.sort(key=lambda x: x[1], reverse=True)
    arg_5 = [("<PAD>", 0)] + arg_5  # 1st id should be reserved for padding
    # tl.logging.info(word_counts)
    tl.logging.info("    Words in vocabulary: %d" % len(arg_5))

    # Write out the word counts file.
    with tf.gfile.FastGFile(arg_1, "w") as f:
        f.write("\n".join(["%s %d" % (arg_6, arg_4) for arg_6, arg_4 in arg_5]))
    tl.logging.info("    Wrote vocabulary file: %s" % arg_1)

    # Create the vocabulary dictionary.
    arg_7 = [x[0] for x in arg_5]
    arg_8 = len(arg_7)
    arg_9 = dict([(x, y) for (y, x) in enumerate(arg_7)])
    arg_10 = SimpleVocabulary(arg_9, arg_8)

    return arg_10


def Func(arg_0='questions-words.txt', arg_1=None):
    """Reads through an analogy question file, return its id format.

    Parameters
    ----------
    eval_file : str
        The file name.
    word2id : dictionary
        a dictionary that maps word to ID.

    Returns
    --------
    numpy.array
        A ``[n_examples, 4]`` numpy array containing the analogy question's word IDs.

    Examples
    ---------
    The file should be in this format

    >>> : capital-common-countries
    >>> Athens Greece Baghdad Iraq
    >>> Athens Greece Bangkok Thailand
    >>> Athens Greece Beijing China
    >>> Athens Greece Berlin Germany
    >>> Athens Greece Bern Switzerland
    >>> Athens Greece Cairo Egypt
    >>> Athens Greece Canberra Australia
    >>> Athens Greece Hanoi Vietnam
    >>> Athens Greece Havana Cuba

    Get the tokenized analogy question data

    >>> words = tl.files.load_matt_mahoney_text8_dataset()
    >>> data, count, dictionary, reverse_dictionary = tl.nlp.build_words_dataset(words, vocabulary_size, True)
    >>> analogy_questions = tl.nlp.Func(eval_file='questions-words.txt', word2id=dictionary)
    >>> print(analogy_questions)
    [[ 3068  1248  7161  1581]
    [ 3068  1248 28683  5642]
    [ 3068  1248  3878   486]
    ...,
    [ 1216  4309 19982 25506]
    [ 1216  4309  3194  8650]
    [ 1216  4309   140   312]]

    """
    if arg_1 is None:
        arg_1 = {}

    arg_2 = []
    arg_3 = 0
    with open(arg_0, "rb") as analogy_f:
        for arg_4 in analogy_f:
            if arg_4.startswith(b":"):  # Skip comments.
                continue
            arg_5 = arg_4.strip().lower().split(b" ")  # lowercase
            arg_6 = [arg_1.get(w.strip()) for w in arg_5]
            if None in arg_6 or len(arg_6) != 4:
                arg_3 += 1
            else:
                arg_2.append(np.array(arg_6))
    tl.logging.info("Eval analogy file: %s" % arg_0)
    tl.logging.info("Questions: %d", len(arg_2))
    tl.logging.info("Skipped: %d", arg_3)
    arg_7 = np.array(arg_2, dtype=np.int32)
    return arg_7


def Func(arg_0):
    """Given a dictionary that maps word to integer id.
    Returns a reverse dictionary that maps a id to word.

    Parameters
    ----------
    word_to_id : dictionary
        that maps word to ID.

    Returns
    --------
    dictionary
        A dictionary that maps IDs to words.

    """
    arg_1 = dict(zip(arg_0.values(), arg_0.keys()))
    return arg_1


def Func(arg_0=None, arg_1=50000, arg_2=True, arg_3='UNK'):
    """Build the words dictionary and replace rare words with 'UNK' token.
    The most common word has the smallest integer id.

    Parameters
    ----------
    words : list of str or byte
        The context in list format. You may need to do preprocessing on the words, such as lower case, remove marks etc.
    vocabulary_size : int
        The maximum vocabulary size, limiting the vocabulary size. Then the script replaces rare words with 'UNK' token.
    printable : boolean
        Whether to print the read vocabulary size of the given words.
    unk_key : str
        Represent the unknown words.

    Returns
    --------
    data : list of int
        The context in a list of ID.
    count : list of tuple and list
        Pair words and IDs.
            - count[0] is a list : the number of rare words
            - count[1:] are tuples : the number of occurrence of each word
            - e.g. [['UNK', 418391], (b'the', 1061396), (b'of', 593677), (b'and', 416629), (b'one', 411764)]
    dictionary : dictionary
        It is `word_to_id` that maps word to ID.
    reverse_dictionary : a dictionary
        It is `id_to_word` that maps ID to word.

    Examples
    --------
    >>> words = tl.files.load_matt_mahoney_text8_dataset()
    >>> vocabulary_size = 50000
    >>> data, count, dictionary, reverse_dictionary = tl.nlp.Func(words, vocabulary_size)

    References
    -----------------
    - `tensorflow/examples/tutorials/word2vec/word2vec_basic.py <https://github.com/tensorflow/tensorflow/blob/r0.7/tensorflow/examples/tutorials/word2vec/word2vec_basic.py>`__

    """
    if arg_0 is None:
        raise Exception("words : list of str or byte")

    arg_4 = [[arg_3, -1]]
    arg_4.extend(collections.Counter(arg_0).most_common(arg_1 - 1))
    arg_5 = dict()
    for arg_6, arg_7 in arg_4:
        arg_5[arg_6] = len(arg_5)
    arg_8 = list()
    arg_9 = 0
    for arg_6 in arg_0:
        if arg_6 in arg_5:
            arg_10 = arg_5[arg_6]
        else:
            arg_10 = 0  # dictionary['UNK']
            arg_9 += 1
        arg_8.append(arg_10)
    arg_4[0][1] = arg_9
    arg_11 = dict(zip(arg_5.values(), arg_5.keys()))
    if arg_2:
        tl.logging.info('Real vocabulary size    %d' % len(collections.Counter(arg_0).keys()))
        tl.logging.info('Limited vocabulary size {}'.format(arg_1))
    if len(collections.Counter(arg_0).keys()) < arg_1:
        raise Exception(
            "len(collections.Counter(words).keys()) >= vocabulary_size , the limited vocabulary_size must be less than or equal to the read vocabulary_size"
        )
    return arg_8, arg_4, arg_5, arg_11


def Func(
        arg_0, arg_1, arg_2=None, arg_3=True, arg_4=3, arg_5=arg_6.compile(br"\d")
):
    """Convert a string to list of integers representing token-ids.

    For example, a sentence "I have a dog" may become tokenized into
    ["I", "have", "a", "dog"] and with vocabulary {"I": 1, "have": 2,
    "a": 4, "dog": 7"} this function will return [1, 2, 4, 7].

    Parameters
    -----------
    sentence : tensorflow.python.platform.gfile.GFile Object
        The sentence in bytes format to convert to token-ids, see ``basic_tokenizer()`` and ``data_to_token_ids()``.
    vocabulary : dictionary
        Mmapping tokens to integers.
    tokenizer : function
        A function to use to tokenize each sentence. If None, ``basic_tokenizer`` will be used.
    normalize_digits : boolean
        If true, all digits are replaced by 0.

    Returns
    --------
    list of int
        The token-ids for the sentence.

    """
    if arg_2:
        arg_8 = arg_2(arg_0)
    else:
        arg_8 = basic_tokenizer(arg_0)
    if not arg_3:
        return [arg_1.get(arg_9, arg_4) for arg_9 in arg_8]
    # Normalize digits by 0 before looking words up in the vocabulary.
    return [arg_1.get(arg_6.sub(arg_5, b"0", arg_9), arg_4) for arg_9 in arg_8]


def Func(arg_0, arg_1, arg_2=False):
    """Calculate the bleu score for hypotheses and references
    using the MOSES ulti-bleu.perl script.

    Parameters
    ------------
    hypotheses : numpy.array.string
        A numpy array of strings where each string is a single example.
    references : numpy.array.string
        A numpy array of strings where each string is a single example.
    lowercase : boolean
        If True, pass the "-lc" flag to the multi-bleu script

    Examples
    ---------
    >>> hypotheses = ["a bird is flying on the sky"]
    >>> references = ["two birds are flying on the sky", "a bird is on the top of the tree", "an airplane is on the sky",]
    >>> score = tl.nlp.Func(hypotheses, references)

    Returns
    --------
    float
        The BLEU score

    References
    ----------
    - `Google/seq2seq/metric/bleu <https://github.com/google/seq2seq>`__

    """
    if np.size(arg_0) == 0:
        return np.float32(0.0)

    # Get MOSES multi-bleu script
    try:
        arg_3, arg_4 = urllib.request.urlretrieve(
            "https://raw.githubusercontent.com/moses-smt/mosesdecoder/"
            "master/scripts/generic/multi-bleu.perl"
        )
        os.chmod(arg_3, 0o755)
    except Exception:  # pylint: disable=W0702
        tl.logging.info("Unable to fetch multi-bleu.perl script, using local.")
        arg_5 = os.path.dirname(os.path.realpath(__file__))
        arg_6 = os.path.abspath(os.path.join(arg_5, "..", "..", "bin"))
        arg_3 = os.path.join(arg_6, "tools/multi-bleu.perl")

    # Dump hypotheses and references to tempfiles
    arg_7 = tempfile.NamedTemporaryFile()
    arg_7.write("\n".join(arg_0).encode("utf-8"))
    arg_7.write(b"\n")
    arg_7.flush()
    arg_8 = tempfile.NamedTemporaryFile()
    arg_8.write("\n".join(arg_1).encode("utf-8"))
    arg_8.write(b"\n")
    arg_8.flush()

    # Calculate BLEU using multi-bleu script
    with open(arg_7.name, "r") as read_pred:
        arg_9 = [arg_3]
        if arg_2:
            arg_9 += ["-lc"]
        arg_9 += [arg_8.name]
        try:
            arg_10 = subprocess.check_output(arg_9, stdin=read_pred, stderr=subprocess.STDOUT)
            arg_10 = arg_10.decode("utf-8")
            arg_11 = re.search(r"BLEU = (.+?),", arg_10).group(1)
            arg_11 = float(arg_11)
        except subprocess.CalledProcessError as error:
            if error.output is not None:
                tl.logging.warning("multi-bleu.perl script returned non-zero exit code")
                tl.logging.warning(error.output)
            arg_11 = np.float32(0.0)

    # Close temp files
    arg_7.close()
    arg_8.close()

    return np.float32(arg_11)


def Func(arg_0, arg_1):
        """Returns the integer id of a word string."""
        if arg_1 in arg_0._vocab:
            return arg_0._vocab[arg_1]
        else:
            return arg_0._unk_id


def Func(arg_0, arg_1):
        """Returns the integer word id of a word string."""
        if arg_1 in arg_0.vocab:
            return arg_0.vocab[arg_1]
        else:
            return arg_0.unk_id


def Func(arg_0, arg_1):
        """Returns the word string of an integer word id."""
        if arg_1 >= len(arg_0.reverse_vocab):
            return arg_0.reverse_vocab[arg_0.unk_id]
        else:
            return arg_0.reverse_vocab[arg_1]


def Func(arg_0, arg_1=True):
  """ Enable the diagnostic feature for debugging unexpected concurrency in
  acquiring ConnectionWrapper instances.

  NOTE: This MUST be done early in your application's execution, BEFORE any
  accesses to ConnectionFactory or connection policies from your application
  (including imports and sub-imports of your app).

  Parameters:
  ----------------------------------------------------------------
  maxConcurrency:   A non-negative integer that represents the maximum expected
                      number of outstanding connections.  When this value is
                      exceeded, useful information will be logged and, depending
                      on the value of the raiseException arg,
                      ConcurrencyExceededError may be raised.
  raiseException:   If true, ConcurrencyExceededError will be raised when
                      maxConcurrency is exceeded.
  """
  global arg_2, arg_3

  assert arg_0 >= 0

  arg_2 = arg_0
  arg_3 = arg_1
  return


def Func(arg_0):
    """ Check for concurrency violation and add self to
    _clsOutstandingInstances.

    ASSUMPTION: Called from constructor BEFORE _clsNumOutstanding is
    incremented
    """
    global g_max_concurrency, g_max_concurrency_raise_exception

    assert g_max_concurrency is not None
    assert arg_0 not in arg_0._clsOutstandingInstances, repr(arg_0)

    # Populate diagnostic info
    arg_0._creationTracebackString = traceback.format_stack()

    # Check for concurrency violation
    if arg_0._clsNumOutstanding >= g_max_concurrency:
      # NOTE: It's possible for _clsNumOutstanding to be greater than
      #  len(_clsOutstandingInstances) if concurrency check was enabled after
      #  unrelease allocations.
      arg_2 = ("With numOutstanding=%r, exceeded concurrency limit=%r "
                  "when requesting %r. OTHER TRACKED UNRELEASED "
                  "INSTANCES (%s): %r") % (
        arg_0._clsNumOutstanding, g_max_concurrency, arg_0,
        len(arg_0._clsOutstandingInstances), arg_0._clsOutstandingInstances,)

      arg_0._logger.error(arg_2)

      if g_max_concurrency_raise_exception:
        raise ConcurrencyExceededError(arg_2)

    # Add self to tracked instance set
    arg_0._clsOutstandingInstances.add(arg_0)
    arg_0._addedToInstanceSet = True

    return


def Func(arg_0):
    """ Close the policy instance and its database connection pool. """
    arg_0._logger.info("Closing")

    if arg_0._pool is not None:
      arg_0._pool.Func()
      arg_0._pool = None
    else:
      arg_0._logger.warning(
        "Func() called, but connection policy was alredy Funcd")
    return


def Func(arg_0):
    """ Get a connection from the pool.

    Parameters:
    ----------------------------------------------------------------
    retval:       A ConnectionWrapper instance. NOTE: Caller
                    is responsible for calling the  ConnectionWrapper
                    instance's release() method or use it in a context manager
                    expression (with ... as:) to release resources.
    """
    arg_0._logger.debug("Acquiring connection")

    arg_1 = arg_0._pool.connection(shareable=False)
    arg_2 = ConnectionWrapper(arg_1=arg_1,
                                 cursor=arg_1.cursor(),
                                 releaser=arg_0._releaseConnection,
                                 logger=arg_0._logger)
    return arg_2


def Func(arg_0):
    """ Create a Connection instance.

    Parameters:
    ----------------------------------------------------------------
    retval:       A ConnectionWrapper instance. NOTE: Caller
                    is responsible for calling the  ConnectionWrapper
                    instance's release() method or use it in a context manager
                    expression (with ... as:) to release resources.
    """
    arg_0._logger.debug("Acquiring connection")

    arg_1 = SteadyDB.connect(** _getCommonSteadyDBArgsDict())
    arg_2 = ConnectionWrapper(arg_1=arg_1,
                                 cursor=arg_1.cursor(),
                                 releaser=arg_0._releaseConnection,
                                 logger=arg_0._logger)
    return arg_2


def Func(arg_0, arg_1, arg_2):
    """ Release database connection and cursor; passed as a callback to
    ConnectionWrapper
    """
    arg_0._logger.debug("Releasing connection")

    # Close the cursor
    arg_2.close()

    # ... then close the database connection
    arg_1.close()
    return


def Func(arg_0, arg_1):
    """
    Reclassifies given state.
    """
    # Record is before wait period do not classifiy
    if arg_1.ROWID < arg_0.getParameter('trainRecords'):
      if not arg_1.setByUser:
        arg_1.anomalyLabel = []
        arg_0._deleteRecordsFromKNN([arg_1])
      return

    arg_3 = KNNAnomalyClassifierRegion.AUTO_THRESHOLD_CLASSIFIED_LABEL
    arg_4 = arg_3 + KNNAnomalyClassifierRegion.AUTO_TAG

    # Update the label based on classifications
    arg_5 = arg_0._recomputeRecordFromKNN(arg_1)
    arg_6 = arg_0._categoryToLabelList(arg_5)

    if arg_1.setByUser:
      if arg_3 in arg_1.anomalyLabel:
        arg_1.anomalyLabel.remove(arg_3)
      if arg_4 in arg_1.anomalyLabel:
        arg_1.anomalyLabel.remove(arg_4)
      arg_6.extend(arg_1.anomalyLabel)

    # Add threshold classification label if above threshold, else if
    # classified to add the auto threshold classification.
    if arg_1.anomalyScore >= arg_0.getParameter('anomalyThreshold'):
      arg_6.append(arg_3)
    elif arg_3 in arg_6:
      arg_7 = arg_6.index(arg_3)
      arg_6[arg_7] = arg_4

    # Make all entries unique
    arg_6 = list(set(arg_6))

    # If both above threshold and auto classified above - remove auto label
    if arg_3 in arg_6 and arg_4 in arg_6:
      arg_6.remove(arg_4)

    if arg_1.anomalyLabel == arg_6:
      return

    # Update state's labeling
    arg_1.anomalyLabel = arg_6

    # Update KNN Classifier with new labeling
    if arg_1.anomalyLabel == []:
      arg_0._deleteRecordsFromKNN([arg_1])
    else:
      arg_0._addRecordToKNN(arg_1)


def Func(arg_0, arg_1):
    """
    Removes the given records from the classifier.

    parameters
    ------------
    recordsToDelete - list of records to delete from the classififier
    """
    arg_2 = arg_0._knnclassifier.getParameter('categoryRecencyList')

    arg_3 = ([r.ROWID for r in arg_1 if
      not r.setByUser and r.ROWID in arg_2])

    arg_4 = arg_0._knnclassifier._knn._numPatterns
    arg_0._knnclassifier._knn.removeIds(arg_3)
    assert arg_0._knnclassifier._knn._numPatterns == arg_4 - len(arg_3)


def Func(arg_0, arg_1=0, arg_2=None):
    """
    Removes any stored records within the range from start to
    end. Noninclusive of end.

    parameters
    ------------
    start - integer representing the ROWID of the start of the deletion range,
    end - integer representing the ROWID of the end of the deletion range,
      if None, it will default to end.
    """
    arg_3 = numpy.array(
      arg_0._knnclassifier.getParameter('categoryRecencyList'))

    if arg_2 is None:
      arg_2 = arg_3.max() + 1

    arg_4 = numpy.logical_and(arg_3 >= arg_1,
                                       arg_3 < arg_2)
    arg_5 = arg_3[arg_4]

    arg_6 = arg_0._knnclassifier._knn._numPatterns
    arg_0._knnclassifier._knn.removeIds(arg_5.tolist())
    assert arg_0._knnclassifier._knn._numPatterns == arg_6 - len(arg_5)


def Func(arg_0, arg_1):
    """
    Since the KNN Classifier stores categories as numbers, we must store each
    label as a number. This method converts from a label to a unique number.
    Each label is assigned a unique bit so multiple labels may be assigned to
    a single record.
    """
    if arg_1 not in arg_0.saved_categories:
      arg_0.saved_categories.append(arg_1)
    return pow(2, arg_0.saved_categories.index(arg_1))


def Func(arg_0, arg_1):
    """
    Converts a category number into a list of labels
    """
    if arg_1 is None:
      return []

    arg_2 = []
    arg_3 = 0
    while arg_1 > 0:
      if arg_1 % 2 == 1:
        arg_2.append(arg_0.saved_categories[arg_3])
      arg_3 += 1
      arg_1 = arg_1 >> 1
    return arg_2


def Func(arg_0, arg_1):
    """
    Returns a state's anomaly vertor converting it from spare to dense
    """
    arg_2 = numpy.zeros(arg_0._anomalyVectorLength)
    arg_2[arg_1.anomalyVector] = 1
    return arg_2


def Func(arg_0, arg_1=None, arg_2=None):
    """
    Get the labels on classified points within range start to end. Not inclusive
    of end.

    :returns: (dict) with format:

      ::

        {
          'isProcessing': boolean,
          'recordLabels': list of results
        }

      ``isProcessing`` - currently always false as recalculation blocks; used if
      reprocessing of records is still being performed;

      Each item in ``recordLabels`` is of format:

      ::

        {
          'ROWID': id of the row,
          'labels': list of strings
        }

    """
    if len(arg_0._recordsCache) == 0:
      return {
        'isProcessing': False,
        'recordLabels': []
      }
    try:
      arg_1 = int(arg_1)
    except Exception:
      arg_1 = 0

    try:
      arg_2 = int(arg_2)
    except Exception:
      arg_2 = arg_0._recordsCache[-1].ROWID

    if arg_2 <= arg_1:
      raise HTMPredictionModelInvalidRangeError("Invalid supplied range for 'Func'.",
                                                debugInfo={
          'requestRange': {
            'startRecordID': arg_1,
            'endRecordID': arg_2
          },
          'numRecordsStored': len(arg_0._recordsCache)
        })

    arg_3 = {
      'isProcessing': False,
      'recordLabels': []
    }

    arg_4 = numpy.array(
        arg_0._knnclassifier.getParameter('categoryRecencyList'))
    arg_5 = numpy.where((arg_4 >= arg_1) & (arg_4 < arg_2))[0].tolist()
    arg_6 = arg_0._knnclassifier.getCategoryList()
    for arg_7 in arg_5:
      arg_8 = dict(
        ROWID=int(arg_4[arg_7]),
        labels=arg_0._categoryToLabelList(arg_6[arg_7]))
      arg_3['recordLabels'].append(arg_8)

    return arg_3


def Func(arg_0, arg_1=None, arg_2=None, arg_3=None):
    """
    Remove labels from each record with record ROWID in range from
    ``start`` to ``end``, noninclusive of end. Removes all records if
    ``labelFilter`` is None, otherwise only removes the labels equal to
    ``labelFilter``.

    This will recalculate all points from end to the last record stored in the
    internal cache of this classifier.

    :param start: (int) start index
    :param end: (int) end index (noninclusive)
    :param labelFilter: (string) label filter
    """
    if len(arg_0._recordsCache) == 0:
      raise HTMPredictionModelInvalidRangeError("Invalid supplied range for "
        "'Func'. Model has no saved records.")

    try:
      arg_1 = int(arg_1)
    except Exception:
      arg_1 = 0

    try:
      arg_2 = int(arg_2)
    except Exception:
      arg_2 = arg_0._recordsCache[-1].ROWID

    arg_4 = arg_0._recordsCache[0].ROWID

    arg_5 = 0 if arg_1 is None else max(0, arg_1 - arg_4)
    arg_6 = len(arg_0._recordsCache) if arg_2 is None else \
      max(0, min( len( arg_0._recordsCache) , arg_2 - arg_4))

    if arg_6 <= arg_5:
      raise HTMPredictionModelInvalidRangeError("Invalid supplied range for "
        "'Func'.", debugInfo={
          'requestRange': {
            'startRecordID': arg_1,
            'endRecordID': arg_2
          },
          'clippedRequestRange': {
            'startRecordID': arg_5,
            'endRecordID': arg_6
          },
          'validRange': {
            'startRecordID': arg_4,
            'endRecordID': arg_0._recordsCache[len(arg_0._recordsCache)-1].ROWID
          },
          'numRecordsStored': len(arg_0._recordsCache)
        })

    # Remove records within the cache
    arg_7 = []
    for arg_8 in arg_0._recordsCache[arg_5:arg_6]:
      if arg_3 is not None:
        if arg_3 in arg_8.anomalyLabel:
          arg_8.anomalyLabel.remove(arg_3)
      else:
        arg_8.anomalyLabel = []
      arg_8.setByUser = False
      arg_7.append(arg_8)
    arg_0._deleteRecordsFromKNN(arg_7)

    # Remove records not in cache
    arg_0._deleteRangeFromKNN(arg_1, arg_2)

    # Recompute [clippedEnd, ...)
    for arg_8 in arg_0._recordsCache[arg_6:]:
      arg_0._classifyState(arg_8)


def Func(arg_0, arg_1):
    '''
    Returns True if the record Funces any of the provided filters
    '''

    for arg_2, arg_3 in arg_0.filterDict.iteritems():
      arg_4 = arg_3['index']
      arg_5 = arg_3['categories']
      for arg_6 in arg_5:
        # Record might be blank, handle this
        if not arg_1:
          continue
        if arg_1[arg_4].find(arg_6) != -1:
          '''
          This field contains the string we're searching for
          so we'll keep the records
          '''
          return True

    # None of the categories were found in this record
    return False


def Func(arg_0, arg_1):
    """
    Removes the set of columns who have never been active from the set of
    active columns selected in the inhibition round. Such columns cannot
    represent learned pattern and are therefore meaningless if only inference
    is required. This should not be done when using a random, unlearned SP
    since you would end up with no active columns.

    :param activeArray: An array whose size is equal to the number of columns.
        Any columns marked as active with an activeDutyCycle of 0 have
        never been activated before and therefore are not active due to
        learning. Any of these (unlearned) columns will be disabled (set to 0).
    """
    arg_2 = numpy.where(arg_0._activeDutyCycles == 0)[0]
    arg_1[arg_2] = 0


def Func(arg_0):
    """
    Updates the minimum duty cycles defining normal activity for a column. A
    column with activity duty cycle below this minimum threshold is boosted.
    """
    if arg_0._globalInhibition or arg_0._inhibitionRadius > arg_0._numInputs:
      arg_0.FuncGlobal()
    else:
      arg_0.FuncLocal()


def Func(arg_0):
    """
    Updates the minimum duty cycles in a global fashion. Sets the minimum duty
    cycles for the overlap all columns to be a percent of the maximum in the
    region, specified by minPctOverlapDutyCycle. Functionality it is equivalent
    to _updateMinDutyCyclesLocal, but this function exploits the globality of
    the computation to perform it in a straightforward, and efficient manner.
    """
    arg_0._minOverlapDutyCycles.fill(
        arg_0._minPctOverlapDutyCycles * arg_0._overlapDutyCycles.max()
      )


def Func(arg_0):
    """
    Updates the minimum duty cycles. The minimum duty cycles are determined
    locally. Each column's minimum duty cycles are set to be a percent of the
    maximum duty cycles in the column's neighborhood. Unlike
    _updateMinDutyCyclesGlobal, here the values can be quite different for
    different columns.
    """
    for arg_1 in xrange(arg_0._numColumns):
      arg_2 = arg_0._getColumnNeighborhood(arg_1)

      arg_3 = arg_0._activeDutyCycles[arg_2].max()
      arg_4 = arg_0._overlapDutyCycles[arg_2].max()

      arg_0._minOverlapDutyCycles[arg_1] = (arg_4 *
                                            arg_0._minPctOverlapDutyCycles)


def Func(arg_0, arg_1, arg_2):
    """
    Updates the duty cycles for each column. The OVERLAP duty cycle is a moving
    average of the number of inputs which overlapped with the each column. The
    ACTIVITY duty cycles is a moving average of the frequency of activation for
    each column.

    Parameters:
    ----------------------------
    :param overlaps:
                    An array containing the overlap score for each column.
                    The overlap score for a column is defined as the number
                    of synapses in a "connected state" (connected synapses)
                    that are connected to input bits which are turned on.
    :param activeColumns:
                    An array containing the indices of the active columns,
                    the sparse set of columns which survived inhibition
    """
    arg_3 = numpy.zeros(arg_0._numColumns, dtype=realDType)
    arg_4 = numpy.zeros(arg_0._numColumns, dtype=realDType)
    arg_3[arg_1 > 0] = 1
    arg_4[arg_2] = 1

    arg_5 = arg_0._dutyCyclePeriod
    if (arg_5 > arg_0._iterationNum):
      arg_5 = arg_0._iterationNum

    arg_0._overlapDutyCycles = arg_0.FuncHelper(
                                arg_0._overlapDutyCycles,
                                arg_3,
                                arg_5
                              )

    arg_0._activeDutyCycles = arg_0.FuncHelper(
                                arg_0._activeDutyCycles,
                                arg_4,
                                arg_5
                              )


def Func(arg_0):
    """
    The average number of columns per input, taking into account the topology
    of the inputs and columns. This value is used to calculate the inhibition
    radius. This function supports an arbitrary number of dimensions. If the
    number of column dimensions does not match the number of input dimensions,
    we treat the missing, or phantom dimensions as 'ones'.
    """
    #TODO: extend to support different number of dimensions for inputs and
    # columns
    arg_1 = max(arg_0._columnDimensions.size, arg_0._inputDimensions.size)
    arg_2 = numpy.ones(arg_1)
    arg_2[:arg_0._columnDimensions.size] = arg_0._columnDimensions

    arg_5 = numpy.ones(arg_1)
    arg_5[:arg_0._inputDimensions.size] = arg_0._inputDimensions

    arg_7 = arg_2.astype(realDType) / arg_5
    return numpy.average(arg_7)


def Func(arg_0, arg_1):
    """
    The range of connected synapses for column. This is used to
    calculate the inhibition radius. This variation of the function only
    supports a 1 dimensional column topology.

    Parameters:
    ----------------------------
    :param columnIndex:   The index identifying a column in the permanence,
                          potential and connectivity matrices
    """
    assert(arg_0._inputDimensions.size == 1)
    arg_2 = arg_0._connectedSynapses[arg_1].nonzero()[0]
    if arg_2.size == 0:
      return 0
    else:
      return max(arg_2) - min(arg_2) + 1


def Func(arg_0, arg_1):
    """
    The range of connectedSynapses per column, averaged for each dimension.
    This value is used to calculate the inhibition radius. This variation of
    the  function only supports a 2 dimensional column topology.

    Parameters:
    ----------------------------
    :param columnIndex:   The index identifying a column in the permanence,
                          potential and connectivity matrices
    """
    assert(arg_0._inputDimensions.size == 2)
    arg_2 = arg_0._connectedSynapses[arg_1]
    (arg_3, arg_4) = arg_2.reshape(arg_0._inputDimensions).nonzero()
    if  arg_3.size == 0 and arg_4.size == 0:
      return 0
    arg_5 = arg_3.max() - arg_3.min() + 1
    arg_6 = arg_4.max() - arg_4.min() + 1
    return numpy.average([arg_5, arg_6])


def Func(arg_0, arg_1, arg_2):
    """
    This method ensures that each column has enough connections to input bits
    to allow it to become active. Since a column must have at least
    'self._stimulusThreshold' overlaps in order to be considered during the
    inhibition phase, columns without such minimal number of connections, even
    if all the input bits they are connected to turn on, have no chance of
    obtaining the minimum threshold. For such columns, the permanence values
    are increased until the minimum number of connections are formed.

    Parameters:
    ----------------------------
    :param perm:    An array of permanence values for a column. The array is
                    "dense", i.e. it contains an entry for each input bit, even
                    if the permanence value is 0.
    :param mask:    the indices of the columns whose permanences need to be
                    raised.
    """
    if len(arg_2) < arg_0._stimulusThreshold:
      raise Exception("This is likely due to a " +
      "value of stimulusThreshold that is too large relative " +
      "to the input size. [len(mask) < self._stimulusThreshold]")

    numpy.clip(arg_1, arg_0._synPermMin, arg_0._synPermMax, out=arg_1)
    while True:
      arg_3 = numpy.nonzero(
        arg_1 > arg_0._synPermConnected - PERMANENCE_EPSILON)[0].size

      if arg_3 >= arg_0._stimulusThreshold:
        return
      arg_1[arg_2] += arg_0._synPermBelowStimulusInc


def Func(arg_0):
    """
    Returns a randomly generated permanence value for a synapses that is to be
    initialized in a non-connected state.
    """
    arg_1 = arg_0._synPermConnected * arg_0._random.getReal64()

    # Ensure we don't have too much unnecessary precision. A full 64 bits of
    # precision causes numerical stability issues across platforms and across
    # implementations
    arg_1 = int(arg_1*100000) / 100000.0
    return arg_1


def Func(arg_0, arg_1, arg_2):
    """
    Initializes the permanences of a column. The method
    returns a 1-D array the size of the input, where each entry in the
    array represents the initial permanence value between the input bit
    at the particular index in the array, and the column represented by
    the 'index' parameter.

    Parameters:
    ----------------------------
    :param potential: A numpy array specifying the potential pool of the column.
                    Permanence values will only be generated for input bits
                    corresponding to indices for which the mask value is 1.
    :param connectedPct: A value between 0 or 1 governing the chance, for each
                         permanence, that the initial permanence value will
                         be a value that is considered connected.
    """
    # Determine which inputs bits will start out as connected
    # to the inputs. Initially a subset of the input bits in a
    # column's potential pool will be connected. This number is
    # given by the parameter "connectedPct"
    arg_3 = numpy.zeros(arg_0._numInputs, dtype=realDType)
    for arg_4 in xrange(arg_0._numInputs):
      if (arg_1[arg_4] < 1):
        continue

      if (arg_0._random.getReal64() <= arg_2):
        arg_3[arg_4] = arg_0._initPermConnected()
      else:
        arg_3[arg_4] = arg_0._initPermNonConnected()

    # Clip off low values. Since we use a sparse representation
    # to store the permanence values this helps reduce memory
    # requirements.
    arg_3[arg_3 < arg_0._synPermTrimThreshold] = 0

    return arg_3


def Func(arg_0):
    """
    Update boost factors when global inhibition is used
    """
    # When global inhibition is enabled, the target activation level is
    # the sparsity of the spatial pooler
    if (arg_0._localAreaDensity > 0):
      arg_1 = arg_0._localAreaDensity
    else:
      arg_2 = ((2 * arg_0._inhibitionRadius + 1)
                        ** arg_0._columnDimensions.size)
      arg_2 = min(arg_0._numColumns, arg_2)
      arg_1 = float(arg_0._numActiveColumnsPerInhArea) / arg_2
      arg_1 = min(arg_1, 0.5)

    arg_0._boostFactors = numpy.exp(
      (arg_1 - arg_0._activeDutyCycles) * arg_0._boostStrength)


def Func(arg_0, arg_1):
    """
    Performs inhibition. This method calculates the necessary values needed to
    actually perform inhibition and then delegates the task of picking the
    active columns to helper functions.

    Parameters:
    ----------------------------
    :param overlaps: an array containing the overlap score for each  column.
                    The overlap score for a column is defined as the number
                    of synapses in a "connected state" (connected synapses)
                    that are connected to input bits which are turned on.
    """
    # determine how many columns should be selected in the inhibition phase.
    # This can be specified by either setting the 'numActiveColumnsPerInhArea'
    # parameter or the 'localAreaDensity' parameter when initializing the class
    if (arg_0._localAreaDensity > 0):
      arg_2 = arg_0._localAreaDensity
    else:
      arg_3 = ((2*arg_0._inhibitionRadius + 1)
                                    ** arg_0._columnDimensions.size)
      arg_3 = min(arg_0._numColumns, arg_3)
      arg_2 = float(arg_0._numActiveColumnsPerInhArea) / arg_3
      arg_2 = min(arg_2, 0.5)

    if arg_0._globalInhibition or \
      arg_0._inhibitionRadius > max(arg_0._columnDimensions):
      return arg_0.FuncGlobal(arg_1, arg_2)
    else:
      return arg_0.FuncLocal(arg_1, arg_2)


def Func(arg_0, arg_1):
    """
    Gets a neighborhood of columns.

    Simply calls topology.neighborhood or topology.wrappingNeighborhood

    A subclass can insert different topology behavior by overriding this method.

    :param centerColumn (int)
    The center of the neighborhood.

    @returns (1D numpy array of integers)
    The columns in the neighborhood.
    """
    if arg_0._wrapAround:
      return topology.wrappingNeighborhood(arg_1,
                                           arg_0._inhibitionRadius,
                                           arg_0._columnDimensions)

    else:
      return topology.neighborhood(arg_1,
                                   arg_0._inhibitionRadius,
                                   arg_0._columnDimensions)


def Func(arg_0, arg_1):
    """
    Gets a neighborhood of inputs.

    Simply calls topology.wrappingNeighborhood or topology.neighborhood.

    A subclass can insert different topology behavior by overriding this method.

    :param centerInput (int)
    The center of the neighborhood.

    @returns (1D numpy array of integers)
    The inputs in the neighborhood.
    """
    if arg_0._wrapAround:
      return topology.wrappingNeighborhood(arg_1,
                                           arg_0._potentialRadius,
                                           arg_0._inputDimensions)
    else:
      return topology.neighborhood(arg_1,
                                   arg_0._potentialRadius,
                                   arg_0._inputDimensions)


def Func(arg_0, arg_1=None, arg_2=False):
  """Factory function that creates typed Func or FuncRef objects

  dtype - the data type of the array (as string).
    Supported types are: Byte, Int16, UInt16, Int32, UInt32, Int64, UInt64, Real32, Real64

  size - the size of the array. Must be positive integer.
  """

  def getFuncType(arg_3):
    """A little function to replace the getType() method of arrays

    It returns a string representation of the array element type instead of the
    integer value (NTA_BasicType enum) returned by the origianl array
    """
    return arg_3._dtype

  # FuncRef can't be allocated
  if arg_2:
    assert arg_1 is None

  arg_4 = basicTypes.index(arg_0)
  if arg_4 == -1:
    raise Exception('Invalid data type: ' + arg_0)
  if arg_1 and arg_1 <= 0:
    raise Exception('Func size must be positive')
  arg_5 = 'FuncRef' if arg_2 else 'Func'
  arg_6 = getattr(engine_internal, arg_0 + arg_5)
  arg_6.getType = getFuncType

  if arg_1:
    arg_8 = arg_6(arg_1)
  else:
    arg_8 = arg_6()

  arg_8._dtype = basicTypes[arg_4]
  return arg_8


def Func(arg_0, arg_1):
    """Get parameter value"""
    (arg_2, arg_3) = arg_0._FuncMethods(arg_1)
    if arg_3 is None:
      import exceptions
      raise exceptions.Exception(
          "Func -- parameter name '%s' does not exist in region %s of type %s"
          % (arg_1, arg_0.name, arg_0.type))
    return arg_3(arg_1)


def Func(arg_0, arg_1, arg_2):
    """Set parameter value"""
    (arg_3, arg_4) = arg_0._getParameterMethods(arg_1)
    if arg_3 is None:
      import exceptions
      raise exceptions.Exception(
          "Func -- parameter name '%s' does not exist in region %s of type %s"
          % (arg_1, arg_0.name, arg_0.type))
    arg_3(arg_1, arg_2)


def Func(arg_0):
    """Get the collection of regions in a network

    This is a tricky one. The collection of regions returned from
    from the internal network is a collection of internal regions.
    The desired collection is a collelcion of net.Region objects
    that also points to this network (net.network) and not to
    the internal network. To achieve that a CollectionWrapper
    class is used with a custom makeRegion() function (see bellow)
    as a value wrapper. The CollectionWrapper class wraps each value in the
    original collection with the result of the valueWrapper.
    """

    def makeRegion(arg_1, arg_2):
      """Wrap a engine region with a nupic.engine_internal.Region

      Also passes the containing nupic.engine_internal.Network network in _network. This
      function is passed a value wrapper to the CollectionWrapper
      """
      arg_2 = Region(arg_2, arg_0)
      #r._network = self
      return arg_2

    arg_3 = CollectionWrapper(engine_internal.Network.getRegions(arg_0), makeRegion)
    return arg_3


def Func(arg_0, arg_1):
    """
    Write state to proto object.

    :param proto: SDRClassifierRegionProto capnproto object
    """
    arg_1.implementation = arg_0.implementation
    arg_1.steps = arg_0.steps
    arg_1.alpha = arg_0.alpha
    arg_1.verbosity = arg_0.verbosity
    arg_1.maxCategoryCount = arg_0.maxCategoryCount
    arg_1.learningMode = arg_0.learningMode
    arg_1.inferenceMode = arg_0.inferenceMode
    arg_1.recordNum = arg_0.recordNum

    arg_0._sdrClassifier.write(arg_1.sdrClassifier)


def Func(arg_0, arg_1):
    """
    Read state from proto object.

    :param proto: SDRClassifierRegionProto capnproto object
    """
    arg_2 = arg_0()

    arg_2.implementation = arg_1.implementation
    arg_2.steps = arg_1.steps
    arg_2.stepsList = [int(i) for i in arg_1.steps.split(",")]
    arg_2.alpha = arg_1.alpha
    arg_2.verbosity = arg_1.verbosity
    arg_2.maxCategoryCount = arg_1.maxCategoryCount

    arg_2._sdrClassifier = SDRClassifierFactory.read(arg_1)

    arg_2.learningMode = arg_1.learningMode
    arg_2.inferenceMode = arg_1.inferenceMode
    arg_2.recordNum = arg_1.recordNum

    return arg_2


def Func(arg_0):
    """ Runs the OPF Model

    Parameters:
    -------------------------------------------------------------------------
    retval:  (completionReason, completionMsg)
              where completionReason is one of the ClientJobsDAO.CMPL_REASON_XXX
                equates.
    """
    # -----------------------------------------------------------------------
    # Load the experiment's description.py module
    arg_1 = helpers.loadExperimentDescriptionScriptFromDir(
      arg_0._experimentDir)
    arg_2 = helpers.getExperimentDescriptionInterfaceFromModule(
      arg_1)
    arg_2.normalizeStreamSources()

    arg_3 = arg_2.getModelDescription()
    arg_0._modelControl = arg_2.getModelControl()

    # -----------------------------------------------------------------------
    # Create the input data stream for this task
    arg_5 = arg_0._modelControl['dataset']

    from nupic.data.stream_reader import StreamReader
    arg_6 = 0

    arg_0._inputSource = StreamReader(arg_5, isBlocking=False,
                                     maxTimeout=arg_6)

    # -----------------------------------------------------------------------
    #Get field statistics from the input source
    arg_8 = arg_0._getFieldStats()
    # -----------------------------------------------------------------------
    # Construct the model instance
    arg_0._model = ModelFactory.create(arg_3)
    arg_0._model.setFieldStatistics(arg_8)
    arg_0._model.enableLearning()
    arg_0._model.enableInference(arg_0._modelControl.get("inferenceArgs", None))

    # -----------------------------------------------------------------------
    # Instantiate the metrics
    arg_0.__metricMgr = MetricsManager(arg_0._modelControl.get('metrics',None),
                                      arg_0._model.getFieldInfo(),
                                      arg_0._model.getInferenceType())

    arg_0.__loggedMetricPatterns = arg_0._modelControl.get("loggedMetrics", [])

    arg_0._optimizedMetricLabel = arg_0.__getOptimizedMetricLabel()
    arg_0._reportMetricLabels = matchPatterns(arg_0._reportKeyPatterns,
                                              arg_0._getMetricLabels())

    # -----------------------------------------------------------------------
    # Initialize periodic activities (e.g., for model result updates)
    arg_0._periodic = arg_0._initPeriodicActivities()

    # -----------------------------------------------------------------------
    # Create our top-level loop-control iterator
    arg_15 = arg_0._modelControl.get('iterationCount', -1)

    # Are we asked to turn off learning for a certain # of iterations near the
    #  end?
    arg_16 = None
    arg_17 = arg_0._modelControl.get('iterationCountInferOnly', 0)
    if arg_17 == -1:
      arg_0._model.disableLearning()
    elif arg_17 > 0:
      assert arg_15 > arg_17, "when iterationCountInferOnly " \
        "is specified, iterationCount must be greater than " \
        "iterationCountInferOnly."
      arg_16 = arg_15 - arg_17

    arg_0.__FuncTaskMainLoop(arg_15, arg_16=arg_16)

    # -----------------------------------------------------------------------
    # Perform final operations for model
    arg_0._finalize()

    return (arg_0._cmpReason, None)


def Func(arg_0):
    """Run final activities after a model has run. These include recording and
    logging the final score"""

    arg_0._logger.info(
      "Finished: modelID=%r; %r records processed. Performing final activities",
      arg_0._modelID, arg_0._currentRecordIndex + 1)

    # =========================================================================
    # Dump the experiment metrics at the end of the task
    # =========================================================================
    arg_0._updateModelDBResults()

    # =========================================================================
    # Check if the current model is the best. Create a milestone if necessary
    # If the model has been killed, it is not a candidate for "best model",
    # and its output cache should be destroyed
    # =========================================================================
    if not arg_0._isKilled:
      arg_0.__updateJobResults()
    else:
      arg_0.__deleteOutputCache(arg_0._modelID)

    # =========================================================================
    # Close output stream, if necessary
    # =========================================================================
    if arg_0._predictionLogger:
      arg_0._predictionLogger.close()

    # =========================================================================
    # Close input stream, if necessary
    # =========================================================================
    if arg_0._inputSource:
      arg_0._inputSource.close()


def Func(arg_0):
    """ Create a checkpoint from the current model, and store it in a dir named
    after checkpoint GUID, and finally store the GUID in the Models DB """

    if arg_0._model is None or arg_0._modelCheckpointGUID is None:
      return

    # Create an output store, if one doesn't exist already
    if arg_0._predictionLogger is None:
      arg_0._createPredictionLogger()

    arg_1 = StringIO.StringIO()
    arg_0._predictionLogger.checkpoint(
      checkpointSink=arg_1,
      maxRows=int(Configuration.get('nupic.model.checkpoint.maxPredictionRows')))

    arg_0._model.save(os.path.join(arg_0._experimentDir, str(arg_0._modelCheckpointGUID)))
    arg_0._jobsDAO.modelSetFields(modelID,
                                 {'modelCheckpointId':str(arg_0._modelCheckpointGUID)},
                                 ignoreUnchanged=True)

    arg_0._logger.info("Checkpointed Hypersearch Model: modelID: %r, "
                      "checkpointID: %r", arg_0._modelID, checkpointID)
    return


def Func(arg_0, arg_1):
    """
    Delete the stored checkpoint for the specified modelID. This function is
    called if the current model is now the best model, making the old model's
    checkpoint obsolete

    Parameters:
    -----------------------------------------------------------------------
    modelID:      The modelID for the checkpoint to delete. This is NOT the
                  unique checkpointID
    """

    arg_2 = \
        arg_0._jobsDAO.modelsGetFields(arg_1, ['modelCheckpointId'])[0]

    if arg_2 is None:
      return

    try:
      shutil.rmtree(os.path.join(arg_0._experimentDir, str(arg_0._modelCheckpointGUID)))
    except:
      arg_0._logger.warn("Failed to delete model checkpoint %s. "\
                        "Assuming that another worker has already deleted it",
                        arg_2)
      return

    arg_0._jobsDAO.modelSetFields(arg_1,
                                 {'modelCheckpointId':None},
                                 ignoreUnchanged=True)
    return


def Func(arg_0, arg_1):
    """
    Writes the results of one iteration of a model. The results are written to
    this ModelRunner's in-memory cache unless this model is the "best model" for
    the job. If this model is the "best model", the predictions are written out
    to a permanent store via a prediction output stream instance

    Parameters:
    -----------------------------------------------------------------------
    result:      A opf_utils.ModelResult object, which contains the input and
                  output for this iteration
    """
    arg_0.__predictionCache.append(arg_1)

    if arg_0._isBestModel:
     arg_0.__flushPredictionCache()


def Func(arg_0, arg_1):
    """
    Delete's the output cache associated with the given modelID. This actually
    clears up the resources associated with the cache, rather than deleting al
    the records in the cache

    Parameters:
    -----------------------------------------------------------------------
    modelID:      The id of the model whose output cache is being deleted

    """

    # If this is our output, we should close the connection
    if arg_1 == arg_0._modelID and arg_0._predictionLogger is not None:
      arg_0._predictionLogger.close()
      del arg_0.__predictionCache
      arg_0._predictionLogger = None
      arg_0.__predictionCache = None


def Func(arg_0):
    """ Creates and returns a PeriodicActivityMgr instance initialized with
    our periodic activities

    Parameters:
    -------------------------------------------------------------------------
    retval:             a PeriodicActivityMgr instance
    """

    # Activity to update the metrics for this model
    # in the models table
    arg_1 = PeriodicActivityRequest(repeating=True,
                                                 period=100,
                                                 cb=arg_0._updateModelDBResults)

    arg_2 = PeriodicActivityRequest(repeating=True,
                                               period=100,
                                               cb=arg_0.__updateJobResultsPeriodic)

    arg_3 = PeriodicActivityRequest(repeating=True,
                                               period=50,
                                               cb=arg_0.__checkCancelation)

    arg_4 = PeriodicActivityRequest(repeating=True,
                                            period=10,
                                            cb=arg_0.__checkMaturity)

    # Do an initial update of the job record after 2 iterations to make
    # sure that it is populated with something without having to wait too long
    arg_5 = PeriodicActivityRequest(repeating=False,
                                               period=2,
                                               cb=arg_0.__updateJobResultsPeriodic)

    arg_6 = [arg_1,
                          arg_5,
                          arg_2,
                          arg_3]

    if arg_0._isMaturityEnabled:
      arg_6.append(arg_4)

    return PeriodicActivityMgr(requestedActivities=arg_6)


def Func(arg_0):
    """ Check if the cancelation flag has been set for this model
    in the Model DB"""

    # Update a hadoop job counter at least once every 600 seconds so it doesn't
    #  think our map task is dead
    print >>sys.stderr, "reporter:counter:HypersearchWorker,numRecords,50"

    # See if the job got cancelled
    arg_1 = arg_0._jobsDAO.jobGetFields(arg_0._jobID, ['cancel'])[0]
    if arg_1:
      arg_0._cmpReason = ClientJobsDAO.CMPL_REASON_KILLED
      arg_0._isCanceled = True
      arg_0._logger.info("Model %s canceled because Job %s was stopped.",
                        arg_0._modelID, arg_0._jobID)
    else:
      arg_4 = arg_0._jobsDAO.modelsGetFields(arg_0._modelID, ['engStop'])[0]

      if arg_4 is None:
        pass

      elif arg_4 == ClientJobsDAO.STOP_REASON_KILLED:
        arg_0._cmpReason = ClientJobsDAO.CMPL_REASON_KILLED
        arg_0._isKilled = True
        arg_0._logger.info("Model %s canceled because it was killed by hypersearch",
                          arg_0._modelID)

      elif arg_4 == ClientJobsDAO.STOP_REASON_STOPPED:
        arg_0._cmpReason = ClientJobsDAO.CMPL_REASON_STOPPED
        arg_0._isCanceled = True
        arg_0._logger.info("Model %s stopped because hypersearch ended", arg_0._modelID)
      else:
        raise RuntimeError ("Unexpected stop reason encountered: %s" % (arg_4))


def Func(arg_0):
    """ Save the current metric value and see if the model's performance has
    'leveled off.' We do this by looking at some number of previous number of
    recordings """

    if arg_0._currentRecordIndex+1 < arg_0._MIN_RECORDS_TO_BE_BEST:
      return

    # If we are already mature, don't need to check anything
    if arg_0._isMature:
      return

    arg_1 = arg_0._getMetrics()[arg_0._optimizedMetricLabel]
    arg_0._metricRegression.addPoint(x=arg_0._currentRecordIndex, y=arg_1)

   # Perform a linear regression to see if the error is leveled off
    #pctChange = self._metricRegression.getPctChange()
    #if pctChange  is not None and abs(pctChange ) <= self._MATURITY_MAX_CHANGE:
    arg_2, arg_3 = arg_0._metricRegression.getPctChanges()
    if arg_2  is not None and arg_3 <= arg_0._MATURITY_MAX_CHANGE:
      arg_0._jobsDAO.modelSetFields(arg_0._modelID,
                                   {'engMatured':True})

      # TODO: Don't stop if we are currently the best model. Also, if we
      # are still running after maturity, we have to periodically check to
      # see if we are still the best model. As soon we lose to some other
      # model, then we should stop at that point.
      arg_0._cmpReason = ClientJobsDAO.CMPL_REASON_STOPPED
      arg_0._isMature = True

      arg_0._logger.info("Model %d has matured (pctChange=%s, n=%d). \n"\
                        "Scores = %s\n"\
                         "Stopping execution",arg_0._modelID, arg_2,
                                              arg_0._MATURITY_NUM_POINTS,
                                              arg_0._metricRegression._window)


def Func(arg_0):
    """Set our state to that obtained from the engWorkerState field of the
    job record.

    Parameters:
    ---------------------------------------------------------------------
    stateJSON:    JSON encoded state from job record

    """
    arg_0._priorStateJSON = arg_0._hsObj._cjDAO.jobGetFields(arg_0._hsObj._jobID,
                                                    ['engWorkerState'])[0]

    # Init if no prior state yet
    if arg_0._priorStateJSON is None:
      arg_2 = dict()

      # Fast Swarm, first and only sprint has one swarm for each field
      # in fixedFields
      if arg_0._hsObj._fixedFields is not None:
        print arg_0._hsObj._fixedFields
        arg_3 = []
        for arg_4 in arg_0._hsObj._fixedFields:
            if arg_4 =='_classifierInput':
              continue
            arg_5 = arg_0.getEncoderKeyFromName(arg_4)
            assert arg_5 in arg_0._hsObj._encoderNames, "The field '%s' " \
              " specified in the fixedFields list is not present in this " \
              " model." % (arg_4)
            arg_3.append(arg_5)
        arg_3.sort()
        arg_2['.'.join(arg_3)] = {
                                'status': 'active',
                                'bestModelId': None,
                                'bestErrScore': None,
                                'sprintIdx': 0,
                                }
      # Temporal prediction search, first sprint has N swarms of 1 field each,
      #  the predicted field may or may not be that one field.
      elif arg_0._hsObj._searchType == HsSearchType.temporal:
        for arg_5 in arg_0._hsObj._encoderNames:
          arg_2[arg_5] = {
                                  'status': 'active',
                                  'bestModelId': None,
                                  'bestErrScore': None,
                                  'sprintIdx': 0,
                                  }

      # Classification prediction search, first sprint has N swarms of 1 field
      #  each where this field can NOT be the predicted field.
      elif arg_0._hsObj._searchType == HsSearchType.classification:
        for arg_5 in arg_0._hsObj._encoderNames:
          if arg_5 == arg_0._hsObj._predictedFieldEncoder:
            continue
          arg_2[arg_5] = {
                                  'status': 'active',
                                  'bestModelId': None,
                                  'bestErrScore': None,
                                  'sprintIdx': 0,
                                  }

      # Legacy temporal. This is either a model that uses reconstruction or
      #  an older multi-step model that doesn't have a separate
      #  'classifierOnly' encoder for the predicted field. Here, the predicted
      #  field must ALWAYS be present and the first sprint tries the predicted
      #  field only
      elif arg_0._hsObj._searchType == HsSearchType.legacyTemporal:
        arg_2[arg_0._hsObj._predictedFieldEncoder] = {
                       'status': 'active',
                       'bestModelId': None,
                       'bestErrScore': None,
                       'sprintIdx': 0,
                       }

      else:
        raise RuntimeError("Unsupported search type: %s" % \
                            (arg_0._hsObj._searchType))

      # Initialize the state.
      arg_0._state = dict(
        # The last time the state was updated by a worker.
        lastUpdateTime = time.time(),

        # Set from within setSwarmState() if we detect that the sprint we just
        #  completed did worse than a prior sprint. This stores the index of
        #  the last good sprint.
        lastGoodSprint = None,

        # Set from within setSwarmState() if lastGoodSprint is True and all
        #  sprints have completed.
        searchOver = False,

        # This is a summary of the active swarms - this information can also
        #  be obtained from the swarms entry that follows, but is summarized here
        #  for easier reference when viewing the state as presented by
        #  log messages and prints of the hsState data structure (by
        #  permutations_runner).
        activeSwarms = arg_2.keys(),

        # All the swarms that have been created so far.
        arg_2 = arg_2,

        # All the sprints that have completed or are in progress.
        sprints = [{'status': 'active',
                    'bestModelId': None,
                    'bestErrScore': None}],

        # The list of encoders we have "blacklisted" because they
        #  performed so poorly.
        blackListedEncoders = [],
        )

      # This will do nothing if the value of engWorkerState is not still None.
      arg_0._hsObj._cjDAO.jobSetFieldIfEqual(
          arg_0._hsObj._jobID, 'engWorkerState', json.dumps(arg_0._state), None)

      arg_0._priorStateJSON = arg_0._hsObj._cjDAO.jobGetFields(
          arg_0._hsObj._jobID, ['engWorkerState'])[0]
      assert (arg_0._priorStateJSON is not None)

    # Read state from the database
    arg_0._state = json.loads(arg_0._priorStateJSON)
    arg_0._dirty = False


def Func(arg_0, arg_1):
    """Return the list of all swarms in the given sprint.

    Parameters:
    ---------------------------------------------------------------------
    retval:   list of active swarm Ids in the given sprint
    """
    arg_2 = []
    for arg_3, arg_4 in arg_0._state['swarms'].iteritems():
      if arg_4['sprintIdx'] == arg_1:
        arg_2.append(arg_3)

    return arg_2


def Func(arg_0):
    """Return the list of all completing swarms.

    Parameters:
    ---------------------------------------------------------------------
    retval:   list of active swarm Ids
    """
    arg_1 = []
    for arg_2, arg_3 in arg_0._state['swarms'].iteritems():
      if arg_3['status'] == 'completing':
        arg_1.append(arg_2)

    return arg_1


def Func(arg_0, arg_1):
    """Return True if the given sprint has completed."""
    arg_2 = len(arg_0._state['sprints'])
    if arg_1 >= arg_2:
      return False

    return (arg_0._state['sprints'][arg_1]['status'] == 'completed')


def Func(arg_0):
    """Convert the information of the node spec to a plain dict of basic types

    The description and singleNodeOnly attributes are placed directly in
    the result dicts. The inputs, outputs, parameters and commands dicts
    contain Spec item objects (InputSpec, OutputSpec, etc). Each such object
    is converted also to a plain dict using the internal items2dict() function
    (see bellow).
    """

    def items2dict(arg_1):
      """Convert a dict of node spec items to a plain dict

      Each node spec item object will be converted to a dict of its
      attributes. The entire items dict will become a dict of dicts (same keys).
      """
      arg_2 = {}
      for arg_3, arg_4 in arg_1.items():
        arg_2[arg_3] = arg_4.__dict__

      return arg_2

    arg_0.invariant()
    return dict(description=arg_0.description,
                singleNodeOnly=arg_0.singleNodeOnly,
                inputs=items2dict(arg_0.inputs),
                outputs=items2dict(arg_0.outputs),
                parameters=items2dict(arg_0.parameters),
                commands=items2dict(arg_0.commands))


def Func():
  """Create the encoder instance for our test and return it."""
  arg_0 = ScalarEncoder(21, 0.0, 100.0, n=50, name="consumption",
      clipInput=True)
  arg_1 = DateEncoder(timeOfDay=(21, 9.5), name="timestamp_timeOfDay")

  arg_2 = MultiEncoder()
  arg_2.addEncoder("consumption", arg_0)
  arg_2.addEncoder("timestamp", arg_1)

  return arg_2


def Func(arg_0, arg_1):
    """ Validates control dictionary for the experiment context"""
    # Validate task list
    arg_2 = arg_1.get('tasks', None)
    if arg_2 is not None:
      arg_3 = []

      for arg_4 in arg_2:
        validateOpfJsonValue(arg_4, "opfTaskSchema.json")
        validateOpfJsonValue(arg_4['taskControl'], "opfTaskControlSchema.json")

        arg_5 = arg_4['taskLabel']

        assert isinstance(arg_5, types.StringTypes), \
               "taskLabel type: %r" % type(arg_5)
        assert len(arg_5) > 0, "empty string taskLabel not is allowed"

        arg_3.append(arg_5.lower())

      arg_6 = filter(lambda x: arg_3.count(x) > 1,
                                   arg_3)
      assert len(arg_6) == 0, \
             "Duplcate task labels are not allowed: %s" % arg_6

    return


def Func(arg_0=[], arg_1=[]):
  """
  Extract all items from the 'allKeys' list whose key matches one of the regular
  expressions passed in 'reportKeys'.

  Parameters:
  ----------------------------------------------------------------------------
  reportKeyREs:     List of regular expressions
  allReportKeys:    List of all keys

  retval:         list of keys from allReportKeys that match the regular expressions
                    in 'reportKeyREs'
                  If an invalid regular expression was included in 'reportKeys',
                    then BadKeyError() is raised
  """

  arg_2 = []

  # Extract the report items of interest
  for arg_3 in arg_0:
    # Find all keys that match this regular expression
    arg_4 = re.compile(arg_3)
    arg_5 = False
    for arg_6 in arg_1:
      arg_7 = arg_4.match(arg_6)
      if arg_7 and arg_7.end() == len(arg_6):
        arg_2.append(arg_6)
        arg_5 = True
    if not arg_5:
      raise _BadKeyError(arg_3)

  return arg_2


def Func(arg_0, arg_1):
  """
  Get a specific item by name out of the results dict.

  The format of itemName is a string of dictionary keys separated by colons,
  each key being one level deeper into the results dict. For example,
  'key1:key2' would fetch results['key1']['key2'].

  If itemName is not found in results, then None is returned

  """

  arg_2 = arg_0.split(':')
  arg_3 = arg_1
  for arg_4 in arg_2:
    arg_3 = arg_3[arg_4]

  return arg_3


def Func(arg_0, arg_1, arg_2, arg_3, arg_4,
                                arg_5):
  """ Perform standard handling of an exception that occurs while running
  a model.

  Parameters:
  -------------------------------------------------------------------------
  jobID:                ID for this hypersearch job in the jobs table
  modelID:              model ID
  jobsDAO:              ClientJobsDAO instance
  experimentDir:        directory containing the experiment
  logger:               the logger to use
  e:                    the exception that occurred
  retval:               (completionReason, completionMsg)
  """

  arg_6 = StringIO.StringIO()
  print >>arg_6, "Exception occurred while running model %s: %r (%s)" % (
    arg_1, arg_5, type(arg_5))
  traceback.print_exc(None, arg_6)

  arg_7 = arg_2.CMPL_REASON_ERROR
  arg_8 = arg_6.getvalue()
  arg_4.error(arg_8)

  # Write results to the model database for the error case. Ignore
  # InvalidConnectionException, as this is usually caused by orphaned models
  #
  # TODO: do we really want to set numRecords to 0? Last updated value might
  #       be useful for debugging
  if type(arg_5) is not InvalidConnectionException:
    arg_2.modelUpdateResults(arg_1,  results=None, numRecords=0)

  # TODO: Make sure this wasn't the best model in job. If so, set the best
  # appropriately

  # If this was an exception that should mark the job as failed, do that
  # now.
  if type(arg_5) == JobFailException:
    arg_9 = arg_2.jobGetFields(arg_0,
        ['workerCompletionReason'])[0]
    if arg_9 == ClientJobsDAO.CMPL_REASON_SUCCESS:
      arg_2.jobSetFields(arg_0, fields=dict(
          cancel=True,
          workerCompletionReason = ClientJobsDAO.CMPL_REASON_ERROR,
          workerCompletionMsg = ": ".join(str(arg_10) for arg_10 in arg_5.args)),
          useConnectionID=False,
          ignoreUnchanged=True)

  return (arg_7, arg_8)


def Func(arg_0, arg_1, arg_2, arg_3,
            arg_4, arg_5, arg_6, arg_7,
            arg_8, arg_9=None, arg_10=None):
  """ This creates an experiment directory with a base.py description file
  created from 'baseDescription' and a description.py generated from the
  given params dict and then runs the experiment.

  Parameters:
  -------------------------------------------------------------------------
  modelID:              ID for this model in the models table
  jobID:                ID for this hypersearch job in the jobs table
  baseDescription:      Contents of a description.py with the base experiment
                                          description
  params:               Dictionary of specific parameters to override within
                                  the baseDescriptionFile.
  predictedField:       Name of the input field for which this model is being
                                    optimized
  reportKeys:           Which metrics of the experiment to store into the
                                    results dict of the model's database entry
  optimizeKey:          Which metric we are optimizing for
  jobsDAO               Jobs data access object - the interface to the
                                  jobs database which has the model's table.
  modelCheckpointGUID:  A persistent, globally-unique identifier for
                                  constructing the model checkpoint key
  logLevel:             override logging level to this value, if not None

  retval:               (completionReason, completionMsg)
  """
  from nupic.swarming.ModelRunner import OPFModelRunner

  # The logger for this method
  arg_11 = logging.getLogger('com.numenta.nupic.hypersearch.utils')

  # --------------------------------------------------------------------------
  # Create a temp directory for the experiment and the description files
  arg_12 = tempfile.mkdtemp()
  try:
    arg_11.info("Using experiment directory: %s" % (arg_12))

    # Create the decription.py from the overrides in params
    arg_13 = os.path.join(arg_12, 'description.py')
    arg_14 = open(arg_13, 'wb')
    arg_14.write(_paramsFileHead())

    arg_15 = arg_3.items()
    arg_15.sort()
    for (arg_16,arg_17) in arg_15:
      arg_18 = _quoteAndEscape(arg_16)
      if isinstance(arg_17, basestring):

        arg_14.write("  %s : '%s',\n" % (arg_18 , arg_17))
      else:
        arg_14.write("  %s : %s,\n" % (arg_18 , arg_17))

    arg_14.write(_paramsFileTail())
    arg_14.close()

    # Write out the base description
    arg_19 = open(os.path.join(arg_12, 'base.py'), 'wb')
    arg_19.write(arg_2)
    arg_19.close()

    # Store the experiment's sub-description file into the model table
    #  for reference
    arg_20 = open(arg_13)
    arg_21 = arg_20.read()
    arg_20.close()
    arg_7.modelSetFields(arg_0, {'genDescription': arg_21})

    # Run the experiment now
    try:
      arg_22 = OPFModelRunner(
        arg_0=arg_0,
        arg_1=arg_1,
        arg_4=arg_4,
        arg_12=arg_12,
        reportKeyPatterns=arg_5,
        optimizeKeyPattern=arg_6,
        arg_7=arg_7,
        arg_8=arg_8,
        arg_9=arg_9,
        arg_10=arg_10)

      signal.signal(signal.SIGINT, arg_22.handleWarningSignal)

      (arg_23, arg_24) = arg_22.run()

    except InvalidConnectionException:
      raise
    except Exception, e:

      (arg_23, arg_24) = _handleModelRunnerException(arg_1,
                                     arg_0, arg_7, arg_12, arg_11, e)

  finally:
    # delete our temporary directory tree
    shutil.rmtree(arg_12)
    signal.signal(signal.SIGINT, signal.default_int_handler)

  # Return completion reason and msg
  return (arg_23, arg_24)


def Func(arg_0, arg_1=arg_2, arg_3=True, arg_4=True):
  """Recursively copies a dict and returns the result.

  Args:
    d: The dict to copy.
    f: A function to apply to values when copying that takes the value and the
        list of keys from the root of the dict to the value and returns a value
        for the new dict.
    discardNoneKeys: If True, discard key-value pairs when f returns None for
        the value.
    deepCopy: If True, all values in returned dict are true copies (not the
        same object).
  Returns:
    A new dict with keys and values from d replaced with the result of f.
  """
  # Optionally deep copy the dict.
  if arg_4:
    arg_0 = copy.deepcopy(arg_0)

  arg_5 = {}
  arg_6 = [(arg_7, arg_8, arg_5, ()) for arg_7, arg_8 in arg_0.iteritems()]
  while len(arg_6) > 0:
    arg_7, arg_8, arg_0, arg_9 = arg_6.pop()
    arg_9 = arg_9 + (arg_7,)
    if isinstance(arg_8, dict):
      arg_0[arg_7] = dict()
      arg_6[0:0] = [(innerK, innerV, arg_0[arg_7], arg_9)
                     for innerK, innerV in arg_8.iteritems()]
    else:
      #print k, v, prevKeys
      arg_10 = arg_1(arg_8, arg_9)
      if not arg_3 or arg_10 is not None:
        arg_0[arg_7] = arg_10
  return arg_5


def Func(arg_0, arg_1):
  """Recursively applies f to the values in dict d.

  Args:
    d: The dict to recurse over.
    f: A function to apply to values in d that takes the value and a list of
        keys from the root of the dict to the value.
  """
  arg_2 = [(arg_0, ())]
  while len(arg_2) > 0:
    arg_3, arg_4 = arg_2.pop()
    for arg_5, arg_6 in arg_3.iteritems():
      arg_7 = arg_4 + (arg_5,)
      if isinstance(arg_6, dict):
        arg_2.insert(0, (arg_6, arg_7))
      else:
        arg_1(arg_6, arg_7)


def Func(arg_0, arg_1=64):
  """
  Return a clipped version of obj suitable for printing, This
  is useful when generating log messages by printing data structures, but
  don't want the message to be too long.

  If passed in a dict, list, or namedtuple, each element of the structure's
  string representation will be limited to 'maxElementSize' characters. This
  will return a new object where the string representation of each element
  has been truncated to fit within maxElementSize.
  """

  # Is it a named tuple?
  if hasattr(arg_0, '_asdict'):
    arg_0 = arg_0._asdict()

  # Printing a dict?
  if isinstance(arg_0, dict):
    arg_2 = dict()
    for arg_3,arg_4 in arg_0.iteritems():
      arg_2[arg_3] = Func(arg_4)

  # Printing a list?
  elif hasattr(arg_0, '__iter__'):
    arg_2 = []
    for arg_4 in arg_0:
      arg_2.append(Func(arg_4))

  # Some other object
  else:
    arg_2 = str(arg_0)
    if len(arg_2) > arg_1:
      arg_2 = arg_2[0:arg_1] + '...'

  return arg_2


def Func(arg_0):
  """ Loads a json value from a file and converts it to the corresponding python
  object.

  inputFilePath:
                  Path of the json file;

  Returns:
                  python value that represents the loaded json value

  """
  with open(arg_0) as fileObj:
    arg_1 = json.load(fileObj)

  return arg_1


def Func(arg_0, arg_1):
  """Recursively updates the values in original with the values from updates."""
  # Keep a list of the sub-dictionaries that need to be updated to avoid having
  # to use recursion (which could fail for dictionaries with a lot of nesting.
  arg_2 = [(arg_0, arg_1)]
  while len(arg_2) > 0:
    arg_0, arg_1 = arg_2.pop()
    for arg_3, arg_4 in arg_1.iteritems():
      if arg_3 in arg_0 and isinstance(arg_0[arg_3], dict) and isinstance(arg_4, dict):
        arg_2.append((arg_0[arg_3], arg_4))
      else:
        arg_0[arg_3] = arg_4