def Func(
        arg_0=None,
        arg_1=None,
        arg_2=None,
        arg_3=None,
        arg_4=None,
        arg_5=None,
        arg_6=None,
        arg_7=None,
        arg_8=None,
        arg_9=True,
        arg_10=True,
        arg_11=True,
        arg_12=False,
        arg_13=None,
        arg_14=None
):
    """This wraps the function above to allow for loading previous state from a
    file.

    Parameters
    ----------

    use_saved_state : str or None
        This is the path to the saved state pickle file produced by a previous
        run of `runcp_producer_loop`. Will get all of the arguments to run
        another instance of the loop from that pickle file. If this is None, you
        MUST provide all of the appropriate arguments to that function.

    lightcurve_list : str or list of str or None
        This is either a string pointing to a file containing a list of light
        curves filenames to process or the list itself. The names must
        correspond to the full filenames of files stored on S3, including all
        prefixes, but not include the 's3://<bucket name>/' bit (these will be
        added automatically).

    input_queue : str or None
        This is the name of the SQS queue which will receive processing tasks
        generated by this function. The queue URL will automatically be obtained
        from AWS.

    input_bucket : str or None
        The name of the S3 bucket containing the light curve files to process.

    result_queue : str or None
        This is the name of the SQS queue that this function will listen to for
        messages from the workers as they complete processing on their input
        elements. This function will attempt to match input sent to the
        `input_queue` with results coming into the `result_queue` so it knows
        how many objects have been successfully processed. If this function
        receives task results that aren't in its own input queue, it will
        acknowledge them so they complete successfully, but not download them
        automatically. This handles leftover tasks completing from a previous
        run of this function.

    result_bucket : str or None
        The name of the S3 bucket which will receive the results from the
        workers.

    pfresult_list : list of str or None
        This is a list of periodfinder result pickle S3 URLs associated with
        each light curve. If provided, this will be used to add in phased light
        curve plots to each checkplot pickle. If this is None, the worker loop
        will produce checkplot pickles that only contain object information,
        neighbor information, and unphased light curves.

    runcp_kwargs : dict or None
        This is a dict used to pass any extra keyword arguments to the
        `lcproc.checkplotgen.runcp` function that will be run by the worker
        loop.

    process_list_slice : list or None
        This is used to index into the input light curve list so a subset of the
        full list can be processed in this specific run of this function.

        Use None for a slice index elem to emulate single slice spec behavior:

        process_list_slice = [10, None]  -> lightcurve_list[10:]
        process_list_slice = [None, 500] -> lightcurve_list[:500]

    purge_queues_when_done : bool or None
        If this is True, and this function exits (either when all done, or when
        it is interrupted with a Ctrl+C), all outstanding elements in the
        input/output queues that have not yet been acknowledged by workers or by
        this function will be purged. This effectively cancels all outstanding
        work.

    delete_queues_when_done : bool or None
        If this is True, and this function exits (either when all done, or when
        it is interrupted with a Ctrl+C'), all outstanding work items will be
        purged from the input/queues and the queues themselves will be deleted.

    download_when_done : bool or None
        If this is True, the generated checkplot pickle for each input work item
        will be downloaded immediately to the current working directory when the
        worker functions report they're done with it.

    save_state_when_done : bool or None
        If this is True, will save the current state of the work item queue and
        the work items acknowledged as completed to a pickle in the current
        working directory. Call the `Func` function
        below to resume processing from this saved state later.

    s3_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its S3 download operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    sqs_client : boto3.Client or None
        If None, this function will instantiate a new `boto3.Client` object to
        use in its SQS operations. Alternatively, pass in an existing
        `boto3.Client` instance to re-use it here.

    Returns
    -------

    dict or str
        Returns the current work state as a dict or str path to the generated
        work state pickle depending on if `save_state_when_done` is True.

    """

    if arg_0 is not None and os.path.exists(arg_0):

        with open(arg_0,'rb') as infd:
            arg_15 = pickle.load(infd)

        # run the producer loop using the saved state's todo list
        return runcp_producer_loop(
            arg_15['in_progress'],
            arg_15['args'][1],
            arg_15['args'][2],
            arg_15['args'][3],
            arg_15['args'][4],
            **arg_15['kwargs']
        )

    else:

        return runcp_producer_loop(
            arg_1,
            arg_2,
            arg_3,
            arg_4,
            arg_5,
            arg_6=arg_6,
            arg_7=arg_7,
            arg_8=arg_8,
            arg_9=arg_9,
            arg_10=arg_10,
            arg_11=arg_11,
            arg_12=arg_12,
            arg_13=arg_13,
            arg_14=arg_14
        )


def Func(arg_0):
    '''
    This is the worker for running checkplots.

    Parameters
    ----------

    task : tuple
        This is of the form: (pfpickle, outdir, lcbasedir, kwargs).

    Returns
    -------

    list of str
        The list of checkplot pickles returned by the `runcp` function.

    '''

    arg_1, arg_2, arg_3, arg_4 = arg_0

    try:

        return runcp(arg_1, arg_2, arg_3, **arg_4)

    except Exception as e:

        LOGEXCEPTION(' could not make checkplots for %s: %s' % (arg_1, e))
        return None


def Func(
        arg_0,
        arg_1,
        arg_2,
        arg_3=False,
        arg_4=None,
        arg_5=False,
        arg_6=None,
        arg_7=60.0,
        arg_8=None,
        arg_9=60.0,
        arg_10=5,
        arg_11=True,
        arg_12=None,
        arg_13=3.0,
        arg_14=10.0,
        arg_15=99,
        arg_16='hat-sql',
        arg_17=None,
        arg_18=None,
        arg_19=None,
        arg_20=None,
        arg_21=False,
        arg_22=None,
        arg_23=None,
        arg_24=None,
        arg_25=None,
        arg_26=None,
        arg_27=arg_28,
):
    '''This drives the parallel execution of `runcp` for a list of periodfinding
    result pickles.

    Parameters
    ----------

    pfpicklelist : list of str or list of Nones
        This is the list of the filenames of the period-finding result pickles
        to process. To make checkplots using the light curves directly, set this
        to a list of Nones with the same length as the list of light curve files
        that you provide in `lcfnamelist`.

    outdir : str
        The directory the checkplot pickles will be written to.

    lcbasedir : str
        The base directory that this function will look in to find the light
        curves pointed to by the period-finding result files. If you're using
        `lcfnamelist` to provide a list of light curve filenames directly, this
        arg is ignored.

    lcfnamelist : list of str or None
        If this is provided, it must be a list of the input light curve
        filenames to process. These can either be associated with each input
        period-finder result pickle, or can be provided standalone to make
        checkplots without phased LC plots in them. In the second case, you must
        set `pfpicklelist` to a list of Nones that matches the length of
        `lcfnamelist`.

    cprenorm : bool
        Set this to True if the light curves should be renormalized by
        `checkplot.checkplot_pickle`. This is set to False by default because we
        do our own normalization in this function using the light curve's
        registered normalization function and pass the normalized times, mags,
        errs to the `checkplot.checkplot_pickle` function.

    lclistpkl : str or dict
        This is either the filename of a pickle or the actual dict produced by
        lcproc.make_lclist. This is used to gather neighbor information.

    nbrradiusarcsec : float
        The radius in arcseconds to use for a search conducted around the
        coordinates of this object to look for any potential confusion and
        blending of variability amplitude caused by their proximity.

    maxnumneighbors : int
        The maximum number of neighbors that will have their light curves and
        magnitudes noted in this checkplot as potential blends with the target
        object.

    makeneighborlcs : bool
        If True, will make light curve and phased light curve plots for all
        neighbors found in the object collection for each input object.

    fast_mode : bool or float
        This runs the external catalog operations in a "fast" mode, with short
        timeouts and not trying to hit external catalogs that take a long time
        to respond.

        If this is set to True, the default settings for the external requests
        will then become::

                skyview_lookup = False
                skyview_timeout = 10.0
                skyview_retry_failed = False
                dust_timeout = 10.0
                gaia_submit_timeout = 7.0
                gaia_max_timeout = 10.0
                gaia_submit_tries = 2
                complete_query_later = False
                search_simbad = False

        If this is a float, will run in "fast" mode with the provided timeout
        value in seconds and the following settings::

                skyview_lookup = True
                skyview_timeout = fast_mode
                skyview_retry_failed = False
                dust_timeout = fast_mode
                gaia_submit_timeout = 0.66*fast_mode
                gaia_max_timeout = fast_mode
                gaia_submit_tries = 2
                complete_query_later = False
                search_simbad = False

    gaia_max_timeout : float
        Sets the timeout in seconds to use when waiting for the GAIA service to
        respond to our request for the object's information. Note that if
        `fast_mode` is set, this is ignored.

    gaia_mirror : str or None
        This sets the GAIA mirror to use. This is a key in the
        `services.gaia.GAIA_URLS` dict which defines the URLs to hit for each
        mirror.

    xmatchinfo : str or dict
        This is either the xmatch dict produced by the function
        `load_xmatch_external_catalogs` above, or the path to the xmatch info
        pickle file produced by that function.

    xmatchradiusarcsec : float
        This is the cross-matching radius to use in arcseconds.

    minobservations : int
        The minimum of observations the input object's mag/flux time-series must
        have for this function to plot its light curve and phased light
        curve. If the object has less than this number, no light curves will be
        plotted, but the checkplotdict will still contain all of the other
        information.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    timecols : list of str or None
        The timecol keys to use from the lcdict in generating this checkplot.

    magcols : list of str or None
        The magcol keys to use from the lcdict in generating this checkplot.

    errcols : list of str or None
        The errcol keys to use from the lcdict in generating this checkplot.

    skipdone : bool
        This indicates if this function will skip creating checkplots that
        already exist corresponding to the current `objectid` and `magcol`. If
        `skipdone` is set to True, this will be done.

    done_callback : Python function or None
        This is used to provide a function to execute after the checkplot
        pickles are generated. This is useful if you want to stream the results
        of checkplot making to some other process, e.g. directly running an
        ingestion into an LCC-Server collection. The function will always get
        the list of the generated checkplot pickles as its first arg, and all of
        the kwargs for runcp in the kwargs dict. Additional args and kwargs can
        be provided by giving a list in the `done_callbacks_args` kwarg and a
        dict in the `done_callbacks_kwargs` kwarg.

        NOTE: the function you pass in here should be pickleable by normal
        Python if you want to use it with the Func and Func_lcdir
        functions below.

    done_callback_args : tuple or None
        If not None, contains any args to pass into the `done_callback`
        function.

    done_callback_kwargs : dict or None
        If not None, contains any kwargs to pass into the `done_callback`
        function.

    liststartindex : int
        The index of the `pfpicklelist` (and `lcfnamelist` if provided) to start
        working at.

    maxobjects : int
        The maximum number of objects to process in this run. Use this with
        `liststartindex` to effectively distribute working on a large list of
        input period-finding result pickles (and light curves if `lcfnamelist`
        is also provided) over several sessions or machines.

    nworkers : int
        The number of parallel workers that will work on the checkplot
        generation process.

    Returns
    -------

    dict
        This returns a dict with keys = input period-finding pickles and vals =
        list of the corresponding checkplot pickles produced.

    '''

    # work around the Darwin segfault after fork if no network activity in
    # main thread bug: https://bugs.python.org/issue30385#msg293958
    if sys.platform == 'darwin':
        import requests
        requests.get('http://captive.apple.com/hotspot-detect.html')

    if not os.path.exists(arg_1):
        os.mkdir(arg_1)

    # handle the start and end indices
    if (arg_25 is not None) and (arg_26 is None):
        arg_0 = arg_0[arg_25:]
        if arg_4 is not None:
            arg_4 = arg_4[arg_25:]

    elif (arg_25 is None) and (arg_26 is not None):
        arg_0 = arg_0[:arg_26]
        if arg_4 is not None:
            arg_4 = arg_4[:arg_26]

    elif (arg_25 is not None) and (arg_26 is not None):
        arg_0 = (
            arg_0[arg_25:arg_25+arg_26]
        )
        if arg_4 is not None:
            arg_4 = arg_4[arg_25:arg_25+arg_26]

    # if the lcfnamelist is not provided, create a dummy
    if arg_4 is None:
        arg_4 = [None]*len(arg_0)

    arg_29 = [(x, arg_1, arg_2,
                 {'lcformat':arg_16,
                  'lcformatdir':arg_17,
                  'lcfname':y,
                  'timecols':arg_18,
                  'magcols':arg_19,
                  'errcols':arg_20,
                  'lclistpkl':arg_6,
                  'gaia_max_timeout':arg_7,
                  'gaia_mirror':arg_8,
                  'nbrradiusarcsec':arg_9,
                  'maxnumneighbors':arg_10,
                  'makeneighborlcs':arg_11,
                  'xmatchinfo':arg_12,
                  'xmatchradiusarcsec':arg_13,
                  'sigclip':arg_14,
                  'minobservations':arg_15,
                  'skipdone':arg_21,
                  'cprenorm':arg_5,
                  'fast_mode':arg_3,
                  'done_callback':arg_22,
                  'done_callback_args':arg_23,
                  'done_callback_kwargs':arg_24}) for
                x,y in zip(arg_0, arg_4)]

    arg_30 = []
    arg_31 = []

    with ProcessPoolExecutor(max_workers=arg_27) as executor:
        arg_30 = executor.map(runcp_worker, arg_29)

    arg_31 = [x for x in arg_30]

    executor.shutdown()
    return arg_31


def Func(arg_0,
                      arg_1,
                      arg_2,
                      arg_3='periodfinding-*.pkl*',
                      arg_4=None,
                      arg_5=False,
                      arg_6=60.0,
                      arg_7=5,
                      arg_8=True,
                      arg_9=False,
                      arg_10=60.0,
                      arg_11=None,
                      arg_12=None,
                      arg_13=3.0,
                      arg_14=99,
                      arg_15=10.0,
                      arg_16='hat-sql',
                      arg_17=None,
                      arg_18=None,
                      arg_19=None,
                      arg_20=None,
                      arg_21=False,
                      arg_22=None,
                      arg_23=None,
                      arg_24=None,
                      arg_25=None,
                      arg_26=32):

    '''This drives the parallel execution of `runcp` for a directory of
    periodfinding pickles.

    Parameters
    ----------

    pfpickledir : str
        This is the directory containing all of the period-finding pickles to
        process.

    outdir : str
        The directory the checkplot pickles will be written to.

    lcbasedir : str
        The base directory that this function will look in to find the light
        curves pointed to by the period-finding result files. If you're using
        `lcfnamelist` to provide a list of light curve filenames directly, this
        arg is ignored.

    pkpickleglob : str
        This is a UNIX file glob to select period-finding result pickles in the
        specified `pfpickledir`.

    lclistpkl : str or dict
        This is either the filename of a pickle or the actual dict produced by
        lcproc.make_lclist. This is used to gather neighbor information.

    cprenorm : bool
        Set this to True if the light curves should be renormalized by
        `checkplot.checkplot_pickle`. This is set to False by default because we
        do our own normalization in this function using the light curve's
        registered normalization function and pass the normalized times, mags,
        errs to the `checkplot.checkplot_pickle` function.

    nbrradiusarcsec : float
        The radius in arcseconds to use for a search conducted around the
        coordinates of this object to look for any potential confusion and
        blending of variability amplitude caused by their proximity.

    maxnumneighbors : int
        The maximum number of neighbors that will have their light curves and
        magnitudes noted in this checkplot as potential blends with the target
        object.

    makeneighborlcs : bool
        If True, will make light curve and phased light curve plots for all
        neighbors found in the object collection for each input object.

    fast_mode : bool or float
        This runs the external catalog operations in a "fast" mode, with short
        timeouts and not trying to hit external catalogs that take a long time
        to respond.

        If this is set to True, the default settings for the external requests
        will then become::

                skyview_lookup = False
                skyview_timeout = 10.0
                skyview_retry_failed = False
                dust_timeout = 10.0
                gaia_submit_timeout = 7.0
                gaia_max_timeout = 10.0
                gaia_submit_tries = 2
                complete_query_later = False
                search_simbad = False

        If this is a float, will run in "fast" mode with the provided timeout
        value in seconds and the following settings::

                skyview_lookup = True
                skyview_timeout = fast_mode
                skyview_retry_failed = False
                dust_timeout = fast_mode
                gaia_submit_timeout = 0.66*fast_mode
                gaia_max_timeout = fast_mode
                gaia_submit_tries = 2
                complete_query_later = False
                search_simbad = False

    gaia_max_timeout : float
        Sets the timeout in seconds to use when waiting for the GAIA service to
        respond to our request for the object's information. Note that if
        `fast_mode` is set, this is ignored.

    gaia_mirror : str or None
        This sets the GAIA mirror to use. This is a key in the
        `services.gaia.GAIA_URLS` dict which defines the URLs to hit for each
        mirror.

    xmatchinfo : str or dict
        This is either the xmatch dict produced by the function
        `load_xmatch_external_catalogs` above, or the path to the xmatch info
        pickle file produced by that function.

    xmatchradiusarcsec : float
        This is the cross-matching radius to use in arcseconds.

    minobservations : int
        The minimum of observations the input object's mag/flux time-series must
        have for this function to plot its light curve and phased light
        curve. If the object has less than this number, no light curves will be
        plotted, but the checkplotdict will still contain all of the other
        information.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    lcformat : str
        This is the `formatkey` associated with your light curve format, which
        you previously passed in to the `lcproc.register_lcformat`
        function. This will be used to look up how to find and read the light
        curves specified in `basedir` or `use_list_of_filenames`.

    lcformatdir : str or None
        If this is provided, gives the path to a directory when you've stored
        your lcformat description JSONs, other than the usual directories lcproc
        knows to search for them in. Use this along with `lcformat` to specify
        an LC format JSON file that's not currently registered with lcproc.

    timecols : list of str or None
        The timecol keys to use from the lcdict in generating this checkplot.

    magcols : list of str or None
        The magcol keys to use from the lcdict in generating this checkplot.

    errcols : list of str or None
        The errcol keys to use from the lcdict in generating this checkplot.

    skipdone : bool
        This indicates if this function will skip creating checkplots that
        already exist corresponding to the current `objectid` and `magcol`. If
        `skipdone` is set to True, this will be done.

    done_callback : Python function or None
        This is used to provide a function to execute after the checkplot
        pickles are generated. This is useful if you want to stream the results
        of checkplot making to some other process, e.g. directly running an
        ingestion into an LCC-Server collection. The function will always get
        the list of the generated checkplot pickles as its first arg, and all of
        the kwargs for runcp in the kwargs dict. Additional args and kwargs can
        be provided by giving a list in the `done_callbacks_args` kwarg and a
        dict in the `done_callbacks_kwargs` kwarg.

        NOTE: the function you pass in here should be pickleable by normal
        Python if you want to use it with the parallel_cp and parallel_cp_lcdir
        functions below.

    done_callback_args : tuple or None
        If not None, contains any args to pass into the `done_callback`
        function.

    done_callback_kwargs : dict or None
        If not None, contains any kwargs to pass into the `done_callback`
        function.

    maxobjects : int
        The maximum number of objects to process in this run.

    nworkers : int
        The number of parallel workers that will work on the checkplot
        generation process.

    Returns
    -------

    dict
        This returns a dict with keys = input period-finding pickles and vals =
        list of the corresponding checkplot pickles produced.

    '''

    arg_27 = sorted(glob.glob(os.path.join(arg_0, arg_3)))

    LOGINFO('found %s period-finding pickles, running cp...' %
            len(arg_27))

    return parallel_cp(arg_27,
                       arg_1,
                       arg_2,
                       arg_9=arg_9,
                       arg_4=arg_4,
                       arg_6=arg_6,
                       arg_10=arg_10,
                       arg_11=arg_11,
                       arg_7=arg_7,
                       arg_8=arg_8,
                       arg_12=arg_12,
                       arg_13=arg_13,
                       arg_15=arg_15,
                       arg_14=arg_14,
                       arg_5=arg_5,
                       arg_25=arg_25,
                       arg_16=arg_16,
                       arg_17=arg_17,
                       arg_18=arg_18,
                       arg_19=arg_19,
                       arg_20=arg_20,
                       arg_21=arg_21,
                       arg_26=arg_26,
                       arg_22=arg_22,
                       arg_23=arg_23,
                       arg_24=arg_24)


def Func(arg_0):
    '''
    This runs the runpf function.

    '''

    (arg_1, arg_2, arg_3, arg_4, arg_5, arg_6, arg_7,
     arg_8, arg_9, arg_10, arg_11, arg_12, arg_13,
     arg_14) = arg_0

    if os.path.exists(arg_1):
        arg_15 = runpf(arg_1,
                         arg_2,
                         arg_3=arg_3,
                         arg_4=arg_4,
                         arg_5=arg_5,
                         arg_6=arg_6,
                         arg_7=arg_7,
                         arg_8=arg_8,
                         arg_9=arg_9,
                         arg_10=arg_10,
                         arg_11=arg_11,
                         arg_12=arg_12,
                         arg_13=arg_13,
                         arg_14=arg_14)
        return arg_15
    else:
        LOGERROR('LC does not exist for requested file %s' % arg_1)
        return None


def Func(
        arg_0,
        arg_1,
        arg_2,
        arg_3='varfeatures-*.pkl',
        arg_4=arg_5,
        arg_6=None,
        arg_7=None,
        arg_8='binary',
):
    '''This collects variability features into arrays for use with the classifer.

    Parameters
    ----------

    featuresdir : str
        This is the directory where all the varfeatures pickles are. Use
        `pklglob` to specify the glob to search for. The `varfeatures` pickles
        contain objectids, a light curve magcol, and features as dict
        key-vals. The :py:mod:`astrobase.lcproc.lcvfeatures` module can be used
        to produce these.

    magcol : str
        This is the key in each varfeatures pickle corresponding to the magcol
        of the light curve the variability features were extracted from.

    outfile : str
        This is the filename of the output pickle that will be written
        containing a dict of all the features extracted into np.arrays.

    pklglob : str
        This is the UNIX file glob to use to search for varfeatures pickle files
        in `featuresdir`.

    featurestouse : list of str
        Each varfeatures pickle can contain any combination of non-periodic,
        stellar, and periodic features; these must have the same names as
        elements in the list of strings provided in `featurestouse`.  This tries
        to get all the features listed in NONPERIODIC_FEATURES_TO_COLLECT by
        default. If `featurestouse` is provided as a list, gets only the
        features listed in this kwarg instead.

    maxobjects : int or None
        The controls how many pickles from the featuresdir to process. If None,
        will process all varfeatures pickles.

    labeldict : dict or None
        If this is provided, it must be a dict with the following key:val list::

            '<objectid>':<label value>

        for each objectid collected from the varfeatures pickles. This will turn
        the collected information into a training set for classifiers.

        Example: to carry out non-periodic variable feature collection of fake
        LCS prepared by :py:mod:`astrobase.fakelcs.generation`, use the value
        of the 'isvariable' dict elem from the `fakelcs-info.pkl` here, like
        so::

            labeldict={x:y for x,y in zip(fakelcinfo['objectid'],
                                          fakelcinfo['isvariable'])}

    labeltype : {'binary', 'classes'}
        This is either 'binary' or 'classes' for binary/multi-class
        classification respectively.

    Returns
    -------

    dict
        This returns a dict with all of the features collected into np.arrays,
        ready to use as input to a scikit-learn classifier.

    '''

    # list of input pickles generated by varfeatures in lcproc.py
    arg_9 = glob.glob(os.path.join(arg_0, arg_3))

    if arg_6:
        arg_9 = arg_9[:arg_6]

    # fancy progress bar with tqdm if present
    if TQDM:
        arg_10 = tqdm(arg_9)
    else:
        arg_10 = arg_9

    # go through all the varfeatures arrays

    arg_11 = {'objectids':[],'magcol':arg_1, 'availablefeatures':[]}

    LOGINFO('collecting features for magcol: %s' % arg_1)

    for arg_12 in arg_10:

        with open(arg_12,'rb') as infd:
            arg_13 = pickle.load(infd)

        # update the objectid list
        arg_14 = arg_13['objectid']
        if arg_14 not in arg_11['objectids']:
            arg_11['objectids'].append(arg_14)

        arg_15 = arg_13[arg_1]

        if arg_4 and len(arg_4) > 0:
            arg_16 = arg_4
        else:
            arg_16 = arg_5

        # collect all the features for this magcol/objectid combination
        for arg_17 in arg_16:

            # update the global feature list if necessary
            if ((arg_17 not in arg_11['availablefeatures']) and
                (arg_17 in arg_15)):

                arg_11['availablefeatures'].append(arg_17)
                arg_11[arg_17] = []

            if arg_17 in arg_15:

                arg_11[arg_17].append(
                    arg_15[arg_17]
                )

    # now that we've collected all the objects and their features, turn the list
    # into arrays, and then concatenate them
    for arg_18 in arg_11['availablefeatures']:
        arg_11[arg_18] = np.array(arg_11[arg_18])

    arg_11['objectids'] = np.array(arg_11['objectids'])

    arg_19 = np.column_stack([arg_11[arg_18] for arg_18 in
                                     arg_11['availablefeatures']])
    arg_11['features_array'] = arg_19

    # if there's a labeldict available, use it to generate a label array. this
    # feature collection is now a training set.
    if isinstance(arg_7, dict):

        arg_20 = np.zeros(arg_11['objectids'].size, dtype=np.int64)

        # populate the labels for each object in the training set
        for arg_21, arg_14 in enumerate(arg_11['objectids']):

            if arg_14 in arg_7:

                # if this is a binary classifier training set, convert bools to
                # ones and zeros
                if arg_8 == 'binary':

                    if arg_7[arg_14]:
                        arg_20[arg_21] = 1

                # otherwise, use the actual class label integer
                elif arg_8 == 'classes':
                    arg_20[arg_21] = arg_7[arg_14]

        arg_11['labels_array'] = arg_20

    arg_11['kwargs'] = {'pklglob':arg_3,
                              'featurestouse':arg_4,
                              'maxobjects':arg_6,
                              'labeltype':arg_8}

    # write the info to the output pickle
    with open(arg_2,'wb') as outfd:
        pickle.dump(arg_11, outfd, pickle.HIGHEST_PROTOCOL)

    # return the feature_dict
    return arg_11


def Func(
        arg_0,
        arg_1=0.25,
        arg_2=20,
        arg_3=5,
        arg_4='f1',
        arg_5=None,
        arg_6=-1,
):

    '''This gets the best RF classifier after running cross-validation.

    - splits the training set into test/train samples
    - does `KFold` stratified cross-validation using `RandomizedSearchCV`
    - gets the `RandomForestClassifier` with the best performance after CV
    - gets the confusion matrix for the test set

    Runs on the output dict from functions that produce dicts similar to that
    produced by `collect_nonperiodic_features` above.

    Parameters
    ----------

    collected_features : dict or str
        This is either the dict produced by a `collect_*_features` function or
        the pickle produced by the same.

    test_fraction : float
        This sets the fraction of the input set that will be used as the
        test set after training.

    n_crossval_iterations : int
        This sets the number of iterations to use when running the
        cross-validation.

    n_kfolds : int
        This sets the number of K-folds to use on the data when doing a
        test-train split.

    crossval_scoring_metric : str
        This is a string that describes how the cross-validation score is
        calculated for each iteration. See the URL below for how to specify this
        parameter:

        http://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter
        By default, this is tuned for binary classification and uses the F1
        scoring metric. Change the `crossval_scoring_metric` to another metric
        (probably 'accuracy') for multi-class classification, e.g. for periodic
        variable classification.

    classifier_to_pickle : str
        If this is a string indicating the name of a pickle file to write, will
        write the trained classifier to the pickle that can be later loaded and
        used to classify data.

    nworkers : int
        This is the number of parallel workers to use in the
        RandomForestClassifier. Set to -1 to use all CPUs on your machine.

    Returns
    -------

    dict
        A dict containing the trained classifier, cross-validation results, the
        input data set, and all input kwargs used is returned, along with
        cross-validation score metrics.

    '''

    if (isinstance(arg_0,str) and
        os.path.exists(arg_0)):
        with open(arg_0,'rb') as infd:
            arg_7 = pickle.load(infd)
    elif isinstance(arg_0, dict):
        arg_7 = arg_0
    else:
        LOGERROR("can't figure out the input collected_features arg")
        return None

    arg_8 = arg_7['features_array']
    arg_9 = arg_7['labels_array']
    arg_10 = arg_7['availablefeatures']
    arg_11 = arg_7['magcol']
    arg_12 = arg_7['objectids']

    # split the training set into training/test samples using stratification
    # to keep the same fraction of variable/nonvariables in each
    arg_13, arg_14, arg_15, arg_16 = (
        train_test_split(
            arg_8,
            arg_9,
            test_size=arg_1,
            random_state=RANDSEED,
            stratify=arg_9
        )
    )

    # get a random forest classifier
    arg_17 = RandomForestClassifier(n_jobs=arg_6,
                                 random_state=RANDSEED)

    # this is the grid def for hyperparam optimization
    arg_18 = {
        "max_depth": [3,4,5,None],
        "n_estimators":sp_randint(100,2000),
        "max_features": sp_randint(1, 5),
        "min_samples_split": sp_randint(2, 11),
        "min_samples_leaf": sp_randint(2, 11),
    }

    # run the stratified kfold cross-validation on training features using our
    # random forest classifier object
    arg_19 = RandomizedSearchCV(
        arg_17,
        param_distributions=arg_18,
        n_iter=arg_2,
        scoring=arg_4,
        cv=StratifiedKFold(n_splits=arg_3,
                           shuffle=True,
                           random_state=RANDSEED),
        random_state=RANDSEED
    )

    LOGINFO('running grid-search CV to optimize RF hyperparameters...')
    arg_20 = arg_19.fit(arg_13,
                                        arg_15)

    # report on the classifiers' performance
    _gridsearch_report(arg_20.cv_results_)

    # get the best classifier after CV is done
    arg_21 = arg_20.best_estimator_
    arg_22 = arg_20.best_score_
    arg_23 = arg_20.best_params_

    # test this classifier on the testing set
    arg_24 = arg_21.predict(arg_14)

    arg_25 = recall_score(arg_16, arg_24)
    arg_26 = precision_score(arg_16,arg_24)
    arg_27 = f1_score(arg_16, arg_24)
    arg_28 = confusion_matrix(arg_16, arg_24)

    # write the classifier, its training/testing set, and its stats to the
    # pickle if requested
    arg_29 = {'features':arg_8,
               'labels':arg_9,
               'feature_names':arg_10,
               'magcol':arg_11,
               'objectids':arg_12,
               'kwargs':{'test_fraction':arg_1,
                         'n_crossval_iterations':arg_2,
                         'n_kfolds':arg_3,
                         'crossval_scoring_metric':arg_4,
                         'nworkers':arg_6},
               'collect_kwargs':arg_7['kwargs'],
               'testing_features':arg_14,
               'testing_labels':arg_16,
               'training_features':arg_13,
               'training_labels':arg_15,
               'best_classifier':arg_21,
               'best_score':arg_22,
               'best_hyperparams':arg_23,
               'best_recall':arg_25,
               'best_precision':arg_26,
               'best_f1':arg_27,
               'best_confmatrix':arg_28}

    if arg_5:

        with open(arg_5,'wb') as outfd:
            pickle.dump(arg_29, outfd, pickle.HIGHEST_PROTOCOL)

    # return this classifier and accompanying info
    return arg_29


def Func(arg_0,
                        arg_1,
                        arg_2,
                        arg_3=None):
    '''This applys an RF classifier trained using `train_rf_classifier`
    to varfeatures pickles in `varfeaturesdir`.

    Parameters
    ----------

    classifier : dict or str
        This is the output dict or pickle created by `get_rf_classifier`. This
        will contain a `features_name` key that will be used to collect the same
        features used to train the classifier from the varfeatures pickles in
        varfeaturesdir.

    varfeaturesdir : str
        The directory containing the varfeatures pickles for objects that will
        be classified by the trained `classifier`.

    outpickle : str
        This is a filename for the pickle that will be written containing the
        result dict from this function.

    maxobjects : int
        This sets the number of objects to process in `varfeaturesdir`.

    Returns
    -------

    dict
        The classification results after running the trained `classifier` as
        returned as a dict. This contains predicted labels and their prediction
        probabilities.

    '''

    if isinstance(arg_0,str) and os.path.exists(arg_0):
        with open(arg_0,'rb') as infd:
            arg_4 = pickle.load(infd)
    elif isinstance(arg_0, dict):
        arg_4 = arg_0
    else:
        LOGERROR("can't figure out the input classifier arg")
        return None

    # get the features to extract from clfdict
    if 'feature_names' not in arg_4:
        LOGERROR("feature_names not present in classifier input, "
                 "can't figure out which ones to extract from "
                 "varfeature pickles in %s" % arg_1)
        return None

    # get the feature labeltype, pklglob, and maxobjects from classifier's
    # collect_kwargs elem.
    arg_5 = arg_4['feature_names']
    arg_6 = arg_4['collect_kwargs']['pklglob']
    arg_7 = arg_4['magcol']

    # extract the features used by the classifier from the varfeatures pickles
    # in varfeaturesdir using the pklglob provided
    arg_8 = os.path.join(
        os.path.dirname(arg_2),
        'actual-collected-features.pkl'
    )

    arg_9 = collect_nonperiodic_features(
        arg_1,
        arg_7,
        arg_8,
        arg_6=arg_6,
        arg_5=arg_5,
        arg_3=arg_3
    )

    # now use the trained classifier on these features
    arg_10 = arg_4['best_classifier']

    arg_11 = arg_10.predict(arg_9['features_array'])

    # FIXME: do we need to use the probability calibration curves to fix these
    # probabilities? probably. figure out how to do this.
    arg_12 = arg_10.predict_proba(
        arg_9['features_array']
    )

    arg_13 = {
        'features':arg_9,
        'featfile':arg_8,
        'classifier':arg_4,
        'predicted_labels':arg_11,
        'predicted_label_probs':arg_12,
    }

    with open(arg_2,'wb') as outfd:
        pickle.dump(arg_13, outfd, pickle.HIGHEST_PROTOCOL)

    return arg_13


def Func(arg_0, arg_1, arg_2):
    '''This returns a summed Fourier cosine series.

    Parameters
    ----------

    fourierparams : list
        This MUST be a list of the following form like so::

            [period,
             epoch,
             [amplitude_1, amplitude_2, amplitude_3, ..., amplitude_X],
             [phase_1, phase_2, phase_3, ..., phase_X]]

        where X is the Fourier order.

    phase,mags : np.array
        The input phase and magnitude areas to use as the basis for the cosine
        series. The phases are used directly to generate the values of the
        function, while the mags array is used to generate the zeroth order
        amplitude coefficient.

    Returns
    -------

    np.array
        The Fourier cosine series function evaluated over `phase`.

    '''

    # figure out the order from the length of the Fourier param list
    arg_3 = int(len(arg_0)/2)

    # get the amplitude and phase coefficients
    arg_4 = arg_0[:arg_3]
    arg_5 = arg_0[arg_3:]

    # calculate all the individual terms of the series
    arg_6 = [arg_4[x]*npcos(2.0*pi_value*x*arg_1 + arg_5[x])
                for x in range(arg_3)]

    # this is the zeroth order coefficient - a constant equal to median mag
    arg_7 = npmedian(arg_2)

    # sum the series
    for arg_8 in arg_6:
        arg_7 += arg_8

    return arg_7


def Func(arg_0,
                   arg_1,
                   arg_2,
                   arg_3):
    '''This is the chisq objective function to be minimized by `scipy.minimize`.

    The parameters are the same as `_fourier_func` above. `errs` is used to
    calculate the chisq value.

    '''

    arg_4 = _fourier_func(arg_0, arg_1, arg_2)
    arg_5 = npsum(((arg_2 - arg_4)*(arg_2 - arg_4))/(arg_3*arg_3))

    return arg_5


def Func(arg_0, arg_1=None, arg_2=100):

    '''Makes a plot of periodograms obtained from `periodbase` functions.

    This takes the output dict produced by any `astrobase.periodbase`
    period-finder function or a pickle filename containing such a dict and makes
    a periodogram plot.

    Parameters
    ----------

    lspinfo : dict or str
        If lspinfo is a dict, it must be a dict produced by an
        `astrobase.periodbase` period-finder function or a dict from your own
        period-finder function or routine that is of the form below with at
        least these keys::

            {'periods': np.array of all periods searched by the period-finder,
             'lspvals': np.array of periodogram power value for each period,
             'bestperiod': a float value that is the period with the highest
                           peak in the periodogram, i.e. the most-likely actual
                           period,
             'method': a three-letter code naming the period-finder used; must
                       be one of the keys in the `METHODLABELS` dict above,
             'nbestperiods': a list of the periods corresponding to periodogram
                             peaks (`nbestlspvals` below) to annotate on the
                             periodogram plot so they can be called out
                             visually,
             'nbestlspvals': a list of the power values associated with
                             periodogram peaks to annotate on the periodogram
                             plot so they can be called out visually; should be
                             the same length as `nbestperiods` above}

        If lspinfo is a str, then it must be a path to a pickle file that
        contains a dict of the form described above.

    outfile : str or None
        If this is a str, will write the periodogram plot to the file specified
        by this string. If this is None, will write to a file called
        'lsp-plot.png' in the current working directory.

    plotdpi : int
        Sets the resolution in DPI of the output periodogram plot PNG file.

    Returns
    -------

    str
        Absolute path to the periodogram plot file created.

    '''

    # get the lspinfo from a pickle file transparently
    if isinstance(arg_0,str) and os.path.exists(arg_0):
        LOGINFO('loading LSP info from pickle %s' % arg_0)
        with open(arg_0,'rb') as infd:
            arg_0 = pickle.load(infd)

    try:

        # get the things to plot out of the data
        arg_3 = arg_0['periods']
        arg_4 = arg_0['lspvals']
        arg_5 = arg_0['bestperiod']
        arg_6 = arg_0['method']

        # make the LSP plot on the first subplot
        plt.plot(arg_3, arg_4)
        plt.xscale('log',basex=10)
        plt.xlabel('Period [days]')
        plt.ylabel(PLOTYLABELS[arg_6])
        arg_7 = '%s best period: %.6f d' % (METHODSHORTLABELS[arg_6],
                                                arg_5)
        plt.title(arg_7)

        # show the best five peaks on the plot
        for arg_5, arg_8 in zip(arg_0['nbestperiods'],
                                        arg_0['nbestlspvals']):

            plt.annotate('%.6f' % arg_5,
                         xy=(arg_5, arg_8), xycoords='data',
                         xytext=(0.0,25.0), textcoords='offset points',
                         arrowprops=dict(arrowstyle="->"),fontsize='x-small')

        # make a grid
        plt.grid(color='#a9a9a9',
                 alpha=0.9,
                 zorder=0,
                 linewidth=1.0,
                 linestyle=':')

        # make the figure
        if arg_1 and isinstance(arg_1, str):

            if arg_1.endswith('.png'):
                plt.savefig(arg_1,bbox_inches='tight',dpi=arg_2)
            else:
                plt.savefig(arg_1,bbox_inches='tight')

            plt.close()
            return os.path.abspath(arg_1)

        elif dispok:

            plt.show()
            plt.close()
            return

        else:

            LOGWARNING('no output file specified and no $DISPLAY set, '
                       'saving to lsp-plot.png in current directory')
            arg_1 = 'lsp-plot.png'
            plt.savefig(arg_1,bbox_inches='tight',dpi=arg_2)
            plt.close()
            return os.path.abspath(arg_1)

    except Exception as e:

        LOGEXCEPTION('could not plot this LSP, appears to be empty')
        return


def Func(arg_0, arg_1=None):
    '''This just writes the lcdict to a pickle.

    If outfile is None, then will try to get the name from the
    lcdict['objectid'] and write to <objectid>-hptxtlc.pkl. If that fails, will
    write to a file named hptxtlc.pkl'.

    '''

    if not arg_1 and arg_0['objectid']:
        arg_1 = '%s-hplc.pkl' % arg_0['objectid']
    elif not arg_1 and not arg_0['objectid']:
        arg_1 = 'hplc.pkl'

    with open(arg_1,'wb') as outfd:
        pickle.dump(arg_0, outfd, protocol=pickle.HIGHEST_PROTOCOL)

    if os.path.exists(arg_1):
        LOGINFO('lcdict for object: %s -> %s OK' % (arg_0['objectid'],
                                                    arg_1))
        return arg_1
    else:
        LOGERROR('could not make a pickle for this lcdict!')
        return None


def Func(arg_0):
    '''
    This just reads a pickle LC. Returns an lcdict.

    '''

    try:

        if arg_0.endswith('.gz'):
            arg_1 = gzip.open(arg_0,'rb')
        else:
            arg_1 = open(arg_0,'rb')

        arg_2 = pickle.load(arg_1)
        arg_1.close()

        return arg_2

    except UnicodeDecodeError:

        if arg_0.endswith('.gz'):
            arg_1 = gzip.open(arg_0,'rb')
        else:
            arg_1 = open(arg_0,'rb')

        LOGWARNING('pickle %s was probably from Python 2 '
                   'and failed to load without using "latin1" encoding. '
                   'This is probably a numpy issue: '
                   'http://stackoverflow.com/q/11305790' % arg_0)
        arg_2 = pickle.load(arg_1, encoding='latin1')
        arg_1.close()

        return arg_2


def Func(arg_0,
                        arg_1='rjd',
                        arg_2=True):
    '''This concatenates a list of light curves.

    Does not care about overlaps or duplicates. The light curves must all be
    from the same aperture.

    The intended use is to concatenate light curves across CCDs or instrument
    changes for a single object. These can then be normalized later using
    standard astrobase tools to search for variablity and/or periodicity.

    sortby is a column to sort the final concatenated light curve by in
    ascending order.

    If normalize is True, then each light curve's magnitude columns are
    normalized to zero.

    The returned lcdict has an extra column: 'lcn' that tracks which measurement
    belongs to which input light curve. This can be used with
    lcdict['concatenated'] which relates input light curve index to input light
    curve filepath. Finally, there is an 'nconcatenated' key in the lcdict that
    contains the total number of concatenated light curves.

    '''

    # read the first light curve
    arg_3 = read_hatpi_textlc(arg_0[0])

    # track which LC goes where
    # initial LC
    arg_4 = 0
    arg_3['concatenated'] = {arg_4: os.path.abspath(arg_0[0])}
    arg_3['lcn'] = np.full_like(arg_3['rjd'], arg_4)

    # normalize if needed
    if arg_2:

        for arg_5 in MAGCOLS:

            if arg_5 in arg_3:
                arg_6 = np.nanmedian(arg_3[arg_5])

                # handle fluxes
                if arg_5 in ('ifl1','ifl2','ifl3'):
                    arg_3[arg_5] = arg_3[arg_5] / arg_6
                # handle mags
                else:
                    arg_3[arg_5] = arg_3[arg_5] - arg_6

    # now read the rest
    for arg_7 in arg_0[1:]:

        arg_8 = read_hatpi_textlc(arg_7)

        # if the columns don't agree, skip this LC
        if arg_8['columns'] != arg_3['columns']:
            LOGERROR('file %s does not have the '
                     'same columns as first file %s, skipping...'
                     % (arg_7, arg_0[0]))
            continue

        # otherwise, go ahead and start concatenatin'
        else:

            LOGINFO('adding %s (ndet: %s) to %s (ndet: %s)'
                    % (arg_7,
                       arg_8['objectinfo']['ndet'],
                       arg_0[0],
                       arg_3[arg_3['columns'][0]].size))

            # update LC tracking
            arg_4 = arg_4 + 1
            arg_3['concatenated'][arg_4] = os.path.abspath(arg_7)
            arg_3['lcn'] = np.concatenate((
                arg_3['lcn'],
                np.full_like(arg_8['rjd'],arg_4)
            ))

            # concatenate the columns
            for arg_5 in arg_3['columns']:

                # handle normalization for magnitude columns
                if arg_2 and arg_5 in MAGCOLS:

                    arg_6 = np.nanmedian(arg_8[arg_5])

                    # handle fluxes
                    if arg_5 in ('ifl1','ifl2','ifl3'):
                        arg_8[arg_5] = arg_8[arg_5] / arg_6
                    # handle mags
                    else:
                        arg_8[arg_5] = arg_8[arg_5] - arg_6

                # concatenate the values
                arg_3[arg_5] = np.concatenate((arg_3[arg_5], arg_8[arg_5]))

    #
    # now we're all done concatenatin'
    #

    # make sure to add up the ndet
    arg_3['objectinfo']['ndet'] = arg_3[arg_3['columns'][0]].size

    # update the stations
    arg_3['objectinfo']['stations'] = [
        'HP%s' % arg_9 for arg_9 in np.unique(arg_3['stf']).tolist()
    ]

    # update the total LC count
    arg_3['nconcatenated'] = arg_4 + 1

    # if we're supposed to sort by a column, do so
    if arg_1 and arg_1 in [arg_9[0] for arg_9 in COLDEFS]:

        LOGINFO('sorting concatenated light curve by %s...' % arg_1)
        arg_10 = np.argsort(arg_3[arg_1])

        # sort all the measurement columns by this index
        for arg_5 in arg_3['columns']:
            arg_3[arg_5] = arg_3[arg_5][arg_10]

        # make sure to sort the lcn index as well
        arg_3['lcn'] = arg_3['lcn'][arg_10]

    LOGINFO('done. concatenated light curve has %s detections' %
            arg_3['objectinfo']['ndet'])
    return arg_3


def Func(arg_0, arg_1, arg_2,
                                arg_3=None):
    '''
    This reads the binned LC and writes it out to a pickle.

    '''

    arg_4 = read_hatpi_binnedlc(arg_0, arg_1, arg_2)

    if arg_4:
        if arg_3 is None:
            arg_3 = os.path.join(
                os.path.dirname(arg_0),
                '%s-hplc.pkl' % (
                    os.path.basename(arg_0).replace('sec-lc.pkl.gz','')
                )
            )

        return lcdict_to_pickle(arg_4, arg_3=arg_3)
    else:
        LOGERROR('could not read binned HATPI LC: %s' % arg_0)
        return None


def Func(arg_0,
                               arg_1,
                               arg_2,
                               arg_3='*binned*sec*.pkl',
                               arg_4='*.tfalc.TF1*'):
    '''
    This generates the binnedlc pkls for a directory of such files.

    FIXME: finish this

    '''

    arg_5 = sorted(glob.glob(os.path.join(arg_0, arg_3)))

    # find all the textlcs associated with these
    arg_6 = []

    for arg_7 in arg_5:

        arg_8 = HATIDREGEX.findall(arg_7)
        if arg_8 is not None:
            arg_8 = arg_8[0]

        arg_9 = os.path.join(arg_1, '%s-%s' % (arg_8, arg_4))
        arg_10 = glob.glob(arg_9)
        if arg_10:
            arg_6.append(arg_10)
        else:
            arg_6.append(None)


def Func(
        arg_0,
        arg_1,
        arg_2=[0,1,2,
                            6,7,
                            8,9,
                            10,11,
                            13,14,15,16,
                            17,18,19,
                            20,21],
        arg_3=['objectid','ra','decl',
                             'jmag','jmag_err',
                             'hmag','hmag_err',
                             'kmag','kmag_err',
                             'bmag','vmag','rmag','imag',
                             'sdssu','sdssg','sdssr',
                             'sdssi','sdssz'],
        arg_4=('U20,f8,f8,'
                               'f8,f8,'
                               'f8,f8,'
                               'f8,f8,'
                               'f8,f8,f8,f8,'
                               'f8,f8,f8,'
                               'f8,f8')
):
    '''Adds catalog info to objectinfo key of all pklcs in lcdir.

    If fovcatalog, fovcatalog_columns, fovcatalog_colnames are provided, uses
    them to find all the additional information listed in the fovcatalog_colname
    keys, and writes this info to the objectinfo key of each lcdict. This makes
    it easier for astrobase tools to work on these light curve.

    The default set up for fovcatalog is to use a text file generated by the
    HATPI pipeline before auto-calibrating a field. The format is specified as
    above in _columns,  _colnames, and _colformats.

    '''

    if arg_1.endswith('.gz'):
        arg_5 = gzip.open(arg_1)
    else:
        arg_5 = open(arg_1)

    # read the catalog using the colformats, etc.
    arg_6 = np.genfromtxt(arg_5,
                           usecols=arg_2,
                           names=arg_3,
                           dtype=arg_4)
    arg_5.close()

    arg_7 = sorted(glob.glob(os.path.join(arg_0, '*HAT*-pklc.pkl')))

    arg_8, arg_9 = [], []

    for arg_10 in arg_7:

        arg_11 = read_hatpi_pklc(arg_10)
        arg_12 = arg_11['objectid']

        arg_13 = np.where(arg_6['objectid'] == arg_12)

        # if we found catalog info for this object, put it into objectinfo
        if len(arg_13) > 0 and arg_13[0]:

            arg_11['objectinfo'].update(
                {arg_14:arg_15 for arg_14,arg_15 in zip(
                    arg_3,
                    [np.asscalar(arg_6[arg_16][arg_13]) for
                     arg_16 in arg_3]
                )
                }
            )

            # write the LC back to the pickle (tempfile for atomicity)
            with open(arg_10+'-tmp','wb') as outfd:
                pickle.dump(arg_11, outfd, pickle.HIGHEST_PROTOCOL)

            # overwrite previous once we know it exists
            if os.path.exists(arg_10+'-tmp'):
                shutil.move(arg_10+'-tmp',arg_10)

                LOGINFO('updated %s with catalog info for %s at %.3f, %.3f OK' %
                        (arg_10, arg_12,
                         arg_11['objectinfo']['ra'],
                         arg_11['objectinfo']['decl']))

                arg_8.append(arg_10)

        # otherwise, do nothing
        else:
            arg_9.append(arg_10)

    # end of pklclist processing
    return arg_8, arg_9


def Func(arg_0, arg_1, arg_2=False):
    '''This converts the base64 encoded string to a file.

    Parameters
    ----------

    b64str : str
        A base64 encoded strin that is the output of `base64.b64encode`.

    outfpath : str
        The path to where the file will be written. This should include an
        appropriate extension for the file (e.g. a base64 encoded string that
        represents a PNG should have its `outfpath` end in a '.png') so the OS
        can open these files correctly.

    writetostrio : bool
        If this is True, will return a StringIO object with the binary stream
        decoded from the base64-encoded input string `b64str`. This can be
        useful to embed these into other files without having to write them to
        disk.

    Returns
    -------

    str or StringIO object
        If `writetostrio` is False, will return the output file's path as a
        str. If it is True, will return a StringIO object directly. If writing
        the file fails in either case, will return None.

    '''

    try:

        arg_3 = base64.b64decode(arg_0)

        # if we're writing back to a stringio object
        if arg_2:

            arg_4 = StrIO(arg_3)
            return arg_4

        # otherwise, we're writing to an actual file
        else:

            with open(arg_1,'wb') as outfd:
                outfd.write(arg_3)

            if os.path.exists(arg_1):
                return arg_1
            else:
                LOGERROR('could not write output file: %s' % arg_1)
                return None

    except Exception as e:

        LOGEXCEPTION('failed while trying to convert '
                     'b64 string to file %s' % arg_1)
        return None


def Func(arg_0):
    '''This reads a checkplot gzipped pickle file back into a dict.

    NOTE: the try-except is for Python 2 pickles that have numpy arrays in
    them. Apparently, these aren't compatible with Python 3. See here:

    http://stackoverflow.com/q/11305790

    The workaround is noted in this answer:

    http://stackoverflow.com/a/41366785

    Parameters
    ----------

    checkplotpickle : str
        The path to a checkplot pickle file. This can be a gzipped file (in
        which case the file extension should end in '.gz')

    Returns
    -------

    dict
        This returns a checkplotdict.

    '''

    if arg_0.endswith('.gz'):

        try:
            with gzip.open(arg_0,'rb') as infd:
                arg_1 = pickle.load(infd)

        except UnicodeDecodeError:

            with gzip.open(arg_0,'rb') as infd:
                arg_1 = pickle.load(infd, encoding='latin1')

    else:

        try:
            with open(arg_0,'rb') as infd:
                arg_1 = pickle.load(infd)

        except UnicodeDecodeError:

            with open(arg_0,'rb') as infd:
                arg_1 = pickle.load(infd, encoding='latin1')

    return arg_1


def Func(arg_0, arg_1, arg_2, arg_3,
                  arg_4, arg_5, arg_6,
                  arg_7,
                  arg_8=False,
                  arg_9=False,
                  arg_10=False):
    '''This makes a plot of the LC model fit.

    Parameters
    ----------

    phase,pmags,perrs : np.array
        The actual mag/flux time-series.

    fitmags : np.array
        The model fit time-series.

    period : float
        The period at which the phased LC was generated.

    mintime : float
        The minimum time value.

    magseriesepoch : float
        The value of time around which the phased LC was folded.

    plotfit : str
        The name of a file to write the plot to.

    magsarefluxes : bool
        Set this to True if the values in `pmags` and `fitmags` are actually
        fluxes.

    wrap : bool
        If True, will wrap the phased LC around 0.0 to make some phased LCs
        easier to look at.

    model_over_lc : bool
        Usually, this function will plot the actual LC over the model LC. Set
        this to True to plot the model over the actual LC; this is most useful
        when you have a very dense light curve and want to be able to see how it
        follows the model.

    Returns
    -------

    Nothing.

    '''

    # set up the figure
    plt.close('all')
    plt.figure(figsize=(8,4.8))

    if arg_10:
        arg_11 = 100
        arg_12 = 0
    else:
        arg_11 = 0
        arg_12 = 100

    if not arg_9:

        plt.plot(arg_0, arg_3, linewidth=3.0, color='red',zorder=arg_11)
        plt.plot(arg_0,arg_1,
                 marker='o',
                 markersize=1.0,
                 linestyle='none',
                 rasterized=True, color='k',zorder=arg_12)

        # set the x axis ticks and label
        plt.gca().set_xticks(
            [0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0]
        )

    else:
        plt.plot(np.concatenate([arg_0-1.0,arg_0]),
                 np.concatenate([arg_3,arg_3]),
                 linewidth=3.0,
                 color='red',zorder=arg_11)
        plt.plot(np.concatenate([arg_0-1.0,arg_0]),
                 np.concatenate([arg_1,arg_1]),
                 marker='o',
                 markersize=1.0,
                 linestyle='none',
                 rasterized=True, color='k',zorder=arg_12)

        plt.gca().set_xlim((-0.8,0.8))
        # set the x axis ticks and label
        plt.gca().set_xticks(
            [-0.8,-0.7,-0.6,-0.5,-0.4,-0.3,-0.2,-0.1,
             0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]
        )

    # set the y axis limit and label
    arg_13, arg_14 = plt.ylim()
    if not arg_8:
        plt.gca().invert_yaxis()
        plt.ylabel('magnitude')
    else:
        plt.ylabel('flux')

    plt.xlabel('phase')
    plt.title('period: %.6f, folded at %.6f, fit epoch: %.6f' %
              (arg_4, arg_5, arg_6))
    plt.savefig(arg_7)
    plt.close()


def Func(arg_0,
                         arg_1=None,
                         arg_2=('source_id',
                                  'ra','dec',
                                  'phot_g_mean_mag',
                                  'l','b',
                                  'parallax, parallax_error',
                                  'pmra','pmra_error',
                                  'pmdec','pmdec_error'),
                         arg_3=None,
                         arg_4='csv',
                         arg_5=False,
                         arg_6='~/.astrobase/gaia-cache',
                         arg_7=True,
                         arg_8=15.0,
                         arg_9=2.0,
                         arg_10=300.0,
                         arg_11=3,
                         arg_12=True):

    '''This queries the GAIA TAP service for a list of objects in an equatorial
    coordinate box.

    Parameters
    ----------

    radeclbox : sequence of four floats
        This defines the box to search in::

            [ra_min, ra_max, decl_min, decl_max]

    gaia_mirror : {'gaia','heidelberg','vizier'} or None
        This is the key used to select a GAIA catalog mirror from the
        `GAIA_URLS` dict above. If set, the specified mirror will be used. If
        None, a random mirror chosen from that dict will be used.

    columns : sequence of str
        This indicates which columns from the GAIA table to request for the
        objects found within the search radius.

    extra_filter: str or None
        If this is provided, must be a valid ADQL filter string that is used to
        further filter the cone-search results.

    returnformat : {'csv','votable','json'}
        The returned file format to request from the GAIA catalog service.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our initial request.

    refresh : float
        This sets the amount of time in seconds to wait before checking if the
        result file is available. If the results file isn't available after
        `refresh` seconds have elapsed, the function will wait for `refresh`
        seconds continuously, until `maxtimeout` is reached or the results file
        becomes available.

    maxtimeout : float
        The maximum amount of time in seconds to wait for a result to become
        available after submitting our query request.

    maxtries : int
        The maximum number of tries (across all mirrors tried) to make to either
        submit the request or download the results, before giving up.

    completequerylater : bool
        If set to True, a submitted query that does not return a result before
        `maxtimeout` has passed will be cancelled but its input request
        parameters and the result URL provided by the service will be saved. If
        this function is then called later with these same input request
        parameters, it will check if the query finally finished and a result is
        available. If so, will download the results instead of submitting a new
        query. If it's not done yet, will start waiting for results again. To
        force launch a new query with the same request parameters, set the
        `forcefetch` kwarg to True.

    Returns
    -------

    dict
        This returns a dict of the following form::

            {'params':dict of the input params used for the query,
             'provenance':'cache' or 'new download',
             'result':path to the file on disk with the downloaded data table}

    '''

    # this was generated using the awesome query generator at:
    # https://gea.esac.esa.int/archive/

    # NOTE: here we don't resolve the table name right away. this is because
    # some of the GAIA mirrors use different table names, so we leave the table
    # name to be resolved by the lower level tap_query function. this is done by
    # the {{table}} construct.
    arg_13 = (
        "select {columns} from {{table}} where "
        "CONTAINS(POINT('ICRS',{{table}}.ra, {{table}}.dec),"
        "BOX('ICRS',{ra_center:.5f},{decl_center:.5f},"
        "{ra_width:.5f},{decl_height:.5f}))=1"
        "{extra_filter_str}"
    )

    arg_14, arg_15, arg_16, arg_17 = arg_0
    arg_18 = (arg_15 + arg_14)/2.0
    arg_19 = (arg_17 + arg_16)/2.0
    arg_20 = arg_15 - arg_14
    arg_21 = arg_17 - arg_16

    if arg_3 is not None:
        arg_22 = ' and %s ' % arg_3
    else:
        arg_22 = ''

    arg_23 = arg_13.format(arg_2=', '.join(arg_2),
                                   arg_22=arg_22,
                                   arg_18=arg_18,
                                   arg_19=arg_19,
                                   arg_20=arg_20,
                                   arg_21=arg_21)

    return tap_query(arg_23,
                     arg_1=arg_1,
                     arg_4=arg_4,
                     arg_5=arg_5,
                     arg_6=arg_6,
                     arg_7=arg_7,
                     arg_8=arg_8,
                     arg_9=arg_9,
                     arg_10=arg_10,
                     arg_11=arg_11,
                     arg_12=arg_12)


def Func(arg_0,
                    arg_1=None,
                    arg_2=('source_id',
                             'ra','dec',
                             'phot_g_mean_mag',
                             'phot_bp_mean_mag',
                             'phot_rp_mean_mag',
                             'l','b',
                             'parallax, parallax_error',
                             'pmra','pmra_error',
                             'pmdec','pmdec_error'),
                    arg_3='csv',
                    arg_4=False,
                    arg_5='~/.astrobase/gaia-cache',
                    arg_6=True,
                    arg_7=15.0,
                    arg_8=2.0,
                    arg_9=300.0,
                    arg_10=3,
                    arg_11=True):

    '''This queries the GAIA TAP service for a single GAIA source ID.

    Parameters
    ----------

    gaiaid : str
        The source ID of the object whose info will be collected.

    gaia_mirror : {'gaia','heidelberg','vizier'} or None
        This is the key used to select a GAIA catalog mirror from the
        `GAIA_URLS` dict above. If set, the specified mirror will be used. If
        None, a random mirror chosen from that dict will be used.

    columns : sequence of str
        This indicates which columns from the GAIA table to request for the
        objects found within the search radius.

    returnformat : {'csv','votable','json'}
        The returned file format to request from the GAIA catalog service.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our initial request.

    refresh : float
        This sets the amount of time in seconds to wait before checking if the
        result file is available. If the results file isn't available after
        `refresh` seconds have elapsed, the function will wait for `refresh`
        seconds continuously, until `maxtimeout` is reached or the results file
        becomes available.

    maxtimeout : float
        The maximum amount of time in seconds to wait for a result to become
        available after submitting our query request.

    maxtries : int
        The maximum number of tries (across all mirrors tried) to make to either
        submit the request or download the results, before giving up.

    completequerylater : bool
        If set to True, a submitted query that does not return a result before
        `maxtimeout` has passed will be cancelled but its input request
        parameters and the result URL provided by the service will be saved. If
        this function is then called later with these same input request
        parameters, it will check if the query finally finished and a result is
        available. If so, will download the results instead of submitting a new
        query. If it's not done yet, will start waiting for results again. To
        force launch a new query with the same request parameters, set the
        `forcefetch` kwarg to True.

    Returns
    -------

    dict
        This returns a dict of the following form::

            {'params':dict of the input params used for the query,
             'provenance':'cache' or 'new download',
             'result':path to the file on disk with the downloaded data table}

    '''

    # NOTE: here we don't resolve the table name right away. this is because
    # some of the GAIA mirrors use different table names, so we leave the table
    # name to be resolved by the lower level tap_query function. this is done by
    # the {{table}} construct.
    arg_12 = (
        "select {columns} from {{table}} where "
        "source_id = {gaiaid}"
    )

    arg_13 = arg_12.format(arg_2=', '.join(arg_2),
                                   arg_0=arg_0)

    return tap_query(arg_13,
                     arg_1=arg_1,
                     arg_3=arg_3,
                     arg_4=arg_4,
                     arg_5=arg_5,
                     arg_6=arg_6,
                     arg_7=arg_7,
                     arg_8=arg_8,
                     arg_9=arg_9,
                     arg_10=arg_10,
                     arg_11=arg_11)


def Func(arg_0, arg_1, arg_2, arg_3):
    '''This calculates the peak associated with the spectral window function
    for times and at the specified omega.

    NOTE: this is classical Lomb-Scargle, not the Generalized
    Lomb-Scargle. `mags` and `errs` are silently ignored since we're calculating
    the periodogram of the observing window function. These are kept to present
    a consistent external API so the `pgen_lsp` function below can call this
    transparently.

    Parameters
    ----------

    times,mags,errs : np.array
        The time-series to calculate the periodogram value for.

    omega : float
        The frequency to calculate the periodogram value at.

    Returns
    -------

    periodogramvalue : float
        The normalized periodogram at the specified test frequency `omega`.

    '''

    arg_4 = arg_0 - arg_0.min()

    arg_5 = (
        (1.0/(2.0*arg_3)) *
        nparctan( npsum(npsin(2.0*arg_3*arg_4)) /
                  npsum(npcos(2.0*arg_3*arg_4)) )
    )

    arg_6 = (npsum(1.0 * npcos(arg_3*(arg_4-arg_5))) *
                      npsum(1.0 * npcos(arg_3*(arg_4-arg_5))))
    arg_7 = npsum( (npcos(arg_3*(arg_4-arg_5))) *
                            (npcos(arg_3*(arg_4-arg_5))) )

    arg_8 = (npsum(1.0 * npsin(arg_3*(arg_4-arg_5))) *
                      npsum(1.0 * npsin(arg_3*(arg_4-arg_5))))
    arg_9 = npsum( (npsin(arg_3*(arg_4-arg_5))) *
                            (npsin(arg_3*(arg_4-arg_5))) )

    arg_10 = 0.5 * ( (arg_6/arg_7) +
                     (arg_8/arg_9) )

    return arg_10


def Func(
        arg_0,
        arg_1,
        arg_2,
        arg_3=False,
        arg_4=None,
        arg_5=None,
        arg_6=1.0e-4,
        arg_7=True,
        arg_8=5,
        arg_9=0.1,
        arg_10=10.0,
        arg_11=None,
        arg_12=arg_13,
        arg_14=True
):
    '''This calculates the spectral window function.

    Wraps the `pgen_lsp` function above to use the specific worker for
    calculating the window-function.

    Parameters
    ----------

    times,mags,errs : np.array
        The mag/flux time-series with associated measurement errors to run the
        period-finding on.

    magsarefluxes : bool
        If the input measurement values in `mags` and `errs` are in fluxes, set
        this to True.

    startp,endp : float or None
        The minimum and maximum periods to consider for the transit search.

    stepsize : float
        The step-size in frequency to use when constructing a frequency grid for
        the period search.

    autofreq : bool
        If this is True, the value of `stepsize` will be ignored and the
        :py:func:`astrobase.periodbase.get_frequency_grid` function will be used
        to generate a frequency grid based on `startp`, and `endp`. If these are
        None as well, `startp` will be set to 0.1 and `endp` will be set to
        `times.max() - times.min()`.

    nbestpeaks : int
        The number of 'best' peaks to return from the periodogram results,
        starting from the global maximum of the periodogram peak values.

    periodepsilon : float
        The fractional difference between successive values of 'best' periods
        when sorting by periodogram power to consider them as separate periods
        (as opposed to part of the same periodogram peak). This is used to avoid
        broad peaks in the periodogram and make sure the 'best' periods returned
        are all actually independent.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    nworkers : int
        The number of parallel workers to use when calculating the periodogram.

    glspfunc : Python function
        The worker function to use to calculate the periodogram. This is used to
        used to make the `pgen_lsp` function calculate the time-series sampling
        window function instead of the time-series measurements' GLS periodogram
        by passing in `_glsp_worker_specwindow` instead of the default
        `_glsp_worker` function.

    verbose : bool
        If this is True, will indicate progress and details about the frequency
        grid used for the period search.

    Returns
    -------

    dict
        This function returns a dict, referred to as an `lspinfo` dict in other
        astrobase functions that operate on periodogram results. This is a
        standardized format across all astrobase period-finders, and is of the
        form below::

            {'bestperiod': the best period value in the periodogram,
             'bestlspval': the periodogram peak associated with the best period,
             'nbestpeaks': the input value of nbestpeaks,
             'nbestlspvals': nbestpeaks-size list of best period peak values,
             'nbestperiods': nbestpeaks-size list of best periods,
             'lspvals': the full array of periodogram powers,
             'periods': the full array of periods considered,
             'method':'win' -> the name of the period-finder method,
             'kwargs':{ dict of all of the input kwargs for record-keeping}}

    '''

    # run the LSP using glsp_worker_specwindow as the worker
    arg_15 = pgen_lsp(
        arg_0,
        arg_1,
        arg_2,
        arg_3=arg_3,
        arg_4=arg_4,
        arg_5=arg_5,
        arg_7=arg_7,
        arg_8=arg_8,
        arg_9=arg_9,
        arg_6=arg_6,
        arg_11=arg_11,
        arg_10=arg_10,
        arg_12=arg_12,
        arg_14=arg_14
    )

    # update the resultdict to indicate we're a spectral window function
    arg_15['method'] = 'win'

    if arg_15['lspvals'] is not None:

        # renormalize the periodogram to between 0 and 1 like the usual GLS.
        arg_16 = npnanmax(arg_15['lspvals'])

        if npisfinite(arg_16):

            arg_15['lspvals'] = arg_15['lspvals']/arg_16
            arg_15['nbestlspvals'] = [
                x/arg_16 for x in arg_15['nbestlspvals']
            ]
            arg_15['bestlspval'] = arg_15['bestlspval']/arg_16

    return arg_15


def Func(arg_0):
    '''This gets a new API key from the specified LCC-Server.

    NOTE: this only gets an anonymous API key. To get an API key tied to a user
    account (and associated privilege level), see the `import_apikey` function
    below.

    Parameters
    ----------

    lcc_server : str
        The base URL of the LCC-Server from where the API key will be fetched.

    Returns
    -------

    (apikey, expiry) : tuple
        This returns a tuple with the API key and its expiry date.

    '''

    arg_1 = os.path.expanduser('~')
    arg_2 = os.path.join(arg_1,
                              '.astrobase',
                              'lccs',
                              'apikey-%s' % arg_0.replace(
                                  'https://',
                                  'https-'
                              ).replace(
                                  'http://',
                                  'http-'
                              ))

    # url for getting an API key
    arg_3 = '%s/api/key' % arg_0

    # get the API key
    arg_4 = urlopen(arg_3)

    if arg_4.code == 200:

        arg_5 = json.loads(arg_4.read())

    else:

        LOGERROR('could not fetch the API key from LCC-Server at: %s' %
                 arg_0)
        LOGERROR('the HTTP status code was: %s' % arg_4.status_code)
        return None

    #
    # now that we have an API key dict, get the API key out of it and write it
    # to the APIKEYFILE
    #
    arg_6 = arg_5['result']['apikey']
    arg_7 = arg_5['result']['expires']

    # write this to the apikey file

    if not os.path.exists(os.path.dirname(arg_2)):
        os.makedirs(os.path.dirname(arg_2))

    with open(arg_2,'w') as outfd:
        outfd.write('%s %s\n' % (arg_6, arg_7))

    # chmod it to the correct value
    os.chmod(arg_2, 0o100600)

    LOGINFO('key fetched successfully from: %s. expires on: %s' % (arg_0,
                                                                   arg_7))
    LOGINFO('written to: %s' % arg_2)

    return arg_6, arg_7


def Func(arg_0, arg_1):
    '''This imports an API key from text and writes it to the cache dir.

    Use this with the JSON text copied from the API key text box on your
    LCC-Server user home page. The API key will thus be tied to the privileges
    of that user account and can then access objects, datasets, and collections
    marked as private for the user only or shared with that user.

    Parameters
    ----------

    lcc_server : str
        The base URL of the LCC-Server to get the API key for.

    apikey_text_json : str
        The JSON string from the API key text box on the user's LCC-Server home
        page at `lcc_server/users/home`.

    Returns
    -------

    (apikey, expiry) : tuple
        This returns a tuple with the API key and its expiry date.

    '''
    arg_2 = os.path.expanduser('~')
    arg_3 = os.path.join(arg_2,
                              '.astrobase',
                              'lccs',
                              'apikey-%s' % arg_0.replace(
                                  'https://',
                                  'https-'
                              ).replace(
                                  'http://',
                                  'http-'
                              ))

    arg_4 = json.loads(arg_1)

    #
    # now that we have an API key dict, get the API key out of it and write it
    # to the APIKEYFILE
    #
    arg_5 = arg_4['apikey']
    arg_6 = arg_4['expires']

    # write this to the apikey file

    if not os.path.exists(os.path.dirname(arg_3)):
        os.makedirs(os.path.dirname(arg_3))

    with open(arg_3,'w') as outfd:
        outfd.write('%s %s\n' % (arg_5, arg_6))

    # chmod it to the correct value
    os.chmod(arg_3, 0o100600)

    LOGINFO('key fetched successfully from: %s. expires on: %s' % (arg_0,
                                                                   arg_6))
    LOGINFO('written to: %s' % arg_3)

    return arg_5, arg_6


def Func(arg_0, arg_1, arg_2):
    '''This submits a POST query to an LCC-Server search API endpoint.

    Handles streaming of the results, and returns the final JSON stream. Also
    handles results that time out.

    Parameters
    ----------

    url : str
        The URL of the search API endpoint to hit. This is something like
        `https://data.hatsurveys.org/api/conesearch`

    data : dict
        A dict of the search query parameters to pass to the search service.

    apikey : str
        The API key to use to access the search service. API keys are required
        for all POST request made to an LCC-Server's API endpoints.

    Returns
    -------

    (status_flag, data_dict, dataset_id) : tuple
        This returns a tuple containing the status of the request: ('complete',
        'failed', 'background', etc.), a dict parsed from the JSON result of the
        request, and a dataset ID, which can be used to reconstruct the URL on
        the LCC-Server where the results can be browsed.

    '''

    # first, we need to convert any columns and collections items to broken out
    # params
    arg_3 = {}

    for arg_4 in arg_1:

        if arg_4 == 'columns':
            arg_3['columns[]'] = arg_1[arg_4]
        elif arg_4 == 'collections':
            arg_3['collections[]'] = arg_1[arg_4]
        else:
            arg_3[arg_4] = arg_1[arg_4]

    # do the urlencode with doseq=True
    # we also need to encode to bytes
    arg_5 = urlencode(arg_3, doseq=True).encode()

    # if apikey is not None, add it in as an Authorization: Bearer [apikey]
    # header
    if arg_2:
        arg_6 = {'Authorization':'Bearer: %s' % arg_2}
    else:
        arg_6 = {}

    LOGINFO('submitting search query to LCC-Server API URL: %s' % arg_0)

    try:

        # hit the server with a POST request
        arg_7 = Request(arg_0, arg_1=arg_5, arg_6=arg_6)
        arg_8 = urlopen(arg_7)

        if arg_8.code == 200:

            # we'll iterate over the lines in the response
            # this works super-well for ND-JSON!
            for arg_9 in arg_8:

                arg_1 = json.loads(arg_9)
                arg_10 = arg_1['message']
                arg_11 = arg_1['status']

                if arg_11 != 'failed':
                    LOGINFO('status: %s, %s' % (arg_11, arg_10))
                else:
                    LOGERROR('status: %s, %s' % (arg_11, arg_10))

                # here, we'll decide what to do about the query

                # completed query or query sent to background...
                if arg_11 in ('ok','background'):

                    arg_12 = arg_1['result']['setid']
                    # save the data pickle to astrobase lccs directory
                    arg_13 = os.path.join(os.path.expanduser('~'),
                                             '.astrobase',
                                             'lccs',
                                             'query-%s.pkl' % arg_12)
                    if not os.path.exists(os.path.dirname(arg_13)):
                        os.makedirs(os.path.dirname(arg_13))

                    with open(arg_13,'wb') as outfd:
                        pickle.dump(arg_1, outfd, pickle.HIGHEST_PROTOCOL)
                        LOGINFO('saved query info to %s, use this to '
                                'download results later with '
                                'retrieve_dataset_files' % arg_13)

                    # we're done at this point, return
                    return arg_11, arg_1, arg_1['result']['setid']

                # the query probably failed...
                elif arg_11 == 'failed':

                    # we're done at this point, return
                    return arg_11, arg_1, None

        # if the response was not OK, then we probably failed
        else:

            try:
                arg_1 = json.load(arg_8)
                arg_10 = arg_1['message']

                LOGERROR(arg_10)
                return 'failed', None, None

            except Exception as e:

                LOGEXCEPTION('failed to submit query to %s' % arg_0)
                return 'failed', None, None

    except HTTPError as e:

        LOGERROR('could not submit query to LCC API at: %s' % arg_0)
        LOGERROR('HTTP status code was %s, reason: %s' % (e.code, e.reason))
        return 'failed', None, None