def start_data_processing(thread_number):
    """ TODO: replace with your code.
    Most work regarding sending to IF is abstracted away for you.
    This function will get the data to send and prepare it for the API.
    The general outline should be:
    0. Define the project type in config.ini
    1. Parse config options
    2. Gather data
    3. Parse each entry
    4. Call the appropriate handoff function
        metric_handoff()
        log_handoff()
        alert_handoff()
        incident_handoff()
        deployment_handoff()
    See zipkin for an example that uses os.fork to send both metric and log data.
    """ 

def reconfigure(ctx):
    """
    Reconfigure the chute without rebuilding.
    """
    url = ctx.obj['chute_url'] + "/config"

    if not os.path.exists("paradrop.yaml"):
        raise Exception("No paradrop.yaml file found in working directory.")

    with open("paradrop.yaml", "r") as source:
        data = yaml.safe_load(source)
        config = data.get('config', {})

    res = router_request("PUT", url, json=config)
    data = res.json()
    ctx.invoke(watch, change_id=data['change_id']) 

def connect_snap_interfaces(ctx):
    """
    Connect all interfaces for installed snaps.
    """
    client = ctx.obj['client']
    result = client.list_snap_interfaces()
    for item in result['result']['plugs']:
        connections = item.get('connections', [])
        if len(connections) > 0:
            continue

        if item['plug'] == 'docker':
            # The docker slot needs to be treated differently from core slots.
            slot = {'snap': 'docker'}
        elif item['plug'] == 'zerotier-control':
            # TODO: This connection is failing, but I am not sure why.
            slot = {'snap': 'zerotier-one', 'slot': 'zerotier-control'}
        else:
            # Most slots are provided by the core snap and specified this way.
            slot = {'slot': item['interface']}

        result = client.connect_snap_interface(slots=[slot], plugs=[{'snap':
            item['snap'], 'plug': item['plug']}])
        if result['type'] == 'error':
            print(result['result']['message']) 

def edit_chute_variables(ctx, chute):
    """
    Interactively edit a chute's environment variables and restart it.

    CHUTE must be the name of an installed chute.

    Open the text editor specified by the EDITOR environment variable
    with the current chute environment variables. If you save and exit,
    the new settings will be applied and the chute restarted.
    """
    client = ctx.obj['client']
    old_data = client.get_chute(chute).get('environment', {})
    new_data, changed = util.open_yaml_editor(old_data, "chute " + chute)
    if changed:
        result = client.set_chute_variables(chute, new_data)
        ctx.invoke(watch_change_logs, change_id=result['change_id'])
    return new_data 

def watch_chute_logs(ctx, chute):
    """
    Stream log messages from a running chute.

    CHUTE must be the name of a running chute.
    """
    url = "ws://{}/sockjs/logs/{}".format(ctx.obj['target'], chute)

    def on_message(ws, message):
        data = json.loads(message)
        time = arrow.get(data['timestamp']).to('local').datetime
        msg = data['message'].rstrip()
        service = data.get("service", chute)
        click.echo("[{}] {}: {}".format(service, time, msg))

    client = ctx.obj['client']
    client.ws_request(url, on_message=on_message) 

def jinja2_filter(app):
    def format_datetime(value):
        dt = arrow.get(value)
        return dt.humanize()

    app.jinja_env.filters["dt"] = format_datetime

    @app.context_processor
    def inject_stage_and_region():
        return dict(
            YEAR=arrow.now().year,
            URL=URL,
            SENTRY_DSN=SENTRY_FRONT_END_DSN,
            VERSION=SHA1,
            FIRST_ALIAS_DOMAIN=FIRST_ALIAS_DOMAIN,
        ) 

def get_entity_data(cls, name, **kwargs):
        account_uuid = kwargs.get("account_uuid", "")
        if not account_uuid:
            LOG.error("Account UUID not supplied for fetching subnet {}".format(name))
            sys.exit(-1)

        cluster_name = kwargs.get("cluster", "")
        try:
            if cluster_name:
                entity = super().get(
                    cls.name == name,
                    cls.cluster == cluster_name,
                    cls.account_uuid == account_uuid,
                )
            else:
                # The get() method is shorthand for selecting with a limit of 1
                # If more than one row is found, the first row returned by the database cursor
                entity = super().get(cls.name == name, cls.account_uuid == account_uuid)
            return entity.get_detail_dict()

        except DoesNotExist:
            return None 

def get_entity_data_using_uuid(cls, uuid, **kwargs):
        account_uuid = kwargs.get("account_uuid", "")
        if not account_uuid:
            LOG.error(
                "Account UUID not supplied for fetching subnet with uuid {}".format(
                    uuid
                )
            )
            sys.exit(-1)

        try:
            entity = super().get(cls.uuid == uuid, cls.account_uuid == account_uuid)
            return entity.get_detail_dict()

        except DoesNotExist:
            return None 

def sync(cls):
        """sync the table data from server"""
        # clear old data
        cls.clear()

        client = get_api_client()
        payload = {"length": 250, "filter": "state==VERIFIED;type==nutanix_pc"}
        account_name_uuid_map = client.account.get_name_uuid_map(payload)

        AhvVmProvider = get_provider("AHV_VM")
        AhvObj = AhvVmProvider.get_api_obj()

        for e_name, e_uuid in account_name_uuid_map.items():
            res = AhvObj.images(account_uuid=e_uuid)
            for entity in res["entities"]:
                name = entity["status"]["name"]
                uuid = entity["metadata"]["uuid"]
                # TODO add proper validation for karbon images
                image_type = entity["status"]["resources"].get("image_type", "")
                cls.create_entry(
                    name=name, uuid=uuid, image_type=image_type, account_uuid=e_uuid
                ) 

def get_entity_data(cls, name, **kwargs):
        account_uuid = kwargs.get("account_uuid", "")
        if not account_uuid:
            LOG.error("Account UUID not supplied for fetching image {}".format(name))
            sys.exit(-1)

        image_type = kwargs.get("image_type", None)
        if not image_type:
            LOG.error("image_type not provided for image {}".format(name))
            sys.exit(-1)

        try:
            entity = super().get(
                cls.name == name,
                cls.image_type == image_type,
                cls.account_uuid == account_uuid,
            )
            return entity.get_detail_dict()

        except DoesNotExist:
            return None 

def get_entity_data_using_uuid(cls, uuid, **kwargs):
        account_uuid = kwargs.get("account_uuid", "")
        if not account_uuid:
            # For now we can attach multiple nutanix_pc account pointing to same pc
            # https://jira.nutanix.com/browse/CALM-19273, So make account_uuid as necessary
            LOG.error(
                "Account UUID not supplied for fetching image with uuid {}".format(uuid)
            )
            sys.exit(-1)

        try:
            entity = super().get(cls.uuid == uuid, cls.account_uuid == account_uuid)
            return entity.get_detail_dict()

        except DoesNotExist:
            return None 

def show_data(cls):
        """display stored data in table"""
        if not len(cls.select()):
            click.echo(highlight_text("No entry found !!!"))
            return

        table = PrettyTable()
        table.field_names = ["NAME", "UUID", "LAST UPDATED"]
        for entity in cls.select():
            entity_data = entity.get_detail_dict()
            last_update_time = arrow.get(
                entity_data["last_update_time"].astimezone(datetime.timezone.utc)
            ).humanize()
            table.add_row(
                [
                    highlight_text(entity_data["name"]),
                    highlight_text(entity_data["uuid"]),
                    highlight_text(last_update_time),
                ]
            )
        click.echo(table) 

def show_data(cls):
        """display stored data in table"""

        if not len(cls.select()):
            click.echo(highlight_text("No entry found !!!"))
            return

        table = PrettyTable()
        table.field_names = ["NAME", "UUID", "LAST UPDATED"]
        for entity in cls.select():
            entity_data = entity.get_detail_dict()
            last_update_time = arrow.get(
                entity_data["last_update_time"].astimezone(datetime.timezone.utc)
            ).humanize()
            table.add_row(
                [
                    highlight_text(entity_data["name"]),
                    highlight_text(entity_data["uuid"]),
                    highlight_text(last_update_time),
                ]
            )
        click.echo(table) 

def get_blueprint(client, name, all=False):

    # find bp
    params = {"filter": "name=={}".format(name)}
    if not all:
        params["filter"] += ";state!=DELETED"

    res, err = client.blueprint.list(params=params)
    if err:
        raise Exception("[{}] - {}".format(err["code"], err["error"]))

    response = res.json()
    entities = response.get("entities", None)
    blueprint = None
    if entities:
        if len(entities) != 1:
            raise Exception("More than one blueprint found - {}".format(entities))

        LOG.info("{} found ".format(name))
        blueprint = entities[0]
    else:
        raise Exception("No blueprint found with name {} found".format(name))
    return blueprint 

def get_val_launch_runtime_vars(launch_runtime_vars, field, path, context):
    """Returns value of variable from launch_runtime_vars(Non-interactive)"""

    filtered_launch_runtime_vars = list(
        filter(
            lambda e: is_launch_runtime_vars_context_matching(e["context"], context)
            and e["name"] == path,
            launch_runtime_vars,
        )
    )
    if len(filtered_launch_runtime_vars) > 1:
        LOG.error(
            "Unable to populate runtime editables: Multiple matches for name {} and context {}".format(
                path, context
            )
        )
        sys.exit(-1)
    if len(filtered_launch_runtime_vars) == 1:
        return filtered_launch_runtime_vars[0].get("value", {}).get(field, None)
    return None 

def get_val_launch_runtime_substrates(launch_runtime_substrates, path, context):
    """Returns value of substrate from launch_runtime_substrates(Non-interactive)"""

    filtered_launch_runtime_substrates = list(
        filter(lambda e: e["name"] == path, launch_runtime_substrates,)
    )
    if len(filtered_launch_runtime_substrates) > 1:
        LOG.error(
            "Unable to populate runtime editables: Multiple matches for name {} and context {}".format(
                path, context
            )
        )
        sys.exit(-1)
    if len(filtered_launch_runtime_substrates) == 1:
        return filtered_launch_runtime_substrates[0].get("value", {})
    return None 

def describe_aws_account(spec):

    click.echo("Access Key ID: {}".format(spec["access_key_id"]))
    regions = spec["regions"]

    click.echo("\nRegions:\n-------------- ")
    for index, region in enumerate(regions):
        click.echo("\t{}. {}".format(str(index + 1), highlight_text(region["name"])))

    click.echo("\nPublic Images:\n-------------- ")
    image_present = False
    for region in regions:
        if region.get("images"):
            click.echo("\nRegion: {}".format(region["name"]))
            click.echo("Images: ")
            for index, image in enumerate(region["images"]):
                image_present = True
                click.echo(
                    "\t{}. {}".format(str(index + 1), highlight_text(image["name"]))
                )

    if not image_present:
        click.echo("\t{}".format(highlight_text("No images provided"))) 

def get_last_full_moon(d):
    """
    Returns the last full moon for d

    Raises ValueError if the d value is not between 2000 - 2099
    """

    now = d.timestamp
    idx = bisect.bisect_right(fullmoons, now)
    if idx in [0, len(fullmoons)]:
        raise ValueError(
            u'watson has only full moon dates from year 2000 to 2099, not {}'
            .format(d.year))

    last = fullmoons[idx - 1]
    return arrow.get(last) 

def current(self, value):
        if not value or 'project' not in value:
            self._current = {}

            if self._old_state is None:
                self._old_state = {}

            return

        start = value.get('start', arrow.now())

        if not isinstance(start, arrow.Arrow):
            start = self._parse_date(start)

        self._current = {
            'project': value['project'],
            'start': start,
            'tags': value.get('tags') or []
        }

        if self._old_state is None:
            self._old_state = self._current 

def _get_remote_projects(self):
        # import when required in order to reduce watson response time (#312)
        import requests
        if not hasattr(self, '_remote_projects'):
            dest, headers = self._get_request_info('projects')

            try:
                response = requests.get(dest, headers=headers)
                assert response.status_code == 200

                self._remote_projects = response.json()
            except requests.ConnectionError:
                raise WatsonError("Unable to reach the server.")
            except AssertionError:
                raise WatsonError(
                    u"An error occurred with the remote "
                    "server: {}".format(response.json())
                )

        return self._remote_projects['projects'] 

def convert(self, value, param, ctx):
        if value:
            date = self._parse_multiformat(value)
            if date is None:
                raise click.UsageError(
                    "Could not match value '{}' to any supported date format"
                    .format(value)
                )
            # When we parse a date, we want to parse it in the timezone
            # expected by the user, so that midnight is midnight in the local
            # timezone, not in UTC. Cf issue #16.
            date.tzinfo = tz.tzlocal()
            # Add an offset to match the week beginning specified in the
            # configuration
            if param.name == "week":
                week_start = ctx.obj.config.get(
                    "options", "week_start", "monday")
                date = apply_weekday_offset(
                    start_time=date, week_start=week_start)
            return date 

def _get_deployment_duration(deployment_status):
        """
        Given a deployment status, calculate and return the duration.
        For some reason the "Duration" parameter is not always populated
        from the OpsWorks API, so this works around that.
        :param deployment_status:
        :return:
        """
        started_at = arrow.get(deployment_status['CreatedAt'])
        completed_at = arrow.get(deployment_status['CompletedAt'])
        return completed_at - started_at 

def get_cli_config_vars():
    """ get CLI options. use of these options should be rare """
    usage = 'Usage: %prog [options]'
    parser = OptionParser(usage=usage)
    """
    ## not ready.
    parser.add_option('--threads', default=1, action='store', dest='threads',
                      help='Number of threads to run')
    """
    parser.add_option('-q', '--quiet', action='store_true', dest='quiet',
                      help='Only display warning and error log messages')
    parser.add_option('-v', '--verbose', action='store_true', dest='verbose',
                      help='Enable verbose logging')
    parser.add_option('-t', '--testing', action='store_true', dest='testing',
                      help='Set to testing mode (do not send data).' +
                           ' Automatically turns on verbose logging')
    (options, args) = parser.parse_args()

    """
    # not ready
    try:
        threads = int(options.threads)
    except ValueError:
        threads = 1
    """

    config_vars = {
        'threads': 1,
        'testing': False,
        'log_level': logging.INFO
    }

    if options.testing:
        config_vars['testing'] = True

    if options.verbose or options.testing:
        config_vars['log_level'] = logging.DEBUG
    elif options.quiet:
        config_vars['log_level'] = logging.WARNING

    return config_vars 

def get_json_size_bytes(json_data):
    """ get size of json object in bytes """
    return len(bytearray(json.dumps(json_data))) 

def get_data_values(timestamp, message):
    setting_values = agent_config_vars['data_fields'] or message.keys()
    # reverse list so it's in priority order, as shared fields names will get overwritten
    setting_values.reverse()
    data = {x: dict() for x in timestamp}
    for setting_value in setting_values:
        name, value = get_data_value(message, setting_value)
        if isinstance(value, (set, tuple, list)):
            for i in range(minlen(timestamp, value)):
                merge_data(name, value[i], data[timestamp[i]])
        else:
            merge_data(name, value, data[timestamp[0]])
    return data 

def get_data_value(message, setting_value):
    if is_named_data_field(setting_value):
        setting_value = setting_value.split(':')
        # get name
        name = setting_value[0]
        if is_formatted(name):
            name = parse_formatted(message,
                                   name,
                                   default=name)
        # get value
        value = setting_value[1]
        # check if math
        evaluate = False
        if is_math_expr(value):
            evaluate = True
            value = get_math_expr(value)
        if is_formatted(value):
            value = parse_formatted(message,
                                    value,
                                    default=value,
                                    allow_list=True)
        if evaluate:
            value = eval(value)
    else:
        name = setting_value
        value = get_json_field(message,
                               setting_value,
                               allow_list=True)
    return (name, value) 

def parse_csv_data(csv_data, instance, device=''):
    """
    parses CSV data, assuming the format is given as:
        header row:  timestamp,field_1,field_2,...,field_n
        n data rows: TIMESTAMP,value_1,value_2,...,value_n
    """

    # get field names from header row
    field_names = csv_data.pop(0).split(CSV_DELIM)[1:]

    # go through each row
    for row in csv_data:
        if len(row) > 0:
            parse_csv_row(row.split(CSV_DELIM), field_names, instance, device) 

def build_metric_name_map():
    '''
    Contstructs a hash of <raw_metric_name>: <formatted_metric_name>
    '''
    # get metrics from the global
    metrics = agent_config_vars['metrics_copy']
    # initialize the hash of formatted names
    agent_config_vars['metrics_names'] = dict()
    tree = build_sentence_tree(metrics)
    min_tree = fold_up(tree, sentence_tree=True) 

def send_request(url, mode='GET', failure_message='Failure!', success_message='Success!', **request_passthrough):
    """ sends a request to the given url """
    # determine if post or get (default)
    req = requests.get
    if mode.upper() == 'POST':
        req = requests.post

    for i in range(ATTEMPTS):
        try:
            response = req(url, **request_passthrough)
            if response.status_code == httplib.OK:
                logger.info(success_message)
                return response
            else:
                logger.warn(failure_message)
                logger.debug('Response Code: {}\nTEXT: {}'.format(
                    response.status_code, response.text))
        # handle various exceptions
        except requests.exceptions.Timeout:
            logger.exception('Timed out. Reattempting...')
            continue
        except requests.exceptions.TooManyRedirects:
            logger.exception('Too many redirects.')
            break
        except requests.exceptions.RequestException as e:
            logger.exception('Exception ' + str(e))
            break

    logger.error('Failed! Gave up after {} attempts.'.format(i))
    return -1 

def get_json_size_bytes(json_data):
    """ get size of json object in bytes """
    return len(bytearray(json.dumps(json_data)))