def ensure_flattened(data):
    """
    Will ensure that the supplied data is "flattened". The input data can be a
    response from the Twitter API, a list of tweet dictionaries, or a single tweet
    dictionary. It will always return a list of tweet dictionaries. A ValueError
    will be thrown if the supplied data is not recognizable or it cannot be
    flattened.

    ensure_flattened is designed for use in twarc plugins and other tweet
    processing applications that want to operate on a stream of tweets, and
    examine included entities like users and tweets without hunting and
    pecking in the response data.
    """

    # If it's a single response from the API, with data and includes, we flatten it:
    if isinstance(data, dict) and "data" in data and "includes" in data:
        return flatten(data)

    # If it's a single response with data, but without includes:
    elif isinstance(data, dict) and "data" in data and "includes" not in data:
        # flatten() will still work, just with {} empty expansions, log a warning.
        log.warning(f"Unable to expand dictionary without includes: {data}")
        return flatten(data)

    # If it's a single response and both "includes" and "data" are missing, it is already flattened
    elif isinstance(data, dict) and "data" not in data and "includes" not in data:
        return [data]

    # If it's a list of objects (could be list of responses, or tweets, or users):
    elif isinstance(data, list) and len(data) > 0 and isinstance(data[0], dict):
        # Same as above,
        if "data" in data[0] and "includes" in data[0]:
            # but flatten each object individually and return a single list
            return list(chain.from_iterable([flatten(item) for item in data]))
        elif "data" in data[0] and "includes" not in data[0]:
            # same as above, log warnings and return a single list
            log.warning(f"Unable to expand dictionary without includes: {data[0]}")
            return list(chain.from_iterable([flatten(item) for item in data]))
        # Return already flattened data as is
        elif "data" not in data[0] and "includes" not in data[0]:
            return data
    # Unknown format, eg: list of lists, or primitive
    else:
        raise ValueError(f"Cannot flatten unrecognized data: {data}") EXPANSIONS = [
    "author_id",
    "in_reply_to_user_id",
    "referenced_tweets.id",
    "referenced_tweets.id.author_id",
    "entities.mentions.username",
    "attachments.poll_ids",
    "attachments.media_keys",
    "geo.place_id",
] TWEET_FIELDS = [
    "attachments",
    "author_id",
    "context_annotations",
    "conversation_id",
    "created_at",
    "entities",
    "geo",
    "id",
    "in_reply_to_user_id",
    "lang",
    "public_metrics",
    # "non_public_metrics", # private
    # "organic_metrics", # private
    # "promoted_metrics", # private
    "text",
    "possibly_sensitive",
    "referenced_tweets",
    "reply_settings",
    "source",
    "withheld",
] USER_FIELDS = [
    "created_at",
    "description",
    "entities",
    "id",
    "location",
    "name",
    "pinned_tweet_id",
    "profile_image_url",
    "protected",
    "public_metrics",
    "url",
    "username",
    "verified",
    "withheld",
] MEDIA_FIELDS = [
    "alt_text",
    "duration_ms",
    "height",
    "media_key",
    "preview_image_url",
    "type",
    "url",
    "width",
    # "non_public_metrics", # private
    # "organic_metrics", # private
    # "promoted_metrics", # private
    "public_metrics",
] POLL_FIELDS = ["duration_minutes", "end_datetime", "id", "options", "voting_status"] PLACE_FIELDS = [
    "contained_within",
    "country",
    "country_code",
    "full_name",
    "geo",
    "id",
    "name",
    "place_type",
] class cli_api_error:
    """
    A decorator to catch HTTP errors for the command line.
    """

    def __init__(self, f):
        self.f = f
        # this is needed for click help docs to work properly
        self.__doc__ = f.__doc__

    def __call__(self, *args, **kwargs):
        try:
            return self.f(*args, **kwargs)
        except requests.exceptions.HTTPError as e:
            try:
                result = e.response.json()
                if "errors" in result:
                    for error in result["errors"]:
                        msg = error.get("message", "Unknown error")
                elif "title" in result:
                    msg = result["title"]
                else:
                    msg = "Unknown error"
            except ValueError:
                msg = f"Unable to parse {e.response.status_code} error as JSON: {e.response.text}"
        except InvalidAuthType as e:
            msg = "This command requires application authentication, try passing --app-auth"
        except ValueError as e:
            msg = str(e)
        click.echo(
            click.style("⚡ ", fg="yellow") + click.style(msg, fg="red"), err=True
        ) class TimestampProgressBar(tqdm):
    """
    A Timestamp based progress bar. Counts timestamp ranges in milliseconds.
    This can be used to display a progress bar for tweet ids and time ranges.
    """

    def __init__(self, since_id, until_id, start_time, end_time, **kwargs):
        self.early_stop = True
        self.tweet_count = 0

        disable = False if "disable" not in kwargs else kwargs["disable"]
        kwargs["disable"] = disable

        if start_time is None and (since_id is None and until_id is None):
            start_time = datetime.datetime.now(
                datetime.timezone.utc
            ) - datetime.timedelta(days=7)
        if end_time is None and (since_id is None and until_id is None):
            end_time = datetime.datetime.now(
                datetime.timezone.utc
            ) - datetime.timedelta(seconds=30)

        if since_id and not until_id:
            until_id = _millis2snowflake(
                _date2millis(datetime.datetime.now(datetime.timezone.utc))
            )

        if until_id and not since_id:
            since_id = 1

        total = (
            _snowflake2millis(until_id) - _snowflake2millis(since_id)
            if (since_id and until_id)
            else _date2millis(end_time) - _date2millis(start_time)
        )

        kwargs["miniters"] = 1
        kwargs["total"] = total
        tweets_timeline_format = "{l_bar}{bar}| Processed {n_time}/{total_time} [{elapsed}<{remaining}, {tweet_count} tweets total {postfix}]"
        kwargs["bar_format"] = (
            tweets_timeline_format
            if "bar_format" not in kwargs
            else kwargs["bar_format"]
        )
        super().__init__(**kwargs)

    def update_with_dates(self, start_span, end_span):
        """
        Update the progress bar with a start and end time span.
        """
        try:
            if isinstance(start_span, str):
                start_span = datetime.datetime.strptime(
                    start_span, "%Y-%m-%dT%H:%M:%S.%fZ"
                )
            if isinstance(end_span, str):
                end_span = datetime.datetime.strptime(end_span, "%Y-%m-%dT%H:%M:%S.%fZ")
            n = _date2millis(end_span) - _date2millis(start_span)
            if self.n + n > self.total:
                self.n = self.total
            else:
                self.update(n)
        except Exception as e:
            log.error(f"Failed to update progress bar: {e}")

    def update_with_result(self, result):
        """
        Update progress bar based on snowflake ids from an API response.
        """
        try:
            newest_id = result["meta"]["newest_id"]
            oldest_id = result["meta"]["oldest_id"]
            n = _snowflake2millis(int(newest_id)) - _snowflake2millis(int(oldest_id))
            self.update(n)
            self.tweet_count += len(result["data"])
        except Exception as e:
            log.error(f"Failed to update progress bar: {e}")

    @property
    def format_dict(self):
        d = super(TimestampProgressBar, self).format_dict  # original format dict
        tweets_per_second = int(self.tweet_count / d["elapsed"] if d["elapsed"] else 0)
        n_time = humanize.naturaldelta(datetime.timedelta(seconds=int(d["n"]) // 1000))
        total_time = humanize.naturaldelta(
            datetime.timedelta(seconds=int(d["total"]) // 1000)
        )
        d.update(n_time=n_time)
        d.update(total_time=total_time)
        d.update(tweet_count=self.tweet_count)
        d.update(tweets_per_second=tweets_per_second)
        return d

    def close(self):
        if not self.early_stop:
            # Finish the bar to 100% even if the last tweet ids do not cover the full time range
            self.update(self.total - self.n)
        super().close() class FileSizeProgressBar(tqdm):
    """
    An input file size based progress bar. Counts an input file in bytes.
    This will also dig into the responses and add up the outputs to match the file size.
    Overrides `disable` parameter if file is a pipe.
    """

    def __init__(self, infile, outfile, **kwargs):
        disable = False if "disable" not in kwargs else kwargs["disable"]
        if infile is not None and (infile.name == "<stdin>"):
            disable = True
        if outfile is not None and (outfile.name == "<stdout>"):
            disable = True
        kwargs["disable"] = disable
        kwargs["unit"] = "B"
        kwargs["unit_scale"] = True
        kwargs["unit_divisor"] = 1024
        kwargs["miniters"] = 1
        kwargs[
            "bar_format"
        ] = "{l_bar}{bar}| Processed {n_fmt}/{total_fmt} of input file [{elapsed}<{remaining}, {rate_fmt}{postfix}]"
        kwargs["total"] = os.stat(infile.name).st_size if not disable else 1
        super().__init__(**kwargs)

    def update_with_result(
        self, result, field="id", error_resource_type=None, error_parameter="ids"
    ):
        """
        Update the progress bar appropriately, with a full API response. For convenience,
        otherwise use twdm's own update() method.
        """
        try:
            if "data" in result:
                for item in result["data"]:
                    # Use the length of the id / name and a newline to match original file
                    self.update(len(item[field]) + len("\n"))
            if error_resource_type and "errors" in result:
                for error in result["errors"]:
                    # Account for deleted data
                    # Errors have very inconsistent format, missing fields for different types of errors...
                    if (
                        "resource_type" in error
                        and error["resource_type"] == error_resource_type
                    ):
                        if (
                            "parameter" in error
                            and error["parameter"] == error_parameter
                        ):
                            self.update(len(error["value"]) + len("\n"))
                            # todo: hide or show this?
                            # self.set_description(
                            #    "Errors encountered, results may be incomplete"
                            # )
                        # print(error["value"], error["resource_type"], error["parameter"])
        except Exception as e:
            log.error(f"Failed to update progress bar: {e}") class FileLineProgressBar(tqdm):
    """
    A progress bar based on input file line count. Counts an input file by lines.
    This tries to read the entire file and count newlines in a robust way.
    """

    def __init__(self, infile, outfile, **kwargs):
        disable = False if "disable" not in kwargs else kwargs["disable"]
        if infile is not None and (infile.name == "<stdin>"):
            disable = True
        if outfile is not None and (outfile.name == "<stdout>"):
            disable = True
        kwargs["disable"] = disable
        kwargs["miniters"] = 1
        kwargs[
            "bar_format"
        ] = "{l_bar}{bar}| Processed {n_fmt}/{total_fmt} lines of input file [{elapsed}<{remaining}, {rate_fmt}{postfix}]"

        # Warn for large (> 1 GB) input files:
        if (os.stat(infile.name).st_size / (1024 * 1024 * 1024)) > 1:
            click.echo(
                click.style(
                    f"Input File Size is {os.stat(infile.name).st_size / (1024*1024):.2f} MB, it may take a while to process. CTRL+C to stop.",
                    fg="yellow",
                    bold=True,
                ),
                err=True,
            )

        with open(infile.name, "r", encoding="utf-8", errors="ignore") as f:
            total_lines = sum(1 for _ in f)

        kwargs["total"] = total_lines if not disable else 1
        super().__init__(**kwargs)

    def update_with_result(
        self, result, field="id", error_resource_type=None, error_parameter="ids"
    ):
        """
        Update the progress bar appropriately, with a full API response. For convenience,
        and drop in compatibility with FileSizeProgressBar otherwise use tqdm's update().
        """

        try:
            if "data" in result:
                for item in result["data"]:
                    self.update()
            if error_resource_type and "errors" in result:
                for error in result["errors"]:
                    # Account for deleted data
                    # Errors have very inconsistent format, missing fields for different types of errors...
                    if (
                        "resource_type" in error
                        and error["resource_type"] == error_resource_type
                    ):
                        if (
                            "parameter" in error
                            and error["parameter"] == error_parameter
                        ):
                            self.update()
                            # todo: hide or show this?
                            # self.set_description(
                            #    "Errors encountered, results may be incomplete"
                            # )
                        # print(error["value"], error["resource_type"], error["parameter"])
        except Exception as e:
            log.error(f"Failed to update progress bar: {e}") def _millis2snowflake(ms):
    return (int(ms) - 1288834974657) << 22 def _date2millis(dt):
    return int(dt.timestamp() * 1000)
from itertools import filterfalse
from tqdm.auto import tqdm
from tqdm.utils import CallbackIOWrapper
from datetime import timezone
from click_plugins import with_plugins
from pkg_resources import iter_entry_points
from twarc.version import version
from twarc.handshake import handshake
from twarc.config import ConfigProvider
from twarc.expansions import (
    ensure_flattened,
    EXPANSIONS,
    TWEET_FIELDS,
    USER_FIELDS,
    MEDIA_FIELDS,
    POLL_FIELDS,
    PLACE_FIELDS,
)
from click import command, option, Option, UsageError
from click_config_file import configuration_option
from twarc.decorators2 import (
    cli_api_error,
    TimestampProgressBar,
    FileSizeProgressBar,
    FileLineProgressBar,
    _millis2snowflake,
    _date2millis,
)
import os
import re
import json
import time
import twarc
import click
import logging
import pathlib
import datetime
import humanize
import requests
import configobj
import threading
)
@click.option(
    "--max-results",
    default=100,
    help="Maximum number of liked tweets per page of results. Default and maximum is 100.",
    type=int,
)
@command_line_progressbar_option
@click.argument("user_id", type=str)
@click.argument("outfile", type=click.File("w"), default="-")
@click.pass_obj
@cli_api_error
def liked_tweets(T, user_id, outfile, limit, max_results, hide_progress):
    """
    Get the tweets liked by a specific user_id.

    Note that the progress bar is approximate.

    """
    count = 0
    lookup_total = 0

    if not re.match("^\d+$", str(user_id)):
        click.echo(click.style("Please enter a user ID", fg="red"), err=True)

    if outfile is not None and (outfile.name == "<stdout>"):
        hide_progress = True

    # NB: there doesn't appear to be anyway to get the total count of likes
    # a user has made, so the progress bar isn't very useful in this case...

    with tqdm(disable=hide_progress) as progress:
        for result in T.liked_tweets(user_id, max_results=max_results):
            _write(result, outfile)
            count += len(result.get("data", []))
            progress.update(len(result.get("data", [])))
            if limit != 0 and count >= limit:
                progress.desc = f"Set --limit of {limit} reached"
                break


@twarc2.command("sample")
@command_line_expansions_shortcuts
@command_line_expansions_options
@click.option("--limit", default=0, help="Maximum number of tweets to save")
@click.argument("outfile", type=click.File("a+"), default="-")
@click.pass_obj
@cli_api_error
def sample(T, outfile, limit, **kwargs):
    """
    Fetch tweets from the sample stream.
    """

    kwargs = _process_expansions_shortcuts(kwargs)

    count = 0
    event = threading.Event()
    click.echo(
        click.style(
            f"Started a random sample stream, writing to {outfile.name}\nCTRL+C to stop...",
            fg="green",
        ),
        err=True,
    )
    for result in T.sample(event=event, **kwargs):
        count += 1
        if limit != 0 and count >= limit:
            event.set()
        _write(result, outfile)

        if result and "data" in result:
            log.info("archived %s", result["data"]["id"])


@twarc2.command("hydrate")
@command_line_expansions_shortcuts
@command_line_expansions_options
@command_line_input_output_file_arguments
@command_line_progressbar_option
@click.pass_obj
@cli_api_error
def hydrate(T, infile, outfile, hide_progress, **kwargs):
    """
    Hydrate tweet ids.
    """

    kwargs = _process_expansions_shortcuts(kwargs)

    with FileLineProgressBar(infile, outfile, disable=hide_progress) as progress:
        for result in T.tweet_lookup(infile, **kwargs):
            _write(result, outfile)
            tweet_ids = [t["id"] for t in result.get("data", [])]
            log.info("archived %s", ",".join(tweet_ids))
            progress.update_with_result(result, error_resource_type="tweet")


@twarc2.command("dehydrate")
@click.option(
    "--id-type",
    default="tweets",
    type=click.Choice(["tweets", "users"], case_sensitive=False),
    help="IDs to extract - either 'tweets' or 'users'.",
)
@command_line_progressbar_option
@command_line_input_output_file_arguments
@cli_api_error
def dehydrate(infile, outfile, id_type, hide_progress):
    """
    Extract tweet or user IDs from a dataset.
    """
    if infile.name == outfile.name:
        click.echo(
            click.style(
                f"💔 Cannot extract files in-place, specify a different output file!",
                fg="red",
            ),
            err=True,
        )
        return


    with FileSizeProgressBar(infile, outfile, disable=hide_progress) as progress: