class D4S2Project(object):
    """
    Transfers a project to another user via share and final delivery.
    Uses eternal API to send messages.
    """
    def __init__(self, config, remote_store, print_func):
        """
        Setup for sharing a project and sending email on the D4S2 service.
        :param config: Config configuration specifying which remote_store to use.
        :param remote_store: RemoteStore remote store we will be sharing a project from
        :param print_func: func used to print output somewhere
        """
        self.config = config
        self.client = Client(config)
        auth = DataServiceAuth(self.config)
        api_token = auth.get_auth()
        self.api = D4S2Api(config.d4s2_url, api_token)
        self.remote_store = remote_store
        self.print_func = print_func

    def share(self, project, to_user, force_send, auth_role, user_message):
        """
        Send mail and give user specified access to the project.
        :param project: RemoteProject project to share
        :param to_user: RemoteUser user to receive email/access
        :param auth_role: str project role eg 'project_admin' to give to the user
        :param user_message: str message to be sent with the share
        :return: str email we share the project with
        """
        if self._is_current_user(to_user):
            raise ShareWithSelfError(SHARE_WITH_SELF_MESSAGE.format("share"))
        if not to_user.email:
            self._raise_user_missing_email_exception("share")
        self.set_user_project_permission(project, to_user, auth_role)
        return self._share_project(D4S2Api.SHARE_DESTINATION, project, to_user, force_send, auth_role, user_message)

    def set_user_project_permission(self, project, user, auth_role):
        """
        Give user access permissions for a project.
        :param project: RemoteProject project to update permissions on
        :param user: RemoteUser user to receive permissions
        :param auth_role: str project role eg 'project_admin'
        """
        self.remote_store.set_user_project_permission(project, user, auth_role)

    def deliver(self, project, new_project_name, to_user, share_users, force_send, path_filter, user_message):
        """
        Remove access to project_name for to_user, copy to new_project_name if not None,
        send message to service to email user so they can have access.
        :param project: RemoteProject pre-existing project to be delivered
        :param new_project_name: str name of non-existing project to copy project_name to, if None we don't copy
        :param to_user: RemoteUser user we are handing over the project to
        :param share_users: [RemoteUser] who will have project shared with them once to_user accepts the project
        :param force_send: boolean enables resending of email for existing projects
        :param path_filter: PathFilter: filters what files are shared
        :param user_message: str message to be sent with the share
        :return: str email we sent deliver to
        """
        if self._is_current_user(to_user):
            raise ShareWithSelfError(SHARE_WITH_SELF_MESSAGE.format("deliver"))
        if not to_user.email:
            self._raise_user_missing_email_exception("deliver")
        self.remove_user_permission(project, to_user)
        if new_project_name:
            project = self._copy_project(project, new_project_name, path_filter)
        return self._share_project(D4S2Api.DELIVER_DESTINATION, project, to_user,
                                   force_send, user_message=user_message, share_users=share_users)

    def remove_user_permission(self, project, user):
        """
        Take away user's access to project.
        :param project: RemoteProject project to remove permissions on
        :param user: RemoteUser user who should no longer have access
        """
        self.remote_store.revoke_user_project_permission(project, user)

    def _share_project(self, destination, project, to_user, force_send, auth_role='', user_message='',
                       share_users=None):
        """
        Send message to remote service to email/share project with to_user.
        :param destination: str which type of sharing we are doing (SHARE_DESTINATION or DELIVER_DESTINATION)
        :param project: RemoteProject project we are sharing
        :param to_user: RemoteUser user we are sharing with
        :param auth_role: str project role eg 'project_admin' email is customized based on this setting.
        :param user_message: str message to be sent with the share
        :param share_users: [RemoteUser] users to have this project shared with after delivery (delivery only)
        :return: the email the user should receive a message on soon
        """
        from_user = self.remote_store.get_current_user()
        share_user_ids = None
        if share_users:
            share_user_ids = [share_user.id for share_user in share_users]
        item = D4S2Item(destination=destination,
                        from_user_id=from_user.id,
                        to_user_id=to_user.id,
                        project_id=project.id,
                        project_name=project.name,
                        auth_role=auth_role,
                        user_message=user_message,
                        share_user_ids=share_user_ids)
        item.send(self.api, force_send)
        return to_user.email

    def _copy_project(self, project, new_project_name, path_filter):
        """
        Copy pre-existing project with name project_name to non-existing project new_project_name.
        :param project: remotestore.RemoteProject project to copy from
        :param new_project_name: str project to copy to
        :param path_filter: PathFilter: filters what files are shared
        :return: RemoteProject new project we copied data to
        """
        temp_directory = tempfile.mkdtemp()
        new_project_name_or_id = ProjectNameOrId.create_from_name(new_project_name)
        remote_project = self.remote_store.fetch_remote_project(new_project_name_or_id)
        if remote_project:
            raise DDSUserException("A project with name '{}' already exists.".format(new_project_name))
        activity = CopyActivity(self.remote_store.data_service, project, new_project_name)
        self._download_project(activity, project.id, temp_directory, path_filter)
        self._upload_project(activity, new_project_name, temp_directory)
        activity.finished()
        shutil.rmtree(temp_directory)
        return self.remote_store.fetch_remote_project(new_project_name_or_id, must_exist=True)

    def _download_project(self, activity, project_id, temp_directory, path_filter):
        """
        Download the project with project_name to temp_directory.
        :param activity: CopyActivity: info about the copy activity are downloading for
        :param project_id: uuid of the project to download
        :param temp_directory: str path to directory we can download into
        :param path_filter: PathFilter: filters what files are shared
        """
        project = self.client.get_project_by_id(project_id)
        self.print_func("Downloading a copy of '{}'.".format(project.name))

        downloader = ProjectFileDownloader(self.config, temp_directory, project, path_filter)
        downloader.run()

        downloaded_file_relations = DownloadedFileRelations(activity)
        for remote_path, dds_file in project.get_path_to_files().items():
            if path_filter.include_path(remote_path):
                downloaded_file_relations.add(self.remote_store.data_service,
                                              remote_path,
                                              dds_file.current_version['id'])

    def _upload_project(self, activity, project_name, temp_directory):
        """
        Upload the contents of temp_directory into project_name
        :param activity: CopyActivity: info about the copy activity are uploading for
        :param project_name: str project name we will upload files to
        :param temp_directory: str path to directory who's files we will upload
        """
        self.print_func("Uploading to '{}'.".format(project_name))
        paths_to_send = [os.path.join(temp_directory, item) for item in os.listdir(os.path.abspath(temp_directory))]
        project_name_or_id = ProjectNameOrId.create_from_name(project_name)

        project_upload = ProjectUpload.create_for_paths(self.config, self.remote_store, project_name_or_id,
                                                        paths_to_send,
                                                        file_upload_post_processor=UploadedFileRelations(activity))
        project_upload.run()

    def _is_current_user(self, some_user):
        """
        Is the specified user the current user?
        :param some_user: RemoteUser user we want to check against the current user
        :return: boolean: True if the current user is the passed in user
        """
        current_user = self.remote_store.get_current_user()
        return current_user.id == some_user.id

    @staticmethod
    def _raise_user_missing_email_exception(cmd):
        msg = USER_WITHOUT_EMAIL_MESSAGE.format(cmd, cmd)
        raise UserMissingEmailError(msg) class D4S2Error(DDSUserException):
    def __init__(self, message, warning=False):
        """
        Setup error.
        :param message: str reason for the error
        :param warning: boolean is this just a warning
        """
        Exception.__init__(self, message)
        self.message = message
        self.warning = warning class RemoteStore(object):
    """
    Fetches project tree data from remote store.
    """
    def __init__(self, config, data_service=None):
        """
        Setup to allow fetching project tree.
        :param config: ddsc.config.Config settings to use for connecting to the dataservice.
        :param data_service: DataServiceApi: optional param to specify an existing DataServiceApi object
        """
        self.config = config
        if data_service:
            self.data_service = data_service
        else:
            auth = DataServiceAuth(self.config)
            self.data_service = DataServiceApi(auth, self.config.url)

    def fetch_remote_project(self, project_name_or_id, must_exist=False, include_children=True):
        """
        Retrieve the project via project name or id.
        :param project_name_or_id: ProjectNameOrId name or id of the project to fetch
        :param must_exist: should we error if the project doesn't exist
        :param include_children: should we read children(folders/files)
        :return: RemoteProject project requested or None if not found(and must_exist=False)
        """
        project = self._get_my_project(project_name_or_id)
        if project:
            if include_children:
                self._add_project_children(project, PROJECT_LIST_EXCLUDE_RESPONSE_FIELDS)
        else:
            if must_exist:
                project_description = project_name_or_id.description()
                raise NotFoundError(u'There is no project with the {}'.format(project_description))
        return project

    def fetch_remote_project_by_id(self, id):
        """
        Retrieves project from via id
        :param id: str id of project from data service
        :return: RemoteProject we downloaded
        """
        response = self.data_service.get_project_by_id(id).json()
        return RemoteProject(response)

    def _get_my_project(self, project_name_or_id):
        """
        Return project tree root for project_name_or_id.
        :param project_name_or_id: ProjectNameOrId name or id of the project to lookup
        :return: RemoteProject project we found or None
        """
        response = self.data_service.get_projects().json()
        for project in response['results']:
            if project_name_or_id.contained_in_dict(project):
                return RemoteProject(project)
        return None

    def _add_project_children(self, project, exclude_response_fields=None):
        """
        Add the rest of the project tree from the remote store to the project object.
        :param project: RemoteProject root of the project tree to add children too
        :param exclude_response_fields: [str]: list of fields to exclude in the children response items
        """
        response = self.data_service.get_project_children(project.id, '', exclude_response_fields).json()
        project_children = RemoteProjectChildren(project.id, response['results'])
        for child in project_children.get_tree():
            project.add_child(child)

    def lookup_or_register_user_by_email_or_username(self, email, username):
        """
        Lookup user by email or username. Only fill in one field.
        For both cases it will try to register if not found.
        :param email: str: email address of the user
        :param username: netid of the user to find
        :return: RemoteUser
        """
        if username:
            return self.get_or_register_user_by_username(username)
        else:
            return self.get_or_register_user_by_email(email)

    def lookup_user_by_name(self, full_name):
        """
        Query remote store for a single user with the name full_name or raise error.
        :param full_name: str Users full name separated by a space.
        :return: RemoteUser user info for single user with full_name
        """
        res = self.data_service.get_users_by_full_name(full_name)
        json_data = res.json()
        results = json_data['results']
        found_cnt = len(results)
        if found_cnt == 0:
            raise NotFoundError("User not found:" + full_name)
        elif found_cnt > 1:
            raise DDSUserException("Multiple users with name:" + full_name)
        user = RemoteUser(results[0])
        if user.full_name.lower() != full_name.lower():
            raise NotFoundError("User not found:" + full_name)
        return user

    def get_or_register_user_by_username(self, username):
        """
        Try to lookup user by username. If not found try registering the user.
        :param username: str: username to lookup
        :return: RemoteUser: user we found
        """
        util = UserUtil(self.data_service)
        user_json = util.find_user_by_username(username)
        if not user_json:
            user_json = util.register_user_by_username(username)
        return RemoteUser(user_json)

    def get_or_register_user_by_email(self, email):
        """
        Try to lookup user by email. If not found try registering the user.
        Raises ValueError when unable to find/register a user for the email.
        :param email: str: email to lookup or register a user for
        :return: RemoteUser: user we found
        """
        util = UserUtil(self.data_service)
        user_json = util.find_user_by_email(email)
        if not user_json:
            affiliate = util.find_affiliate_by_email(email)
            if affiliate:
                user_json = util.register_user_by_username(affiliate['uid'])
            else:
                raise DDSUserException("Unable to find or register a user with email {}".format(email))
        return RemoteUser(user_json)

    def get_auth_providers(self):
        """
        Return the list of authorization providers.
        :return: [RemoteAuthProvider]: list of remote auth providers
        """
        providers = []
        response = self.data_service.get_auth_providers().json()
        for data in response['results']:
            providers.append(RemoteAuthProvider(data))
        return providers

    def get_current_user(self):
        """
        Fetch info about the current user
        :return: RemoteUser user who we are logged in as(auth determines this).
        """
        response = self.data_service.get_current_user().json()
        return RemoteUser(response)

    def fetch_users(self, email=None, username=None):
        """
        Retrieves users with optional email and/or username filtering from data service.
        :param email: str: optional email to filter by
        :param username: str: optional username to filter by
        :return: [RemoteUser] list of all users we downloaded
        """
        users = []
        result = self.data_service.get_users(email=email, username=username)
        user_list_json = result.json()
        for user_json in user_list_json['results']:
            users.append(RemoteUser(user_json))
        return users

    def fetch_user(self, id):
        """
        Retrieves user from data service having a specific id
        :param id: str id of user from data service
        :return: RemoteUser user we downloaded
        """
        response = self.data_service.get_user_by_id(id).json()
        return RemoteUser(response)

    def set_user_project_permission(self, project, user, auth_role):
        """
        Update remote store for user giving auth_role permissions on project.
        :param project: RemoteProject project to give permissions to
        :param user: RemoteUser user who we are giving permissions to
        :param auth_role: str type of authorization to give user(project_admin)
        """
        self.data_service.set_user_project_permission(project.id, user.id, auth_role)

    def revoke_user_project_permission(self, project, user):
        """
        Update remote store for user removing auth_role permissions on project.
        :param project: RemoteProject project to remove permissions from
        :param user: RemoteUser user who we are removing permissions from
        """
        # Server errors out with 500 if a user isn't found.
        try:
            self.data_service.get_user_project_permission(project.id, user.id)
            self.data_service.revoke_user_project_permission(project.id, user.id)
        except DataServiceError as e:
            if e.status_code != 404:
                raise

    def download_file(self, remote_file, path, watcher):
        """
        Download a remote file associated with the remote uuid(file_id) into local path.
        :param remote_file: RemoteFile file to retrieve
        :param path: str file system path to save the contents to.
        :param watcher: object implementing send_item(item, increment_amt) that updates UI
        """
        url_json = self.data_service.get_file_url(remote_file.id).json()
        http_verb = url_json['http_verb']
        host = url_json['host']
        url = url_json['url']
        http_headers = url_json['http_headers']
        response = self.data_service.receive_external(http_verb, host, url, http_headers)
        with open(path, 'wb') as f:
            for chunk in response.iter_content(chunk_size=DOWNLOAD_FILE_CHUNK_SIZE):
                if chunk:  # filter out keep-alive new chunks
                    f.write(chunk)
                    watcher.transferring_item(remote_file, increment_amt=len(chunk))

    def get_project_names(self):
        """
        Return a list of names of the remote projects owned by this user.
        :return: [str]: the list of project names
        """
        names = []
        response = self.data_service.get_projects().json()
        for project in response['results']:
            names.append(project['name'])
        return names

    def get_projects_details(self):
        """
        Return list of top level details for all projects
        """
        return self.data_service.get_projects().json()['results']

    def get_projects_with_auth_role(self, auth_role):
        """
        Return the list of projects that have the specified auth role from the list that the current user has access to.
        :param auth_role: str: auth role we are filtering for
        :return: [dict]: list of projects that have auth_role permissions for the current user
        """
        user = self.get_current_user()
        # user.id
        projects = []
        response = self.data_service.get_projects().json()
        for project in response['results']:
            project_id = project['id']
            permissions = self.data_service.get_user_project_permission(project_id, user.id).json()
            if auth_role == permissions['auth_role']['id']:
                projects.append(project)
        return projects

    def delete_project(self, project_name_or_id):
        """
        Find the project with project_name_or_id and delete it raise error if not found.
        :param project_name_or_id: ProjectNameOrId: name or id of the project we want to be deleted
        """
        project = self._get_my_project(project_name_or_id)
        if project:
            self.data_service.delete_project(project.id)
        else:
            raise DDSUserException("No project with {} found.\n".format(project_name_or_id.description()))

    def get_active_auth_roles(self, context):
        """
        Retrieve non-deprecated authorization roles based on a context.
        Context should be RemoteAuthRole.PROJECT_CONTEXT or RemoteAuthRole.SYSTEM_CONTEXT.
        :param context: str: context for which auth roles to retrieve
        :return: [RemoteAuthRole]: list of active auth_role objects
        """
        response = self.data_service.get_auth_roles(context).json()
        return self.get_active_auth_roles_from_json(response)

    @staticmethod
    def get_active_auth_roles_from_json(json_data):
        """
        Given a json blob response containing a list of authorization roles return the active ones
        in an array of RemoteAuthRole objects.
        :param json_data: list of dictionaries - data from dds in auth_role format
        :return: [RemoteAuthRole] list of active auth_role objects
        """
        result = []
        for auth_role_properties in json_data['results']:
            auth_role = RemoteAuthRole(auth_role_properties)
            if not auth_role.is_deprecated:
                result.append(auth_role)
        return result

    def get_project_files(self, project):
        """
        Returns a list of project files (files in the project including their download links)
        :param project: RemoteProject
        :return: [ProjectFile]: files in the specified project
        """
        files = []
        result = self.data_service.get_project_files(project.id)
        user_list_json = result.json()
        for user_json in user_list_json['results']:
            files.append(ProjectFile(user_json))
        return files

    def get_file_url(self, file_id):
        """
        Given a file id return the RemoteFileUrl (file download url)
        :param file_id: str: DukeDS file uuid
        :return: RemoteFileUrl
        """
        return RemoteFileUrl(self.data_service.get_file_url(file_id).json())

    def close(self):
        self.data_service.close() class RemoteAuthRole(object):
    PROJECT_CONTEXT = "project"
    SYSTEM_CONTEXT = "system"
    """
    Permissions a user can be given on a project.
    """
    def __init__(self, json_data):
        """
        Set properties based on json_data.
        :param json_data: dict JSON data containing auth_role info
        """
        self.id = json_data['id']
        self.name = json_data['name']
        self.description = json_data['description']
        self.is_deprecated = json_data['is_deprecated']

    def __str__(self):
        return 'id:{} name:{} description:{}'.format(self.id, self.name, self.description) class ProjectNameOrId(object):
    """
    Contains either a project name or a project id(uuid).
    If it contains a project ID the project is assumed to already exist (only DukeDS makes up project uuid).
    If it contains a project name they project may or may not exist.
    """
    def __init__(self, value, is_name):
        self.value = value
        self.is_name = is_name

    def description(self):
        format_str = 'name {}'
        if not self.is_name:
            format_str = 'id {}'
        return format_str.format(self.value)

    def contained_in_dict(self, data):
        if self.is_name:
            return self.value == data.get('name')
        else:
            return self.value == data.get('id')

    def get_name_or_raise(self):
        if self.is_name:
            return self.value
        raise ValueError("Programming Error: Cannot return name. value is project id.")

    def get_id_or_raise(self):
        if not self.is_name:
            return self.value
        raise ValueError("Programming Error: Cannot return id. value is project name.")

    @staticmethod
    def create_from_name(name):
        return ProjectNameOrId(value=name, is_name=True)

    @staticmethod
    def create_from_project_id(project_id):
        return ProjectNameOrId(value=project_id, is_name=False)

    @staticmethod
    def create_from_remote_project(project):
        return ProjectNameOrId.create_from_project_id(project.id) class LocalProject(object):
    """
    Represents a list of folder/file trees on the filesystem.
    Has kind property to allow project tree traversal with ProjectWalker.
    """
    def __init__(self, followsymlinks, file_exclude_regex):
        """
        Creates a list of local file system content that can be sent to a remote project.
        :param followsymlinks: bool follow symbolic links when looking for content
        :param file_exclude_regex: str: regex that should be used to filter out files we do not want to upload
        """
        self.remote_id = ''
        self.kind = KindType.project_str
        self.children = []
        self.sent_to_remote = False
        self.followsymlinks = followsymlinks
        self.file_filter = FileFilter(file_exclude_regex)

    def add_path(self, path):
        """
        Add the path and any children files/folders to the list of content.
        :param path: str path to add
        """
        abspath = os.path.abspath(path)
        child = _build_project_tree(abspath, self.followsymlinks, self.file_filter)
        if child:
            self.children.append(child)

    def add_paths(self, path_list):
        """
        Add a list of paths to the list of content.
        :param path_list: [str] list of file system paths
        """
        for path in path_list:
            self.add_path(path)

    def update_remote_ids(self, remote_project):
        """
        Compare against remote_project saving off the matching uuids of of matching content.
        :param remote_project: RemoteProject project to compare against
        """
        if remote_project:
            self.remote_id = remote_project.id
            _update_remote_children(remote_project, self.children)

    def set_remote_id_after_send(self, remote_id):
        """
        Save remote_id after creating on remote side.
        :param remote_id: str uuid of the project
        """
        self.remote_id = remote_id
        self.sent_to_remote = True

    def count_local_items(self):
        return LocalItemsCounter(self)

    def count_items_to_send(self, upload_bytes_per_chunk):
        return ItemsToSendCounter(self, upload_bytes_per_chunk)

    def __str__(self):
        child_str = ', '.join([str(child) for child in self.children])
        return 'project: [{}]'.format(child_str) class ProjectUpload(object):
    """
    Allows uploading a local project to a remote duke-data-service.
    """
    def __init__(self, config, project_name_or_id, local_project, items_to_send_count, file_upload_post_processor=None):
        """
        Setup for uploading folders dictionary of paths to project_name using config.
        :param config: Config configuration for performing the upload(url, keys, etc)
        :param project_name_or_id: ProjectNameOrId: name or id of the project we will upload files to
        :param local_project: LocalProject: contains files and folders to upload
        :param items_to_send_count: ItemsToSendCounter
        :param file_upload_post_processor: object: has run(data_service, file_response) method to run after uploading
        """
        self.config = config
        self.remote_store = RemoteStore(config)
        self.project_name_or_id = project_name_or_id
        self.local_project = local_project
        self.items_to_send_count = items_to_send_count
        self.file_upload_post_processor = file_upload_post_processor

    @staticmethod
    def create_for_paths(config, remote_store, project_name_or_id, paths, follow_symlinks=False,
                         file_upload_post_processor=None):
        local_project = LocalProject(followsymlinks=follow_symlinks, file_exclude_regex=config.file_exclude_regex)
        local_project.add_paths(paths)
        remote_project = remote_store.fetch_remote_project(project_name_or_id)
        local_project.update_remote_ids(remote_project)
        items_to_send_count = local_project.count_items_to_send(config.upload_bytes_per_chunk)
        return ProjectUpload(config, project_name_or_id, local_project, items_to_send_count,
                             file_upload_post_processor=file_upload_post_processor)

    def run(self):
        """
        Upload different items within local_project to remote store showing a progress bar.
        """
        progress_printer = ProgressPrinter(self.items_to_send_count.total_items(), msg_verb='sending')
        upload_settings = UploadSettings(self.config, self.remote_store.data_service, progress_printer,
                                         self.project_name_or_id, self.file_upload_post_processor)
        project_uploader = ProjectUploader(upload_settings)
        project_uploader.run(self.local_project)
        progress_printer.finished()

    def get_upload_report(self):
        """
        Generate and print a report onto stdout.
        """
        project = self.remote_store.fetch_remote_project(self.project_name_or_id,
                                                         must_exist=True,
                                                         include_children=False)
        report = UploadReport(project.name)
        report.walk_project(self.local_project)
        return report

    def get_url_msg(self):
        """
        Print url to view the project via dds portal.
        """
        msg = 'URL to view project'
        project_id = self.local_project.remote_id
        url = '{}: {}'.format(msg, self.remote_store.data_service.portal_url(project_id))
        return url

    def cleanup(self):
        self.remote_store.close() class ProjectUploadDryRun(object):
    """
    Recursively visits children of the project passed to run.
    Builds a list of the names of folders/files that need to be uploaded.
    """
    def __init__(self, local_project):
        self.upload_items = []
        self._run(local_project)

    def add_upload_item(self, name):
        self.upload_items.append(name)

    def _run(self, local_project):
        """
        Appends file/folder paths to upload_items based on the contents of this project that need to be uploaded.
        :param local_project: LocalProject: project we will build the list for
        """
        self._visit_recur(local_project)

    def _visit_recur(self, item):
        """
        Recursively visits children of item.
        :param item: object: project, folder or file we will add to upload_items if necessary.
        """
        if item.kind == KindType.file_str:
            hash_data = item.calculate_local_hash()
            if not item.hash_matches_remote(hash_data):
                self.add_upload_item(item.path)
        else:
            if item.kind == KindType.project_str:
                pass
            else:
                if not item.remote_id:
                    self.add_upload_item(item.path)
            for child in item.children:
                self._visit_recur(child)

    def get_report(self):
        """
        Returns text displaying the items that need to be uploaded or a message saying there are no files/folders
        to upload.
        :return: str: report text
        """
        if not self.upload_items:
            return "\n\nNo changes found. Nothing needs to be uploaded.\n\n"
        else:
            result = "\n\nFiles/Folders that need to be uploaded:\n"
            for item in self.upload_items:
                result += "{}\n".format(item)
            result += "\n"
            return result class UploadDetails(object):
class ProjectChecker(object):
    def __init__(self, dds_file, remote_path):
    def inconsistent(self):
    def had_error(self):
    def is_bad(self):
    def name(self):
    def status_str(self):
    def file_id(self):
    def message(self):
    def __init__(self, config, project):
    def files_are_ok(self):
    def _try_fetch_project_files(self):
    def get_bad_uploads(self):
    def get_bad_uploads_table_data(self):
    def delete_bad_uploads(self):
    def print_bad_uploads_table(self):
    def wait_for_consistency(self, wait_sec=5): class CommandParser(object):
    """
    Root command line parser. Supports the following commands: upload and add_user.
    You must register external functions to called for the various commands.
    Commands must be registered to appear in help.
    """
    def __init__(self, version_str):
        self.parser = argparse.ArgumentParser(description=DESCRIPTION_STR.format(version_str))
        _skip_config_file_permission_check(self.parser)
        self.subparsers = self.parser.add_subparsers()
        self.upload_func = None
        self.add_user_func = None
        self.download_func = None

    def register_upload_command(self, upload_func):
        """
        Add the upload command to the parser and call upload_func(project_name, folders, follow_symlinks) when chosen.
        :param upload_func: func Called when this option is chosen: upload_func(project_name, folders, follow_symlinks).
        """
        description = "Uploads local files and folders to a remote store. After files are uploaded this command waits for all files to be valid/downloadable."
        upload_parser = self.subparsers.add_parser('upload', description=description)
        _add_dry_run(upload_parser, help_text="Instead of uploading displays a list of folders/files that "
                                              "need to be uploaded.")
        add_project_name_or_id_arg(upload_parser, help_text_suffix="upload files/folders to.")
        _add_folders_positional_arg(upload_parser)
        _add_follow_symlinks_arg(upload_parser)
        upload_parser.add_argument(
            "--no-check",
            help="Skip checking/waiting for uploaded files to be in the valid/downloadable state.",
            action='store_false',
            dest='check',
            default=True)
        upload_parser.set_defaults(func=upload_func)

    def register_add_user_command(self, add_user_func):
        """
        Add the add-user command to the parser and call add_user_func(project_name, user_full_name, auth_role)
        when chosen.
        :param add_user_func: func Called when this option is chosen: upload_func(project_name, user_full_name, auth_role).
        """
        description = "Gives user permission to access a remote project."
        add_user_parser = self.subparsers.add_parser('add-user', description=description)
        add_project_name_or_id_arg(add_user_parser, help_text_suffix="add a user to")
        user_or_email = add_user_parser.add_mutually_exclusive_group(required=True)
        add_user_arg(user_or_email)
        add_email_arg(user_or_email)
        _add_auth_role_arg(add_user_parser, default_permissions='project_admin')
        add_user_parser.set_defaults(func=add_user_func)

    def register_remove_user_command(self, remove_user_func):
        """
        Add the remove-user command to the parser and call remove_user_func(project_name, user_full_name) when chosen.
        :param remove_user_func: func Called when this option is chosen: remove_user_func(project_name, user_full_name).
        """
        description = "Removes user permission to access a remote project."
        remove_user_parser = self.subparsers.add_parser('remove-user', description=description)
        add_project_name_or_id_arg(remove_user_parser, help_text_suffix="remove a user from")
        user_or_email = remove_user_parser.add_mutually_exclusive_group(required=True)
        add_user_arg(user_or_email)
        add_email_arg(user_or_email)
        remove_user_parser.set_defaults(func=remove_user_func)

    def register_download_command(self, download_func):
        """
        Add 'download' command for downloading a project to a directory.
        For non empty directories it will download remote files replacing local files.
        :param download_func: function to run when user choses this option
        """
        description = "Download the contents of a remote remote project to a local folder."
        download_parser = self.subparsers.add_parser('download', description=description)
        add_project_name_or_id_arg(download_parser, help_text_suffix="download")
        _add_folder_positional_arg(download_parser)
        include_or_exclude = download_parser.add_mutually_exclusive_group(required=False)
        _add_include_arg(include_or_exclude)
        _add_exclude_arg(include_or_exclude)
        download_parser.set_defaults(func=download_func)

    def register_share_command(self, share_func):
        """
        Add 'share' command for adding view only project permissions and sending email via another service.
        :param share_func: function to run when user choses this option
        """
        description = "Share a project with another user with specified  permissions. " \
                      "Sends the other user an email message via D4S2 service. " \
                      "If not specified this command gives user download permissions."
        share_parser = self.subparsers.add_parser('share', description=description)
        add_project_name_or_id_arg(share_parser)
        user_or_email = share_parser.add_mutually_exclusive_group(required=True)
        add_user_arg(user_or_email)
        add_email_arg(user_or_email)
        _add_auth_role_arg(share_parser, default_permissions='file_downloader')
        _add_resend_arg(share_parser, "Resend share")
        _add_message_file(share_parser, "Filename containing a message to be sent with the share. "
                                        "Pass - to read from stdin.")
        share_parser.set_defaults(func=share_func)

    def register_deliver_command(self, deliver_func):
        """
        Add 'deliver' command for transferring a project to another user.,
        :param deliver_func: function to run when user choses this option
        """
        description = "Initiate delivery of a project to another user. Removes other user's current permissions. " \
                      "Send message to D4S2 service to send email and allow access to the project once user " \
                      "acknowledges receiving the data."
        deliver_parser = self.subparsers.add_parser('deliver', description=description)
        add_project_name_or_id_arg(deliver_parser)
        user_or_email = deliver_parser.add_mutually_exclusive_group(required=True)
        add_user_arg(user_or_email)
        add_email_arg(user_or_email)
        add_share_usernames_arg(deliver_parser)
        add_share_emails_arg(deliver_parser)
        _add_copy_project_arg(deliver_parser)
        _add_resend_arg(deliver_parser, "Resend delivery")
        include_or_exclude = deliver_parser.add_mutually_exclusive_group(required=False)
        _add_include_arg(include_or_exclude)
        _add_exclude_arg(include_or_exclude)
        _add_message_file(deliver_parser, "Filename containing a message to be sent with the delivery. "
                                          "Pass - to read from stdin.")
        deliver_parser.set_defaults(func=deliver_func)

    def register_list_command(self, list_func):
        """
        Add 'list' command to get a list of projects or details about one project.
        :param list_func: function: run when user choses this option.
        """
        description = "Show a list of project names or folders/files of a single project."
        list_parser = self.subparsers.add_parser('list', description=description)
        project_name_or_auth_role = list_parser.add_mutually_exclusive_group(required=False)
        _add_project_filter_auth_role_arg(project_name_or_auth_role)
        add_project_name_or_id_arg(project_name_or_auth_role, required=False,
                                   help_text_suffix="show details for")
        _add_long_format_option(list_parser, 'Display long format.')
        list_parser.set_defaults(func=list_func)

    def register_delete_command(self, delete_func):
        """
        Add 'delete' command delete a project or a file/folder within a project from the remote store.
        :param delete_func: function: run when user choses this option.
        """
        description = "Permanently delete a project or file/folder."
        delete_parser = self.subparsers.add_parser('delete', description=description)
        add_project_name_or_id_arg(delete_parser, help_text_suffix="delete")
        delete_parser.add_argument('--path',
                                   metavar='RemotePath',
                                   type=to_unicode,
                                   dest='remote_path',
                                   help="remote path specifying the file/folder to be deleted instead of the entire project")
        _add_force_arg(delete_parser, "Do not prompt before deleting.")
        delete_parser.set_defaults(func=delete_func)

    def register_list_auth_roles_command(self, list_auth_roles_func):
        """
        Add 'list_auth_roles' command to list project authorization roles that can be used with add_user.
        :param list_auth_roles_func: function: run when user choses this option.
        """
        description = "List authorization roles for use with add_user command."
        list_auth_roles_parser = self.subparsers.add_parser('list-auth-roles', description=description)
        list_auth_roles_parser.set_defaults(func=list_auth_roles_func)

    def register_move_command(self, move_func):
        """
        Add 'move' command to move a file/folder within a remote project.
        :param move_func: function: run when user choses this option.
        """
        description = "Moves and/or renames a file or folder within a project. \n" \
                      "When Target is a directory Source will be moved into it that directory. " \
                      "Otherwise Source will be renamed to the filename of Target and moved into the parent" \
                      " directory of Target. Target and Source are remote paths that must start with a '/'."
        parser = self.subparsers.add_parser('move', description=description)
        add_project_name_or_id_arg(parser, help_text_suffix="containing a file/folder to move")
        parser.add_argument("source_remote_path",
                            metavar='Source',
                            type=to_unicode,
                            help='remote path specifying the file/folder to be moved')
        parser.add_argument("target_remote_path",
                            metavar='Target',
                            type=to_unicode,
                            help='remote path specifying where to move the file/folder to')
        parser.set_defaults(func=move_func)

    def register_info_command(self, info_func):
        """
        Add 'info' command to get details of a remote project.
        :param info_func: function: run when user choses this option.
        """
        description = "Print information about a project."
        parser = self.subparsers.add_parser('info', description=description)
        add_project_name_or_id_arg(parser, help_text_suffix="to show the information about")
        parser.set_defaults(func=info_func)

    def register_check_command(self, check_func):
        """
        Add 'check' command to verify a project is consistent.
        :param check_func: function: run when user choses this option.
        """
        description = "Check that a project is in a consistent state."
        check_parser = self.subparsers.add_parser('check', description=description)
        add_project_name_or_id_arg(check_parser, help_text_suffix="check")
        check_parser.add_argument('--wait',
                                  help="Wait for project to become consistent.",
                                  action='store_true')
        check_parser.set_defaults(func=check_func)

    def run_command(self, args):
        """
        Parse command line arguments and run function registered for the appropriate command.
        :param args: [str] command line arguments
        """
        parsed_args = self.parser.parse_args(args)
        if hasattr(parsed_args, 'func'):
            parsed_args.func(parsed_args)
        else:
            self.parser.print_help() def format_destination_path(path):
    """
    Formats command line destination path.
    :param path: str path to check
    :return: str path
    """
    path = to_unicode(path)
    return _path_has_ok_chars(path) def replace_invalid_path_chars(path):
    """
    Converts bad path characters to '_'.
    :param path: str path to fix
    :return: str fixed path
    """
    for bad_char in INVALID_PATH_CHARS:
        path = path.replace(bad_char, '_')
    return path class ProjectDetailsList(object):
    """
    Walks a project and saves the project name and filenames to [str] filenames property.
    """
    def __init__(self, long_format):
        self.long_format = long_format
        self.details = []
        self.id_to_path = {}

    def walk_project(self, project):
        """
        Walks a project and saves the project name and filenames to [str] filenames property.
        :param project: LocalProject project we will read details from.
        """
        # This method will call visit_project, visit_folder, and visit_file below as it walks the project tree.
        ProjectWalker.walk_project(project, self)

    def visit_project(self, item):
        if self.long_format:
            self.details.append("{} - Project {} Contents:".format(item.id, item.name))
        else:
            self.details.append("Project {} Contents:".format(item.name))

    def visit_folder(self, item, parent):
        name = self.get_name(item, parent)
        self.id_to_path[item.id] = name
        if self.long_format:
            self.details.append('{}\t{}'.format(item.id, name))
        else:
            self.details.append(name)

    def visit_file(self, item, parent):
        name = self.get_name(item, parent)
        if self.long_format:
            self.details.append('{}\t{}\t({}:{})'.format(item.id, name, item.hash_alg, item.file_hash))
        else:
            self.details.append(name)

    def get_name(self, item, parent):
        if parent:
            if parent.kind == KindType.project_str:
                return '{}{}'.format(REMOTE_PATH_SEP, item.name)
            parent_name = self.id_to_path.get(parent.id)
            if parent_name:
                return "{}{}{}".format(parent_name, REMOTE_PATH_SEP, item.name)
        return item.name def verify_terminal_encoding(encoding):
    """
    Raises ValueError with error message when terminal encoding is not Unicode(contains UTF ignoring case).
    :param encoding: str: encoding we want to check
    """
    if encoding and not ("UTF" in encoding.upper()):
        raise DDSUserException(TERMINAL_ENCODING_NOT_UTF_ERROR) def boolean_input_prompt(message):
    if sys.version_info >= (3, 0, 0):
        result = input(message)
    else:
        result = input(message)
    result = result.upper()
    return result == "Y" or result == "YES" or result == "T" or result == "TRUE" class PathFilter(object):
    """
    Represents a list of path filters that allows a path to be checked via include_path method.
    By default all paths return True. Specify the list to include or exclude by calling set_include_paths or
    set_exclude_paths. Returns which paths have been seen via get_unused_paths method.
    """
    def __init__(self, include_paths, exclude_paths):
        """
        Creates a path filter based on either include_paths or exclude_paths if both are filled in raises error.
        :param include_paths: [str]: list of paths that should be included
        :param exclude_paths: [str]: list of paths that should be excluded
        """
        if include_paths and exclude_paths:
            raise ValueError("Programming Error: Should not specify both include_path and exclude_paths")

        path_filter = IncludeAll()
        if include_paths:
            path_filter = IncludeFilter(include_paths)
        elif exclude_paths:
            path_filter = ExcludeFilter(exclude_paths)

        self.filter = path_filter
        self.seen_paths = set()

    def include_path(self, path):
        """
        Should this path be included based on the include_paths or exclude_paths.
        Keeps track of paths seen to allow finding unused filters.
        :param path: str: remote path to be filtered
        :return: bool: True if we should include the path
        """
        self.seen_paths.add(path)
        return self.filter.include(path)

    def reset_seen_paths(self):
        """
        Clear list of paths seen via include_path method.
        """
        self.seen_paths = set()

    def get_unused_paths(self):
        """
        Returns which include_paths or exclude_paths that were not used via include_path method.
        :return: [str] list of filtering paths that were not used.
        """
        return [path for path in self.filter.paths if path not in self.seen_paths] def check_version():
    """
    Check version and raises VersionException if the installed DukeDSClient is out of date
    or unable to access pypi.
    """
    pypi_version = get_pypi_version()
    internal_version = get_internal_version()

    if pypi_version > internal_version:
        raise VersionException(UPDATE_VERSION_MESSAGE) class VersionException(DDSUserException):
    def __init__(self, message):
        super(VersionException, self).__init__(message) def get_internal_version_str():
    """
    Returns internal version string.
    """
    return pkg_resources.get_distribution(APP_NAME).version def create_config(allow_insecure_config_file=False):
    """
    Create config based on /etc/ddsclient.conf and ~/.ddsclient.conf($DDSCLIENT_CONF)
    :param allow_insecure_config_file: bool: when true we will not check ~/.ddsclient permissions.
    :return: Config with the configuration to use for DDSClient.
    """
    config = Config()
    config.add_properties(GLOBAL_CONFIG_FILENAME)
    user_config_filename = get_user_config_filename()
    if user_config_filename == LOCAL_CONFIG_FILENAME and not allow_insecure_config_file:
        verify_file_private(user_config_filename)
    config.add_properties(user_config_filename)
    return config class ProjectFileDownloader(object):
    def __init__(self, config, dest_directory, project, path_filter):
        self.config = config
        self.dest_directory = dest_directory
        self.project = project
        self.dds_connection = project.dds_connection
        self.num_workers = config.download_workers
        self.path_filter = path_filter
        self.async_download_results = []
        self.message_queue = multiprocessing.Manager().Queue()
        self.files_downloaded = 0
        self.files_to_download = None
        self.file_download_statuses = {}
        self.download_status_list = []
        self.spinner_chars = "|/-\\"
        self.start_time = None

    def run(self):
        self.start_time = time.time()
        self._download_files()
        self._show_downloaded_files_status()

    def _download_files(self):
        pool = multiprocessing.Pool(self.num_workers)
        try:
            for project_file in self._get_project_files():
                self._download_file(pool, project_file)
                while self._work_queue_is_full():
                    self._wait_for_and_retry_failed_downloads(pool)
            while self._work_queue_is_not_empty():
                self._wait_for_and_retry_failed_downloads(pool)
        finally:
            pool.close()

    def _show_downloaded_files_status(self):
        print("\nVerifying contents of {} downloaded files using file hashes.".format(self.files_to_download))
        all_good = True
        files_with_mismatched_hashes = 0
        for download_status in self.download_status_list:
            if not download_status.has_a_valid_hash():
                all_good = False
            if download_status.status == FileHashStatus.STATUS_WARNING:
                files_with_mismatched_hashes += 1
            print(download_status.get_status_line())
        if all_good:
            print("All downloaded files have been verified successfully.")
            if files_with_mismatched_hashes:
                print(MISMATCHED_FILE_HASH_WARNING.format(files_with_mismatched_hashes))
        else:
            raise ValueError("ERROR: Downloaded file(s) do not match the expected hashes.")

    def _get_project_files(self):
        project_files_generator = self.project.get_project_files_generator(self.config.page_size)
        if self.path_filter:
            # fetch all files so we can determine an accurate filtered count
            project_files = self._filter_project_files(project_files_generator)
            self._print_path_filter_warnings()
            self.files_to_download = len(project_files)
            self.show_progress_bar()
            for project_file in project_files:
                yield project_file
        else:
            for project_file, headers in project_files_generator:
                if self.files_to_download is None:
                    self.files_to_download = int(headers.get(DDS_TOTAL_HEADER))
                    self.show_progress_bar()
                yield project_file

    def _filter_project_files(self, project_files_generator):
        project_files = []
        for project_file, headers in project_files_generator:
            if self.path_filter.include_path(project_file.path):
                project_files.append(project_file)
        return project_files

    def _print_path_filter_warnings(self):
        if self.path_filter:
            unused_paths = self.path_filter.get_unused_paths()
            if unused_paths:
                print('WARNING: Path(s) not found: {}.'.format(','.join(unused_paths)))

    def _download_file(self, pool, project_file):
        output_path = project_file.get_local_path(self.dest_directory)
        output_path_parent = os.path.dirname(output_path)
        if not os.path.exists(output_path_parent):
            os.makedirs(output_path_parent)
        file_download_state = FileDownloadState(project_file, output_path, self.config)
        self._async_download_file(pool, file_download_state)

    def _async_download_file(self, pool, file_download_state):
        async_result = pool.apply_async(download_file, (file_download_state, self.message_queue))
        self.async_download_results.append(async_result)

    def _work_queue_is_full(self):
        return len(self.async_download_results) >= self.num_workers

    def _work_queue_is_not_empty(self):
        return len(self.async_download_results) > 0

    def _wait_for_and_retry_failed_downloads(self, pool):
        download_results = self._pop_ready_download_results()
        if download_results:
            self._process_download_results(pool, download_results)
        else:
            self._try_process_message_queue()
            time.sleep(0)  # Pause to give up CPU since no results are ready

    def _try_process_message_queue(self):
        try:
            file_id, bytes_downloaded, file_size, file_state = self.message_queue.get_nowait()
            # This might be out of date for a little bit
            self.file_download_statuses[file_id] = (bytes_downloaded, file_size, file_state)
            self.show_progress_bar()
        except queue.Empty:
            pass

    def show_progress_bar(self):
        files_downloaded, total_bytes_downloaded = self.get_download_progress()
        current_time = time.time()
        bytes_progress = '{} {}'.format(
            humanize_bytes(total_bytes_downloaded),
            self.make_download_speed(current_time, total_bytes_downloaded))
        sys.stdout.write("\r{} downloaded {} ({} of {} files complete)".format(
            self.make_spinner_char(current_time),
            bytes_progress.ljust(22),
            files_downloaded,
            self.files_to_download
        ))
        sys.stdout.flush()

    def make_spinner_char(self, current_time):
        half_seconds = int(current_time)
        return self.spinner_chars[half_seconds % 4]

    def make_download_speed(self, current_time, total_bytes_downloaded):
        return transfer_speed_str(
            current_time=current_time,
            start_time=self.start_time,
            transferred_bytes=total_bytes_downloaded
        )

    def get_download_progress(self):
        files_downloaded = 0
        total_bytes_downloaded = 0
        for file_id, download_info in self.file_download_statuses.items():
            bytes_downloaded, file_size, file_state = download_info
            # do not include files that were already downloaded in bytes downloaded
            if file_state != FileDownloadState.ALREADY_COMPLETE:
                total_bytes_downloaded += bytes_downloaded
            if bytes_downloaded == file_size:
                files_downloaded += 1
        return files_downloaded, total_bytes_downloaded

    def _pop_ready_download_results(self):
        ready_results = []
        for async_result in self._get_ready_async_results():
            result = async_result.get()
            # retrieve the value from the async result
            ready_results.append(result)
            # remove the async result from the list to watch
            self.async_download_results.remove(async_result)
        return ready_results

    def _get_ready_async_results(self):
        ready_results = []
        for async_result in self.async_download_results:
            if async_result.ready():
                ready_results.append(async_result)
        return ready_results

    def _process_download_results(self, pool, download_results):
        for file_download_state in download_results:
            if file_download_state.is_ok_state():
                file_id = file_download_state.file_id
                size = file_download_state.size
                status = file_download_state.status
                self.file_download_statuses[file_id] = (size, size, file_download_state.state)
                self.download_status_list.append(status)
            elif file_download_state.retries:
                file_download_state.retries -= 1
                # Refresh url in file_download_state
                file_download = self.dds_connection.get_file_download(file_download_state.file_id)
                file_download_state.url = file_download.host + file_download.url
                # Re-run download process
                self._async_download_file(pool, file_download_state)
            else:
                raise ValueError("Error downloading {}\n{}".format(
                    file_download_state.output_path,
                    file_download_state.msg
                ))
        self.show_progress_bar() class DDSUserException(Exception):
    """
    Exception with an error message to be displayed to the user on the command line.
    """
    pass class Client(object):
    """
    Client that connects to the DDSConnection base on ~/.ddsclient configuration.
    This configuration can be customized by passing in a ddsc.config.Config object
    """
    def __init__(self, config=None, create_data_service_auth=DataServiceAuth):
        """
        :param config: ddsc.config.Config: settings used to connect to DDSConnection
        :param create_data_service_auth: func(config): function used to create a DataServiceAuth object
        """
        if not config:
            config = create_config()
        self.dds_connection = DDSConnection(config, create_data_service_auth)

    def get_projects(self):
        """
        Get list of all projects user has access to.
        :return: [Project]: list of projects
        """
        return self.dds_connection.get_projects()

    def get_project_by_id(self, project_id):
        """
        Retrieve a single project.
        :param project_id:
        :return: Project
        """
        return self.dds_connection.get_project_by_id(project_id)

    def get_project_by_name(self, project_name):
        """
        Retrieve a single project.
        :param project_name: name of the project to get.
        :return: Project
        """
        projects = [project for project in self.get_projects() if project.name == project_name]
        if not projects:
            raise ItemNotFound("No project named {} found.".format(project_name))
        if len(projects) != 1:
            raise DDSUserException("Multiple projects found with name {}.".format(project_name))
        return projects[0]

    def create_project(self, name, description):
        """
        Create a new project with the specified name and description
        :param name: str: name of the project
        :param description: str: description of the project
        :return: Project
        """
        return self.dds_connection.create_project(name, description)

    def get_folder_by_id(self, folder_id):
        """
        Return details about a folder with the specified uuid
        :param folder_id: str: uuid of the folder to fetch
        :return: Folder
        """
        return self.dds_connection.get_folder_by_id(folder_id)

    def get_file_by_id(self, file_id):
        """
        Return details about a file with the specified uuid
        :param file_id: str: uuid of the file to fetch
        :return: File
        """
        return self.dds_connection.get_file_by_id(file_id)

    def close(self):
        self.dds_connection.close() class AzureListCommand(BaseAzureCommand):
    def run(self, args):
        show_project = self.get_project(args)
        if show_project:
            print("Project {} Contents:".format(show_project.name))
            for azure_file in self.azure.get_files(show_project):
                if args.long_format:
                    print("{} (md5:{})".format(azure_file.name, azure_file.md5))
                else:
                    print(azure_file.name)
        else:
            for project in self.azure.get_projects():
                if args.auth_role:
                    if project.auth_role == args.auth_role:
                        print(project.name)
                else:
                    print(project.name) class AzureUploadCommand(BaseAzureCommand):
    def run(self, args):
        project = self.get_project(args)
        self.azure.upload_files(project=project, paths=args.folders,
                                follow_symlinks=args.follow_symlinks, dry_run=args.dry_run) class AzureAddUserCommand(BaseAzureCommand):
    def run(self, args):
        project = self.get_project(args)
        netid = self.get_netid(args)
        self.azure.add_user(project=project, netid=netid, auth_role=args.auth_role) class AzureRemoveUserCommand(BaseAzureCommand):
    def run(self, args):
        project = self.get_project(args)
        netid = self.get_netid(args)
        self.azure.remove_user(project=project, netid=netid) class AzureDownloadCommand(BaseAzureCommand):
    def run(self, args):
        project = self.get_project(args)
        destination = args.folder
        if not destination:
            destination = replace_invalid_path_chars(project.name.replace(' ', '_'))
        self.azure.download_files(
            project=project,
            include_paths=args.include_paths,
            exclude_paths=args.exclude_paths,
            destination=destination) class AzureShareCommand(BaseAzureCommand):
    def run(self, args):
        project = self.get_project(args)
        netid = self.get_netid(args)
        self.azure.share(project=project, netid=netid, auth_role=args.auth_role) class AzureDeliverCommand(BaseAzureCommand):
    def run(self, args):
        project = self.get_project(args)
        netid = self.get_netid(args)
        self.azure.deliver(project=project, netid=netid, copy_project=args.copy_project, resend=args.resend,
                           msg_file=args.msg_file, share_usernames=args.share_usernames) class AzureDeleteCommand(BaseAzureCommand):
    def run(self, args):
        project = self.get_project(args)
        delete_target_name = project.name
        if args.remote_path:
            delete_target_name = "{} path {}".format(project.name, args.remote_path)
        if not args.force:
            delete_prompt = "Are you sure you wish to delete {} (y/n)?".format(delete_target_name)
            if not boolean_input_prompt(delete_prompt):
                return
        self.azure.delete(project=project, remote_path=args.remote_path) class AzureListAuthRolesCommand(BaseAzureCommand):
    def run(self, args):
        for auth_role in self.azure.get_auth_roles():
            print(auth_role.id, "-", auth_role.description) class AzureMoveCommand(BaseAzureCommand):
    def run(self, args):
        project = self.get_project(args)
        self.azure.move(project=project,
                        source_remote_path=args.source_remote_path,
                        target_remote_path=args.target_remote_path) class AzureInfoCommand(BaseAzureCommand):
    def run(self, args):
        project = self.get_project(args)
        print()
        print("Name:", project.name)
        print("URL:", project.get_url())
        print("Size:", project.get_size_str())
        print() class AzureCheckCommand(BaseAzureCommand):
    def run(self, args):
        raise DDSUserException(CHECK_COMMAND_NOT_SUPPORTED_MSG)
import sys
import datetime
import time
from ddsc.core.d4s2 import D4S2Project, D4S2Error
from ddsc.core.remotestore import RemoteStore, RemoteAuthRole, ProjectNameOrId
from ddsc.core.localstore import LocalProject
from ddsc.core.upload import ProjectUpload
from ddsc.core.projectuploader import ProjectUploadDryRun
from ddsc.core.consistency import ProjectChecker, DSHashMismatchError
from ddsc.cmdparser import CommandParser, format_destination_path, replace_invalid_path_chars
from ddsc.core.util import ProjectDetailsList, verify_terminal_encoding, boolean_input_prompt
from ddsc.core.pathfilter import PathFilter
from ddsc.versioncheck import check_version, VersionException, get_internal_version_str
from ddsc.config import create_config
from ddsc.core.download import ProjectFileDownloader
from ddsc.exceptions import DDSUserException
from ddsc.sdk.client import Client
from ddsc.azure import AzureListCommand, AzureUploadCommand, AzureAddUserCommand, AzureRemoveUserCommand, \
    AzureDownloadCommand, AzureShareCommand, AzureDeliverCommand, AzureDeleteCommand, AzureListAuthRolesCommand, \
    AzureMoveCommand, AzureInfoCommand, AzureCheckCommand
                to_users.append(user)
        return to_users

    def cleanup(self):
        self.remote_store.close()


class ClientCommand(object):
    def __init__(self, config):
        self.client = Client(config)
        self.config = config

    def get_project_by_name_or_id(self, args):
        if args.project_name:
            return self.client.get_project_by_name(args.project_name)
        else:
            return self.client.get_project_by_id(args.project_id)

    def cleanup(self):
        self.client.close()


class UploadCommand(BaseCommand):
    """
    Uploads a folder to a remote project.
    """
    def __init__(self, config):
        """
        Pass in the config containing remote_store/url so we can access the remote data.
        :param config: Config global configuration for use with this command.
        """
        super(UploadCommand, self).__init__(config)

    def run(self, args):
        """
        Upload contents of folders to a project with project_name on remote store.
        If follow_symlinks we will traverse symlinked directories.
        If content is already on remote site it will not be sent.
        :param args: Namespace arguments parsed from the command line.
        """
        project_name_or_id = self.create_project_name_or_id_from_args(args)
        folders = args.folders                  # list of local files/folders to upload into the project
        follow_symlinks = args.follow_symlinks  # should we follow symlinks when traversing folders
        dry_run = args.dry_run                  # do not upload anything, instead print out what you would upload
        check_file_consistency = args.check     # should we check download URLs after uploading

        # Find files and folders to upload
        local_project = LocalProject(followsymlinks=follow_symlinks, file_exclude_regex=self.config.file_exclude_regex)
        local_project.add_paths(folders)
        local_items_count = local_project.count_local_items()
        print(local_items_count.to_str(prefix="Checking"))

        # Fetch remote project (if there is one) and update local_project with details from remote project
        remote_project = self.remote_store.fetch_remote_project(project_name_or_id)
        local_project.update_remote_ids(remote_project)
        items_to_send_count = local_project.count_items_to_send(self.config.upload_bytes_per_chunk)
        print(items_to_send_count.to_str(local_items_count=local_items_count, prefix="Synchronizing"))

        if dry_run:
            # Check hashes to see what needs to be uploaded
            dry_run = ProjectUploadDryRun(local_project)
            print(dry_run.get_report())
        else:
            # Upload files and folders
            project_upload = ProjectUpload(self.config, project_name_or_id, local_project, items_to_send_count)
            project_upload.run()

            # Show user results of upload
            upload_report = project_upload.get_upload_report()
            print(upload_report.summary())
            print()
            if upload_report.sent_data:
                print('\n')
                print(upload_report.get_content())
                print('\n')
            print(project_upload.get_url_msg())
            project_upload.cleanup()

            # check for consistency unless user passes --no-check flag
            if check_file_consistency:
                self.wait_for_consistency(local_project.remote_id)

    def wait_for_consistency(self, project_id):
        client = Client(self.config)
        project = client.get_project_by_id(project_id)
        checker = ProjectChecker(self.config, project)
        try:
            checker.wait_for_consistency()
        except DSHashMismatchError:
            checker.print_bad_uploads_table()
            sys.exit(1)
        finally:
            client.close()


class DownloadCommand(ClientCommand):
    """
    Downloads the content from a remote project into a folder.
    """
    def __init__(self, config):
        """
        Pass in the config who can create a remote_store so we can access the remote data.
        :param config: Config global configuration for use with this command.
        """
        super(DownloadCommand, self).__init__(config)
        self.config = config

    def run(self, args):
        """
        Download a project based on passed in args.
        :param args: Namespace arguments parsed from the command line.
        """
        project = self.get_project_by_name_or_id(args)
        folder = args.folder                # path to a folder to download data into
        # Default to project name with spaces replaced with '_' if not specified
        if not folder:
            folder = replace_invalid_path_chars(project.name.replace(' ', '_'))
        path_filter = None
        if args.include_paths or args.exclude_paths:
            path_filter = PathFilter(args.include_paths, args.exclude_paths)

        destination_path = format_destination_path(folder)