scraper/code_gov/models.py
def from_gitlab(klass, repository, labor_hours=True):
        """
        Create CodeGovProject object from GitLab Repository
        """
        if not isinstance(repository, gitlab.v4.objects.Project):
            raise TypeError('Repository must be a gitlab Repository object')

        project = klass()

        logger.debug(
            'GitLab: repository_id=%d path_with_namespace=%s',
            repository.id,
            repository.path_with_namespace,
        )

        # -- REQUIRED FIELDS --

        project['name'] = repository.name
        project['repositoryURL'] = repository.http_url_to_repo
        project['description'] = repository.description

        # TODO: Update licenses from GitLab API
        project['permissions']['licenses'] = None

        web_url = repository.web_url
        public_server = web_url.startswith('https://gitlab.com')

        if repository.visibility in ('public') and public_server:
            project['permissions']['usageType'] = 'openSource'
        elif date_parse(repository.created_at) < POLICY_START_DATE:
            project['permissions']['usageType'] = 'exemptByPolicyDate'

        if labor_hours:
            project['laborHours'] = labor_hours_from_url(project['repositoryURL'])
        else:
            project['laborHours'] = 0

        project['tags'] = ['gitlab'] + repository.tag_list

        project['contact'] = {
            'email': '',
            'URL': web_url,
        }

        # -- OPTIONAL FIELDS --

        # project['version'] = ''

        project['organization'] = repository.namespace['name']

        # TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370
        project['status'] = 'Development'

        project['vcs'] = 'git'

        project['homepageURL'] = repository.web_url

        api_url = repository.manager.gitlab._url
        archive_suffix = '/projects/%s/repository/archive' % repository.get_id()
        project['downloadURL'] = api_url + archive_suffix

        # project['languages'] = [l for l, _ in repository.languages()]
        # project['partners'] = []
        # project['relatedCode'] = []
        # project['reusedCode'] = []

        project['date'] = {
            'created': date_parse(repository.created_at).date().isoformat(),
            'lastModified': date_parse(repository.last_activity_at).date().isoformat(),
            'metadataLastUpdated': '',
        }

        _prune_dict_null_str(project)

        return project
def from_gitlab(klass, repository, labor_hours=True):
        if not isinstance(repository, gitlab.v4.objects.Project):
            raise TypeError('Repository must be a gitlab Repository object')

        project = klass()

        logger.debug(
            'GitLab: repository_id=%d path_with_namespace=%s',
            repository.id,
            repository.path_with_namespace,
        )

        # -- REQUIRED FIELDS --

        project['name'] = repository.name
        project['repositoryURL'] = repository.http_url_to_repo
        project['description'] = repository.description

        # TODO: Update licenses from GitLab API
        project['permissions']['licenses'] = None

        web_url = repository.web_url
        public_server = web_url.startswith('https://gitlab.com')

        if repository.visibility in ('public') and public_server:
            project['permissions']['usageType'] = 'openSource'
        elif date_parse(repository.created_at) < POLICY_START_DATE:
            project['permissions']['usageType'] = 'exemptByPolicyDate'

        if labor_hours:
            project['laborHours'] = labor_hours_from_url(project['repositoryURL'])
        else:
            project['laborHours'] = 0

        project['tags'] = ['gitlab'] + repository.tag_list

        project['contact'] = {
            'email': '',
            'URL': web_url,
        }

        # -- OPTIONAL FIELDS --

        # project['version'] = ''

        project['organization'] = repository.namespace['name']

        # TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370
        project['status'] = 'Development'

        project['vcs'] = 'git'

        project['homepageURL'] = repository.web_url

        api_url = repository.manager.gitlab._url
        archive_suffix = '/projects/%s/repository/archive' % repository.get_id()
        project['downloadURL'] = api_url + archive_suffix

        # project['languages'] = [l for l, _ in repository.languages()]
        # project['partners'] = []
        # project['relatedCode'] = []
        # project['reusedCode'] = []

        project['date'] = {
            'created': date_parse(repository.created_at).date().isoformat(),
            'lastModified': date_parse(repository.last_activity_at).date().isoformat(),
            'metadataLastUpdated': '',
        }

        _prune_dict_null_str(project)

        return project
def from_gitlab ( klass , repository , labor_hours = True ) : if not isinstance ( repository , gitlab . v4 . objects . Project ) : raise TypeError ( 'Repository must be a gitlab Repository object' ) project = klass ( ) logger . debug ( 'GitLab: repository_id=%d path_with_namespace=%s' , repository . id , repository . path_with_namespace , ) # -- REQUIRED FIELDS -- project [ 'name' ] = repository . name project [ 'repositoryURL' ] = repository . http_url_to_repo project [ 'description' ] = repository . description # TODO: Update licenses from GitLab API project [ 'permissions' ] [ 'licenses' ] = None web_url = repository . web_url public_server = web_url . startswith ( 'https://gitlab.com' ) if repository . visibility in ( 'public' ) and public_server : project [ 'permissions' ] [ 'usageType' ] = 'openSource' elif date_parse ( repository . created_at ) < POLICY_START_DATE : project [ 'permissions' ] [ 'usageType' ] = 'exemptByPolicyDate' if labor_hours : project [ 'laborHours' ] = labor_hours_from_url ( project [ 'repositoryURL' ] ) else : project [ 'laborHours' ] = 0 project [ 'tags' ] = [ 'gitlab' ] + repository . tag_list project [ 'contact' ] = { 'email' : '' , 'URL' : web_url , } # -- OPTIONAL FIELDS -- # project['version'] = '' project [ 'organization' ] = repository . namespace [ 'name' ] # TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370 project [ 'status' ] = 'Development' project [ 'vcs' ] = 'git' project [ 'homepageURL' ] = repository . web_url api_url = repository . manager . gitlab . _url archive_suffix = '/projects/%s/repository/archive' % repository . get_id ( ) project [ 'downloadURL' ] = api_url + archive_suffix # project['languages'] = [l for l, _ in repository.languages()] # project['partners'] = [] # project['relatedCode'] = [] # project['reusedCode'] = [] project [ 'date' ] = { 'created' : date_parse ( repository . created_at ) . date ( ) . isoformat ( ) , 'lastModified' : date_parse ( repository . last_activity_at ) . date ( ) . isoformat ( ) , 'metadataLastUpdated' : '' , } _prune_dict_null_str ( project ) return project
Create CodeGovProject object from GitLab Repository
Create CodeGovProject object from GitLab Repository