main.py

from os import getenv
from os.path import exists
from functools import cache
import json
import streamlit as st
from dotenv import load_dotenv
from googleapiclient.discovery import build
from slugify import slugify
from transformers import pipeline
import uuid

from beautiful_soup.app import get_url_content

@cache
def google_search_api_request( query ):
    load_dotenv()
    api_key = getenv('GOOGLE_SEARCH_API_KEY')
    # cx = os.getenv('GOOGLE_SEARCH_ENGINE_ID')
    service = build(
        "customsearch",
        "v1",
        developerKey=api_key,
        cache_discovery=False
    )

    # Exclude PDFs from search results.
    query = query + ' -filetype:pdf'

    return service.cse().list(
        q=query,
        cx='05048cc2df6134a06',
        num=5,
        ).execute()

def search_results( query ):
    file_path = 'search-results/' + slugify( query ) + '.json'

    results = []
    if exists( file_path ):
        with open( file_path, 'r' ) as results_file:
            results = json.load( results_file )
    else:
        search_result = google_search_api_request( query )
        if (  int( search_result['searchInformation']['totalResults'] ) > 0 ):
            results = search_result['items']
            with open( file_path, 'w' ) as results_file:
                json.dump( results, results_file )

    if ( len( results ) == 0 ) :
        raise Exception('No results found.')
    
    return results

def main():
    st.title('Google Search')
    query = st.text_input('Search query')
    if query :
        with st.spinner('Loading search results...'):
            try:
                results = search_results( query )
            except Exception as exception:
                st.exception(exception)
                return
                

            for result in results:
                url_id = uuid.uuid5( uuid.NAMESPACE_URL, result['link'] ).hex
                st.write(result['link'])
                st.write(url_id)

                try:
                    content = get_url_content( result['link'] )
                except Exception as exception:
                    st.exception(exception)
                    continue

                file_path = 'summaries/' + url_id + '.json'
                if exists( file_path ):
                    with open( file_path, 'r' ) as file:
                        summary = json.load( file )
                else:
                    try:
                        summarizer = pipeline("summarization", model="sshleifer/distilbart-cnn-12-6")
                        summary = summarizer(content, max_length=130, min_length=30, do_sample=False, truncation=True)
                    except Exception as exception:
                        raise exception
                    with open( file_path, 'w' ) as file:
                        json.dump( summary, file )
        
                for sentence in summary:
                    st.write(sentence['summary_text'])

if __name__ == '__main__':
    main()