<p>You can try with request and beautiful soup approach. No need to click more link.</p>
<pre><code>from requests import get
from bs4 import BeautifulSoup

# you can change the size to retrieve all the results at one shot.

url = 'https://arxiv.org/search/?query=healthcare&amp;searchtype=all&amp;abstracts=show&amp;order=-announced_date_first&amp;size=50&amp;start=0'
response = get(url,verify = False)
soup = BeautifulSoup(response.content, &quot;lxml&quot;)
#print(soup)
queryresults = soup.find_all(&quot;li&quot;, attrs={&quot;class&quot;: &quot;arxiv-result&quot;})

for result in queryresults:
    title = result.find(&quot;p&quot;,attrs={&quot;class&quot;: &quot;title is-5 mathjax&quot;})
    print(title.text)

#If you need full abstract content - try this (you do not need to click on more button
    for result in queryresults:
        abstractFullContent = result.find(&quot;span&quot;,attrs={&quot;class&quot;: &quot;abstract-full has-text-grey-dark mathjax&quot;})
        print(abstractFullContent.text)
</code></pre>
<p><strong>Output:</strong></p>
<pre><code> Interpretable Deep Learning for Automatic Diagnosis of 12-lead Electrocardiogram
            
  Leveraging Technology for Healthcare and Retaining Access to Personal Health Data to Enhance Personal Health and Well-being
  Towards new forms of particle sensing and manipulation and 3D imaging on a smartphone for healthcare applications
</code></pre>
Is there a way to optimize the for loop? Selenium is taking too long to scrape 38 pages
<p>I'm trying to scrape <a href="https://arxiv.org/search/?query=healthcare&amp;searchtype=allI" rel="nofollow noreferrer">https://arxiv.org/search/?query=healthcare&amp;searchtype=allI</a> through the Selenium and python. The for loop takes too long to execute. I tried to scrape with headless browsers and PhantomJS, but it doesnt scrape the abstract field (Need the abstract field expanded with the more button clicked)</p>
<pre class="lang-py prettyprint-override"><code>import pandas as pd
import selenium
import re
import time
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import Firefox

browser = Firefox()
url_healthcare = 'https://arxiv.org/search/?query=healthcare&amp;searchtype=all'
browser.get(url_healthcare)

dfs = []
for i in range(1, 39):
    articles = browser.find_elements_by_tag_name('li[class=&quot;arxiv-result&quot;]')

    for article in articles:
        title = article.find_element_by_tag_name('p[class=&quot;title is-5 mathjax&quot;]').text
        arxiv_id = article.find_element_by_tag_name('a').text.replace('arXiv:','')
        arxiv_link = article.find_elements_by_tag_name('a')[0].get_attribute('href') 
        pdf_link = article.find_elements_by_tag_name('a')[1].get_attribute('href')
        authors = article.find_element_by_tag_name('p[class=&quot;authors&quot;]').text.replace('Authors:','')

        try:
                link1 = browser.find_element_by_link_text('▽ More')
                link1.click()
        except:
                time.sleep(0.1)

        abstract = article.find_element_by_tag_name('p[class=&quot;abstract mathjax&quot;]').text
        date = article.find_element_by_tag_name('p[class=&quot;is-size-7&quot;]').text
        date = re.split(r&quot;Submitted|;&quot;,date)[1]
        tag = article.find_element_by_tag_name('div[class=&quot;tags is-inline-block&quot;]').text.replace('\n', ',')
        
        try:
            doi = article.find_element_by_tag_name('div[class=&quot;tags has-addons&quot;]').text
            doi = re.split(r'\s', doi)[1] 
        except NoSuchElementException:
            doi = 'None'

        all_combined = [title, arxiv_id, arxiv_link, pdf_link, authors, abstract, date, tag, doi]
        dfs.append(all_combined)

    print('Finished Extracting Page:', i)

    try:
        link2 = browser.find_element_by_class_name('pagination-next')
        link2.click()
    except:
        browser.close
        
    time.sleep(0.1)


</code></pre>