Academic Web Scraping

import requests
import time

class OpenAlexClient:
    BASE_URL = "https://api.openalex.org"

    def __init__(self, email):
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': f'ResearchBot/1.0 (mailto:{email})'
        })

    def search_works(self, query, filters=None, per_page=25, max_results=100):
        """Search for works with optional filters."""
        results = []
        page = 1

        while len(results) < max_results:
            params = {
                'search': query,
                'per_page': min(per_page, max_results - len(results)),
                'page': page,
            }
            if filters:
                params['filter'] = ','.join(f'{k}:{v}' for k, v in filters.items())

            resp = self.session.get(f'{self.BASE_URL}/works', params=params)
            resp.raise_for_status()
            data = resp.json()

            works = data.get('results', [])
            if not works:
                break

            results.extend(works)
            page += 1
            time.sleep(0.1)  # Polite rate limiting

        return results[:max_results]

    def get_work(self, openalex_id):
        """Get a single work by OpenAlex ID."""
        resp = self.session.get(f'{self.BASE_URL}/works/{openalex_id}')
        resp.raise_for_status()
        return resp.json()

# Usage
client = OpenAlexClient(email="researcher@university.edu")
papers = client.search_works(
    "transformer attention mechanism",
    filters={
        'publication_year': '2023-2024',
        'type': 'journal-article',
        'open_access.is_oa': 'true'
    },
    max_results=200
)

for paper in papers[:5]:
    print(f"- {paper['title']} ({paper['publication_year']})")
    print(f"  DOI: {paper['doi']}")
    print(f"  Citations: {paper['cited_by_count']}")

from Bio import Entrez

Entrez.email = "researcher@university.edu"
Entrez.api_key = os.environ.get("NCBI_API_KEY")  # optional

def search_pubmed(query, max_results=100):
    """Search PubMed and retrieve article details."""
    # Search
    handle = Entrez.esearch(db="pubmed", term=query,
                            retmax=max_results, sort="relevance")
    search_results = Entrez.read(handle)
    id_list = search_results["IdList"]

    if not id_list:
        return []

    # Fetch details
    handle = Entrez.efetch(db="pubmed", id=id_list,
                           rettype="xml", retmode="xml")
    records = Entrez.read(handle)

    articles = []
    for article in records['PubmedArticle']:
        medline = article['MedlineCitation']
        art_info = medline['Article']
        articles.append({
            'pmid': str(medline['PMID']),
            'title': art_info.get('ArticleTitle', ''),
            'abstract': art_info.get('Abstract', {}).get(
                'AbstractText', [''])[0] if 'Abstract' in art_info else '',
            'journal': art_info['Journal']['Title'],
            'year': art_info['Journal']['JournalIssue'].get(
                'PubDate', {}).get('Year', ''),
        })

    return articles

import requests
from bs4 import BeautifulSoup
import time

def scrape_conference_proceedings(url, delay=2.0):
    """Scrape paper titles and links from a conference page."""
    headers = {
        'User-Agent': 'ResearchBot/1.0 (Academic research; contact@university.edu)'
    }

    response = requests.get(url, headers=headers, timeout=30)
    response.raise_for_status()

    soup = BeautifulSoup(response.text, 'html.parser')

    papers = []
    for item in soup.select('.paper-item, .proceeding-entry'):
        title_el = item.select_one('.title, h3, h4')
        link_el = item.select_one('a[href]')
        authors_el = item.select_one('.authors, .author-list')

        if title_el:
            papers.append({
                'title': title_el.get_text(strip=True),
                'url': link_el['href'] if link_el else None,
                'authors': authors_el.get_text(strip=True) if authors_el else '',
            })

    time.sleep(delay)  # Respect the server
    return papers

from playwright.sync_api import sync_playwright

def scrape_dynamic_page(url):
    """Scrape a JavaScript-rendered page using Playwright."""
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page.goto(url, wait_until='networkidle')

        # Wait for content to load
        page.wait_for_selector('.results-container', timeout=10000)

        # Extract data
        items = page.query_selector_all('.result-item')
        results = []
        for item in items:
            title = item.query_selector('.title')
            results.append({
                'title': title.inner_text() if title else '',
            })

        browser.close()
        return results