pub-tool.py

# build_pubmed_timeline_single_html.py
# Produces ONE self-contained HTML with:
# - Plotly interactive timeline (no graph title)
# - Yearly publication title tables INSIDE the timeline (annotations)
# - Horizontal tag bar (click to filter)
# - "Show all" / "Hide all" buttons
# - Fully offline HTML
# - Citation-only (gray dotted) bubbles connected by red dashed arrows

import requests
import plotly.graph_objects as go
import plotly.io as pio
import random
from datetime import datetime, timedelta
from collections import defaultdict, Counter
import re
import json
import xml.etree.ElementTree as ET
import textwrap
import os
import time

import argparse

# Load API key from .env
from dotenv import load_dotenv
load_dotenv()

API_KEY = os.getenv("API_KEY")
if not API_KEY:
    raise ValueError("API_KEY not found in environment. Please set it in .env or your system environment variables.")

# ---------------- CLI ARGUMENTS ----------------
parser = argparse.ArgumentParser(
    description="Build a PubMed timeline HTML visualization for a search term."
)
parser.add_argument(
    "term",
    metavar="TERM",
    type=str,
    help="PubMed search term (required, e.g. 'hydra vulgaris')"
)
parser.add_argument(
    "--retmax",
    type=int,
    default=30,
    help="Maximum number of PubMed results to fetch (default: 30)"
)

args = parser.parse_args()
TERM = args.term
RETMAX = args.retmax
OUTPUT_HTML = "timeline_pubmed_ranked.html"
# ------------------------------------------------

# Layout tuning
SCALE_FACTOR = 5
MARGIN = 0.2
MIN_BUBBLE_SIZE = 26
AVG_CHARS_PER_YEAR = 125
TABLE_Y_OFFSET = -80
BORDER_COLOR = "transparent" #rgba(0,0,0,0.2)
# ----------------------------------------

CACHE_FILE = "citations_cache.json"
SUMMARY_CACHE_FILE = "summaries_cache.json"
XML_CACHE_FILE = "xml_cache.json"

def load_xml_cache():
    """Load cached XML records per PMID."""
    if os.path.exists(XML_CACHE_FILE):
        try:
            with open(XML_CACHE_FILE, "r", encoding="utf-8") as f:
                cache = json.load(f)
            print(f"  Loaded {len(cache)} cached XML records from {XML_CACHE_FILE}")
            return cache
        except Exception as e:
            print(f" Failed to load XML cache: {e}")
    return {}

def save_xml_cache(cache):
    """Save cached XML records."""
    try:
        with open(XML_CACHE_FILE, "w", encoding="utf-8") as f:
            json.dump(cache, f, ensure_ascii=False, indent=2)
        print(f" Saved {len(cache)} cached XML records to {XML_CACHE_FILE}")
    except Exception as e:
        print(f" Failed to save XML cache: {e}")
def load_summary_cache():
    """Load cached PubMed summaries."""
    if os.path.exists(SUMMARY_CACHE_FILE):
        try:
            with open(SUMMARY_CACHE_FILE, "r", encoding="utf-8") as f:
                cache = json.load(f)
            print(f"  Loaded {len(cache)} cached summaries from {SUMMARY_CACHE_FILE}")
            return cache
        except Exception as e:
            print(f" Failed to load summary cache: {e}")
    return {}

def save_summary_cache(cache):
    """Save cached PubMed summaries."""
    try:
        with open(SUMMARY_CACHE_FILE, "w", encoding="utf-8") as f:
            json.dump(cache, f, ensure_ascii=False, indent=2)
        print(f" Saved {len(cache)} cached summaries to {SUMMARY_CACHE_FILE}")
    except Exception as e:
        print(f" Failed to save summary cache: {e}")

def load_cache():
    """Load cached citation results from disk."""
    if os.path.exists(CACHE_FILE):
        try:
            with open(CACHE_FILE, "r", encoding="utf-8") as f:
                cache = json.load(f)
            print(f"  Loaded {len(cache)} cached citation entries from {CACHE_FILE}")
            return cache
        except Exception as e:
            print(f" Failed to load cache: {e}")
    return {}

def save_cache(cache):
    """Save citation cache to disk."""
    try:
        with open(CACHE_FILE, "w", encoding="utf-8") as f:
            json.dump(cache, f, ensure_ascii=False, indent=2)
        print(f" Saved {len(cache)} cached citation entries to {CACHE_FILE}")
    except Exception as e:
        print(f" Failed to save cache: {e}")


def fetch_elink(pmids, linkname):
    """Generic PubMed ELink fetcher."""
    if not pmids:
        return []
    url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi"
    params = {
        "dbfrom": "pubmed",
        "linkname": linkname,
        "id": ",".join(pmids),
        "api_key": API_KEY,
        "retmode": "json",
    }
    r = requests.get(url, params=params, timeout=60)
    r.raise_for_status()
    data = r.json()

    links = []
    for ls in data.get("linksets", []):
        for db in ls.get("linksetdbs", []):
            if db.get("linkname") == linkname:
                for l in db.get("links", []):
                    links.append(l["id"] if isinstance(l, dict) else l)
    return links


def fetch_citations(pmid):
    """Return PMIDs that this PubMed article cites."""
    refs = fetch_elink([pmid], "pubmed_pubmed_refs")
    return refs


def parse_pubmed_date(info):
    spd = (info.get("sortpubdate") or "").strip()
    if spd:
        try:
            date_part = spd.split()[0]
            ymd = [int(x) for x in date_part.split("/")]
            y = ymd[0]
            m = ymd[1] if len(ymd) >= 2 else 1
            return datetime(y, m, 1)
        except Exception:
            pass
    for key in ("epubdate", "pubdate"):
        raw = info.get(key) or ""
        if not raw:
            continue
        m = re.match(r"(\d{4})(?:/(\d{1,2}))?", raw)
        if m:
            y = int(m.group(1))
            mo = int(m.group(2)) if m.group(2) else 1
            return datetime(y, mo, 1)
    m = re.search(r"(\d{4})", " ".join(info.values()))
    if m:
        return datetime(int(m.group(1)), 1, 1)
    return datetime(1900, 1, 1)


def wrap_text(text, width=AVG_CHARS_PER_YEAR):
    """
    Wrap visible text to a given width, ignoring HTML tags like <a href=...>.
    Ensures <br> is only inserted between complete tokens, never inside tags.
    """
    import re

    # Split text into link and non-link parts
    parts = re.split(r'(<a .*?>.*?</a>)', text)
    lines = []
    current_line = ""
    visible_len = 0

    for part in parts:
        if part.startswith("<a "):  # keep link as a single token
            token = part
            token_len = len(re.sub(r'<.*?>', '', token))  # visible text length only
        else:
            # Split non-HTML text into words
            words = part.split()

            for w in words:
                w_len = len(w)
                if visible_len + w_len + 1 > width:
                    lines.append(current_line)
                    current_line = w
                    visible_len = w_len
                else:
                    if current_line:
                        current_line += " " + w
                        visible_len += w_len + 1
                    else:
                        current_line = w
                        visible_len = w_len
            continue

        # Handle full <a> block (count visible chars but don’t break inside)
        if visible_len + token_len + 1 > width:
            lines.append(current_line)
            current_line = token
            visible_len = token_len
        else:
            if current_line:
                current_line += " " + token
                visible_len += token_len + 1
            else:
                current_line = token
                visible_len = token_len

    if current_line:
        lines.append(current_line)

    return "<br>".join(lines)

def fetch_pubmed_ids(term, retmax):
    esearch_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"
    params = {"db": "pubmed", "term": term, "retmax": retmax, "retmode": "json", "api_key": API_KEY}
    r = requests.get(esearch_url, params=params, timeout=60)
    r.raise_for_status()
    return r.json()["esearchresult"]["idlist"]


def fetch_pubmed_summaries(ids, batch_size=100, cache=None):
    """Fetch PubMed summaries in batches, caching per PMID."""
    if not ids:
        return {}

    all_results = {}
    esummary_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi"

    cached_ids = set(cache.keys()) if cache else set()
    to_fetch = [pid for pid in ids if pid not in cached_ids]

    print(f" {len(ids)} total IDs, {len(to_fetch)} to fetch, {len(cached_ids)} from cache.")

    # Add cached ones directly
    if cache:
        for pid in ids:
            if pid in cache:
                all_results[pid] = cache[pid]

    # Fetch remaining batches
    for i in range(0, len(to_fetch), batch_size):
        batch = to_fetch[i:i + batch_size]
        params = {
            "db": "pubmed",
            "id": ",".join(batch),
            "retmode": "json",
            "api_key": API_KEY
        }
        try:
            r = requests.get(esummary_url, params=params, timeout=120)
            r.raise_for_status()
            result = r.json().get("result", {})
            result.pop("uids", None)
            for pid, val in result.items():
                all_results[pid] = val
                if cache is not None:
                    cache[pid] = val
            print(f"Fetched summaries for batch {i//batch_size + 1} ({len(batch)} PMIDs).")
        except Exception as e:
            print(f"Failed to fetch summaries for batch {i//batch_size + 1}: {e}")
            continue

    return all_results



def fetch_pubmed_xml(ids, batch_size=100, cache=None):
    """
    Fetch PubMed XML data in batches and cache per PMID.
    Cached XMLs are stored in JSON (PMID -> <PubmedArticle>...</PubmedArticle>).
    """
    if not ids:
        return "<PubmedArticleSet/>"

    efetch_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi"
    xml_parts = []
    cached_ids = set(cache.keys()) if cache else set()
    to_fetch = [pid for pid in ids if pid not in cached_ids]

    print(f" {len(ids)} total IDs, {len(to_fetch)} to fetch, {len(cached_ids)} from cache.")

    # Include already cached XMLs
    for pid in ids:
        if pid in cached_ids:
            xml_parts.append(cache[pid])

    # Fetch remaining XMLs
    for i in range(0, len(to_fetch), batch_size):
        batch = to_fetch[i:i + batch_size]
        params = {
            "db": "pubmed",
            "id": ",".join(batch),
            "retmode": "xml",
            "api_key": API_KEY
        }
        try:
            r = requests.get(efetch_url, params=params, timeout=180)
            r.raise_for_status()
            xml_text = r.text

            # Split into individual <PubmedArticle> entries
            articles = re.findall(r"<PubmedArticle>.*?</PubmedArticle>", xml_text, re.DOTALL)
            for article_xml in articles:
                pmid_match = re.search(r"<PMID.*?>(\d+)</PMID>", article_xml)
                if pmid_match:
                    pmid = pmid_match.group(1)
                    cache[pmid] = article_xml
                    xml_parts.append(article_xml)

            print(f"Fetched XML batch {i//batch_size + 1} ({len(batch)} PMIDs).")
            time.sleep(0.3)  # polite delay
        except Exception as e:
            print(f" Failed to fetch XML batch {i//batch_size + 1}: {e}")
            continue

    # Wrap everything into one combined XML
    return "<PubmedArticleSet>\n" + "\n".join(xml_parts) + "\n</PubmedArticleSet>"



def extract_terms_from_xml_per_article(xml_text):
    """
    Parse PubMed XML that may contain multiple <PubmedArticle> entries.
    Extract terms article-by-article, so one bad record doesn't break everything.
    """
    tags_by_pmid = defaultdict(set)
    counts = Counter()

    # Split by article tags instead of parsing the whole set at once
    articles = re.findall(r"<PubmedArticle>.*?</PubmedArticle>", xml_text, re.DOTALL)
    print(f" Found {len(articles)} articles in XML chunk")

    for i, article_xml in enumerate(articles):
        try:
            root = ET.fromstring(article_xml)
        except ET.ParseError as e:
            print(f" Skipping malformed article #{i+1}: {e}")
            continue

        pmid_el = root.find(".//MedlineCitation/PMID")
        pmid = pmid_el.text.strip() if pmid_el is not None and pmid_el.text else None
        if not pmid:
            continue

        for mh in root.findall(".//MeshHeading/DescriptorName"):
            if mh.text:
                t = mh.text.strip()
                counts[t] += 1
                tags_by_pmid[pmid].add(t)
        for kw in root.findall(".//KeywordList/Keyword"):
            if kw.text:
                t = kw.text.strip()
                counts[t] += 1
                tags_by_pmid[pmid].add(t)

    return counts, tags_by_pmid

def build_annotations(data, min_year, max_year, citations_table=False, wrap_width=AVG_CHARS_PER_YEAR):
    """Return annotations (unused now) and an HTML mapping {year: html_content}."""
    annotations = []
    html_by_year = {}

    for year in range(min_year, max_year + 1):
        pubs = [d for d in data if d["Date"].year == year and d["IsCitationOnly"] == citations_table]
        if not pubs:
            continue
        pubs.sort(key=lambda d: int(re.sub(r"[^0-9]", "", d["Rank"])))

        lines = []
        for p in pubs:
            base = (
                f"{p['Rank']}: {p['Title']} "
                f"(<a href='https://pubmed.ncbi.nlm.nih.gov/{p['PMID']}/' target='_blank'>PubMed</a>)"
            )
            if p.get("Tags"):
                tags_html = ", ".join(p["Tags"])
                # Build PubMed AND search query for all tags
                query = "+AND+".join(re.sub(r'[^A-Za-z0-9+]', '+', t) for t in p["Tags"])
                pubmed_tag_url = f"https://pubmed.ncbi.nlm.nih.gov/?term={query}"
                base += (
                    f"<br><small>Tags: {tags_html} "
                    f"(<a href='{pubmed_tag_url}' target='_blank'>PubMed</a>)</small>"
                )
            lines.append(base)

        html_by_year[year] = "<br><br>".join(lines)

    return annotations, html_by_year

def build_figure(data, min_year, max_year, citation_edges=None):
    fig = go.Figure()

    # Add vertical lines for each year
    for year in range(min_year, max_year + 2):
        fig.add_vline(
            x=datetime(year, 1, 1),
            line=dict(color="gray", dash="dot", width=1),
            layer="below"
        )

    regular = [d for d in data if not d.get("IsCitationOnly")]
    citations = [d for d in data if d.get("IsCitationOnly")]

    # Regular bubbles
    fig.add_trace(go.Scatter(
        x=[d["Date"] for d in regular],
        y=[d["y"] for d in regular],
        mode="markers+text",
        text=[d["Rank"] for d in regular],
        textposition="middle center",
        textfont=dict(size=10, color="black"),
        marker=dict(
            size=[max(MIN_BUBBLE_SIZE, d["Citations"]/SCALE_FACTOR) for d in regular],
            color="skyblue",
            line=dict(width=1, color="black"),
            symbol="circle-open-dot"
        ),
        hovertext=[
            f"<b>{d['Title']}</b><br>PMID:{d['PMID']}"
            for d in regular
        ],
        hoverinfo="text"
    ))

    # Citation-only bubbles (gray dotted)
    if citations:
        fig.add_trace(go.Scatter(
            x=[d["Date"] for d in citations],
            y=[d["y"] for d in citations],
            mode="markers+text",
            text=[d["Rank"] for d in citations],
            textposition="middle center",
            textfont=dict(size=10, color="orange"),
            marker=dict(
                size=[max(MIN_BUBBLE_SIZE, d["Citations"]/SCALE_FACTOR) for d in citations],
                color="white",
                line=dict(width=2, color="gray"),
                symbol="circle"
            ),
            hovertext=[
                f"<b>{d['Title']}</b><br>PMID:{d['PMID']} (citation only)"
                for d in citations
            ],
            hoverinfo="text"
        ))

    # Build annotations (title tables)
    #annotations = build_annotations(data, min_year, max_year, citations_table=False)
    #annotations += build_annotations(data, min_year, max_year, citations_table=True)

    max_y = max(d["y"] for d in data)
    bottom_margin = 600
    buffer = 300

    fig.update_layout(
        #annotations=annotations,
        margin=dict(l=80, r=50, t=40, b=0),
        xaxis=dict(side="top", tickformat="%Y", dtick="M12",
                   range=[datetime(min_year, 1, 1) - timedelta(days=30),
                          datetime(max_year + 1, 1, 1)],
                   fixedrange=True),
        yaxis=dict(visible=False, range=[-bottom_margin, max_y + buffer], fixedrange=True),
        showlegend=False,
        height=1000,
        width=(max_year - min_year + 1) * 700,
    )
    return fig

def main():
    print(f"Fetching PubMed results for: {TERM}")
    ids = fetch_pubmed_ids(TERM, RETMAX)

    # --- Load caches ---
    summary_cache = load_summary_cache()
    citation_cache = load_cache()
    xml_cache = load_xml_cache()

    # --- Summaries for initial search ---
    summaries = fetch_pubmed_summaries(ids, cache=summary_cache)
    save_summary_cache(summary_cache)
    print(f"Initial search returned {len(ids)} PubMed IDs.")

    # --- Fetch citations ---
    citation_edges = []
    all_pmids = set(ids)
    total = len(ids)
    for idx, pid in enumerate(ids, start=1):
        try:
            if pid in citation_cache:
                refs = citation_cache[pid]
                print(f"[{idx}/{total}] {pid} (cached) cites {len(refs)} papers.")
            else:
                refs = fetch_citations(pid)
                citation_cache[pid] = refs
                print(f"[{idx}/{total}] {pid} cites {len(refs)} papers.")
                time.sleep(0.25)
            for ref in refs:
                citation_edges.append((pid, ref))
                all_pmids.add(ref)
        except Exception as e:
            print(f"[{idx}/{total}]  Failed to fetch citations for {pid}: {e}")
    save_cache(citation_cache)

    # --- Fetch missing summaries ---
    new_pmids = list(all_pmids - set(ids))
    if new_pmids:
        print(f"Fetching {len(new_pmids)} cited papers not in main search...")
        cited_summaries = fetch_pubmed_summaries(new_pmids, cache=summary_cache)
        save_summary_cache(summary_cache)
        summaries.update(cited_summaries)
    else:
        print("No additional cited papers found.")

    # --- Build data entries ---
    data = []
    date_groups = defaultdict(list)
    for pid in all_pmids:
        info = summaries.get(pid, {})
        title = info.get("title", f"Unknown Title ({pid})")
        pubdate = info.get("pubdate", "1900")
        xdate = parse_pubmed_date(info)
        citations = random.randint(10, 300)
        d = {
            "Title": title,
            "PubDate": pubdate,
            "Date": xdate,
            "Citations": citations,
            "PMID": pid,
            "y": None,
            "Tags": [],
            "IsCitationOnly": pid not in ids,
        }
        data.append(d)
        date_groups[xdate].append(d)

    # vertical stacking
    for date, group in date_groups.items():
        sorted_group = sorted(group, key=lambda d: -d["Citations"])
        y_offset = 0
        for item in sorted_group:
            bubble_height = max(MIN_BUBBLE_SIZE, item["Citations"] / SCALE_FACTOR)
            item["y"] = y_offset
            y_offset += bubble_height * 8

    years = sorted({d["Date"].year for d in data})
    min_year, max_year = min(years), max(years)

    # --- Assign ranks ---
    main_pubs = [d for d in data if not d["IsCitationOnly"]]
    main_pubs.sort(key=lambda d: -d["Citations"])
    for i, pub in enumerate(main_pubs, start=1):
        pub["Rank"] = f"#{i}"

    citation_pubs = [d for d in data if d["IsCitationOnly"]]
    citation_pubs.sort(key=lambda d: -d["Citations"])
    for i, pub in enumerate(citation_pubs, start=1):
        pub["Rank"] = f"#{i}"

    # --- Extract MeSH / Keyword tags ---
    xml_text = fetch_pubmed_xml(list(all_pmids), cache=xml_cache)
    save_xml_cache(xml_cache)
    _tag_counts_all, tags_by_pmid = extract_terms_from_xml_per_article(xml_text)

    for dct in data:
        if dct["PMID"] in tags_by_pmid:
            dct["Tags"] = sorted(tags_by_pmid[dct["PMID"]])

    # --- Tag frequencies per year ---
    tags_by_year = defaultdict(Counter)
    for d in data:
        if d["PMID"] in tags_by_pmid and d["Date"].year >= 1900:
            y = d["Date"].year
            for tag in tags_by_pmid[d["PMID"]]:
                tags_by_year[y][tag] += 1

    tag_year_counts = Counter()
    for y, counter in tags_by_year.items():
        for t in counter:
            tag_year_counts[t] += 1

    # --- Build filtered_tags_by_year: dominant tags each year (not limited by number of years)
    TOP_TAGS_PER_YEAR = 10  # display top 10 most frequently used tags every year
    filtered_tags_by_year = {}
    for y, counter in tags_by_year.items():
        # vezmeme jen tagy s nejvyšší četností
        top_tags = dict(sorted(counter.items(), key=lambda x: -x[1])[:TOP_TAGS_PER_YEAR])
        filtered_tags_by_year[y] = top_tags

    # --- Build "New Trends" and "New Dominant Trends" (First 3 Years, labeled as 1st / 2nd / 3rd) ---
    TAG_DISPLAY_YEARS = 3

    # zjisti první rok výskytu každého tagu
    first_year_for_tag = {}
    for y in sorted(tags_by_year.keys()):
        for t in tags_by_year[y]:
            if t not in first_year_for_tag:
                first_year_for_tag[t] = y

    # zjisti první rok výskytu dominantních tagů
    first_year_for_filtered_tag = {}
    for y in sorted(filtered_tags_by_year.keys()):
        for t in filtered_tags_by_year[y]:
            if t not in first_year_for_filtered_tag:
                first_year_for_filtered_tag[t] = y

    def year_suffix(offset):
        return {0: "1st", 1: "2nd", 2: "3rd"}.get(offset, f"{offset+1}th")

    # --- New Trends (all tags) ---
    new_trends = defaultdict(dict)
    for t, first_y in first_year_for_tag.items():
        for offset in range(TAG_DISPLAY_YEARS):
            y = first_y + offset
            if y in tags_by_year and t in tags_by_year[y]:
                new_trends[y][t] = tags_by_year[y][t]

    # --- New Dominant Trends (from filtered tags) ---
    new_dominant_trends = defaultdict(dict)
    for t, first_y in first_year_for_filtered_tag.items():
        for offset in range(TAG_DISPLAY_YEARS):
            y = first_y + offset
            if y in filtered_tags_by_year and t in filtered_tags_by_year[y]:
                new_dominant_trends[y][t] = filtered_tags_by_year[y][t]

    # Determine the first year each tag appeared
    first_year_for_tag = {}
    for y in sorted(tags_by_year.keys()):
        for t in tags_by_year[y]:
            if t not in first_year_for_tag:
                first_year_for_tag[t] = y

    # Keep tag visible only for first 3 years after it appears
    TAG_DISPLAY_YEARS = 3
    new_tag_trend = defaultdict(dict)
    for t, first_y in first_year_for_tag.items():
        for y in range(first_y, first_y + TAG_DISPLAY_YEARS):
            if y in tags_by_year and t in tags_by_year[y]:
                new_tag_trend[y][t] = tags_by_year[y][t]

    # --- Count tags globally ---
    main_pmids_set = {d["PMID"] for d in data if not d["IsCitationOnly"]}
    filtered_counts = Counter()
    for pmid, tags in tags_by_pmid.items():
        if pmid in main_pmids_set:
            for t in tags:
                filtered_counts[t] += 1

    # --- Build figure and tables ---
    fig = build_figure(data, min_year, max_year, citation_edges)
    _, pubs_by_year = build_annotations(data, min_year, max_year, citations_table=False)
    _, cites_by_year = build_annotations(data, min_year, max_year, citations_table=True)

    years = range(min_year, max_year + 1)
    year_width = 697.5
    left_margin = 135
    total_width = left_margin + (max_year - min_year + 1) * year_width

    # --- Build trend tag table ---
    trend_table_html = [
        "<h2>Trends by Year</h2>",
        f"<div style='overflow-x:auto;'>",
        f"<table style='border-collapse:collapse; width:{total_width}px; text-align:left; border:1px solid {BORDER_COLOR};'>",
        "<tr>"
    ]
    trend_table_html.append(f"<th style='width:{left_margin}px; border:1px solid {BORDER_COLOR};'></th>")
    for y in years:
        trend_table_html.append(
            f"<th style='padding:6px; text-align:center; width:{year_width}px; font-weight:600; "
            f"border:1px solid {BORDER_COLOR};'>{y}</th>"
        )
    trend_table_html.append("</tr><tr>")
    trend_table_html.append(
        f"<td style='width:{left_margin}px; border:1px solid {BORDER_COLOR}; font-weight:600;'>Tags</td>"
    )
    for y in years:
        tag_counts = filtered_tags_by_year.get(y, {})
        if not tag_counts:
            cell_html = "<span style='color:#aaa;'>—</span>"
        else:
            counts = list(tag_counts.values())
            min_c, max_c = min(counts), max(counts)

            def scale_font(c):
                if max_c == min_c:
                    return 14
                return 10 + (c - min_c) / (max_c - min_c) * 16

            tags_html = []
            for t, c in sorted(tag_counts.items(), key=lambda x: -x[1]):
                fs = scale_font(c)
                if fs <= 10.5:
                    continue
                tags_html.append(
                    f"<span class='tag' data-tag='{t}' style='font-size:{fs:.1f}px'>{t}</span>"
                )

            if tags_html:
                cell_html = f"<div class='tagbar' data-year='{y}'>{' '.join(tags_html)}</div>"
            else:
                cell_html = "<span style='color:#aaa;'>—</span>"

        trend_table_html.append(
            f"<td style='vertical-align:top; padding:6px; width:{year_width}px; "
            f"white-space:normal; word-wrap:break-word; border:1px solid {BORDER_COLOR};'>{cell_html}</td>"
        )
    trend_table_html.append("</tr></table></div>")
    trend_table_html = "\n".join(trend_table_html)

    def build_tag_table_html(title, tag_data):
        """Builds a trend table with rows for 1st, 2nd, 3rd years of tag appearance (non-consecutive years handled)."""

        # Choose correct first-year map
        first_year_map = (
            first_year_for_filtered_tag
            if tag_data is new_dominant_trends
            else first_year_for_tag
        )

        # Build lookup: tag -> sorted list of years it appears
        tag_years = defaultdict(list)
        for y, tags in tag_data.items():
            for t in tags:
                tag_years[t].append(y)
        for t in tag_years:
            tag_years[t] = sorted(set(tag_years[t]))

        rows = [
            f"<h2>{title}</h2>",
            f"<div style='overflow-x:auto;'>",
            f"<table style='border-collapse:collapse; width:{total_width}px; text-align:left; border:1px solid {BORDER_COLOR};'>",
        ]

        # Header: list of years
        rows.append("<tr>")
        rows.append(f"<th style='width:{left_margin}px; border:1px solid {BORDER_COLOR};'></th>")
        for y in years:
            rows.append(
                f"<th style='padding:6px; text-align:center; width:{year_width}px; font-weight:600; "
                f"border:1px solid {BORDER_COLOR};'>{y}</th>"
            )
        rows.append("</tr>")

        # --- Separate rows for 1st / 2nd / 3rd year tags ---
        for offset, label in enumerate(["1st year", "2nd year", "3rd year"][:TAG_DISPLAY_YEARS]):
            rows.append("<tr>")
            # left-side label column
            rows.append(
                f"<td style='width:{left_margin}px; border:1px solid {BORDER_COLOR}; "
                f"font-weight:600; background:#f8f8f8;'>{label}</td>"
            )

            # fill columns per year
            for y in years:
                tag_counts = tag_data.get(y, {})
                if not tag_counts:
                    rows.append(
                        f"<td style='vertical-align:top; padding:6px; width:{year_width}px; "
                        f"white-space:normal; border:1px solid {BORDER_COLOR}; color:#aaa;'>—</td>"
                    )
                    continue

                # compute font scaling
                counts = list(tag_counts.values())
                min_c, max_c = min(counts), max(counts)

                def scale_font(c):
                    if max_c == min_c:
                        return 14
                    return 10 + (c - min_c) / (max_c - min_c) * 16

                tags_html = []
                for t, c in sorted(tag_counts.items(), key=lambda x: -x[1]):
                    tag_year_list = tag_years.get(t, [])
                    if y not in tag_year_list:
                        continue
                    index = tag_year_list.index(y)
                    if index != offset:
                        continue
                    fs = scale_font(c)
                    if fs <= 10.5:
                        continue
                    tags_html.append(
                        f"<span class='tag' data-tag='{t}' style='font-size:{fs:.1f}px'>{t}</span>"
                    )

                cell_html = (
                    f"<div class='tagbar' data-year='{y}'>{' '.join(tags_html)}</div>"
                    if tags_html else "<span style='color:#aaa;'>—</span>"
                )
                rows.append(
                    f"<td style='vertical-align:top; padding:6px; width:{year_width}px; "
                    f"white-space:normal; word-wrap:break-word; border:1px solid {BORDER_COLOR};'>{cell_html}</td>"
                )

            rows.append("</tr>")

        rows.append("</table></div>")
        return "\n".join(rows)

    # --- Build both new tables dynamically based on TAG_DISPLAY_YEARS ---
    new_trends_table_html = build_tag_table_html(
        f"New Trends (First {TAG_DISPLAY_YEARS} Year{'s' if TAG_DISPLAY_YEARS > 1 else ''})", 
        new_trends
    )
    new_dominant_table_html = build_tag_table_html(
        f"New Dominant Trends (First {TAG_DISPLAY_YEARS} Year{'s' if TAG_DISPLAY_YEARS > 1 else ''})", 
        new_dominant_trends
    )

    # --- Build first-time tag table (first 3 years of each tag) ---
    new_tags_table_html = [
        "<h2>New Tags (First 3 Years)</h2>",
        f"<div style='overflow-x:auto;'>",
        f"<table style='border-collapse:collapse; width:{total_width}px; text-align:left; border:1px solid {BORDER_COLOR};'>",
        "<tr>"
    ]
    new_tags_table_html.append(f"<th style='width:{left_margin}px; border:1px solid {BORDER_COLOR};'></th>")
    for y in years:
        new_tags_table_html.append(
            f"<th style='padding:6px; text-align:center; width:{year_width}px; font-weight:600; "
            f"border:1px solid {BORDER_COLOR};'>{y}</th>"
        )
    new_tags_table_html.append("</tr><tr>")
    new_tags_table_html.append(
        f"<td style='width:{left_margin}px; border:1px solid {BORDER_COLOR}; font-weight:600;'>Tags</td>"
    )

    for y in years:
        tag_counts = new_tag_trend.get(y, {})
        if not tag_counts:
            cell_html = "<span style='color:#aaa;'>—</span>"
        else:
            counts = list(tag_counts.values())
            min_c, max_c = min(counts), max(counts)

            def scale_font(c):
                if max_c == min_c:
                    return 14
                return 10 + (c - min_c) / (max_c - min_c) * 16

            tags_html = []
            for t, c in sorted(tag_counts.items(), key=lambda x: -x[1]):
                fs = scale_font(c)
                if fs <= 10.5:
                    continue
                tags_html.append(
                    f"<span class='tag' data-tag='{t}' style='font-size:{fs:.1f}px'>{t}</span>"
                )

            cell_html = f"<div class='tagbar' data-year='{y}'>{' '.join(tags_html)}</div>" if tags_html else "<span style='color:#aaa;'>—</span>"

        new_tags_table_html.append(
            f"<td style='vertical-align:top; padding:6px; width:{year_width}px; "
            f"white-space:normal; word-wrap:break-word; border:1px solid {BORDER_COLOR};'>{cell_html}</td>"
        )

    new_tags_table_html.append("</tr></table></div>")
    new_tags_table_html = "\n".join(new_tags_table_html)

    # --- Build publication table ---
    _, pubs_by_year = build_annotations(data, min_year, max_year, citations_table=False)
    _, cites_by_year = build_annotations(data, min_year, max_year, citations_table=True)

    table_html = [
        "<h2>Publications by Year</h2>",
        f"<div style='overflow-x:auto;'>",
        f"<table style='border-collapse:collapse; width:{total_width}px; text-align:left; border:1px solid {BORDER_COLOR};'>",
        "<tr>"
    ]
    table_html.append(f"<th style='width:{left_margin}px; border:1px solid {BORDER_COLOR};'></th>")
    for y in years:
        table_html.append(
            f"<th style='padding:6px; text-align:center; width:{year_width}px; font-weight:600; border:1px solid {BORDER_COLOR};'>{y}</th>"
        )
    table_html.append("</tr><tr>")
    table_html.append(f"<td style='width:{left_margin}px; border:1px solid {BORDER_COLOR};'></td>")
    for y in years:
        pubs = pubs_by_year.get(y, "")
        table_html.append(
            f"<td style='vertical-align:top; padding:6px; font-size:13px; width:{year_width}px; white-space:normal; "
            f"word-wrap:break-word; line-height:1.4em; border:1px solid {BORDER_COLOR};'>{pubs}</td>"
        )
    table_html.append("</tr>")
    table_html.append("<tr class='citation-row' style='display:none;'>")
    table_html.append(f"<td style='width:{left_margin}px; border:1px solid {BORDER_COLOR};'></td>")
    for _ in years:
        table_html.append(
            f"<td style='vertical-align:top; padding:6px; font-size:13px; color:#555; width:{year_width}px; white-space:normal; "
            f"word-wrap:break-word; line-height:1.4em; border:1px solid {BORDER_COLOR};'></td>"
        )
    table_html.append("</tr></table></div>")
    tables_html = "\n".join(table_html)

    # --- JSON for HTML ---
    data_js = []
    for dct in data:
        dj = dict(dct)
        dj["Date"] = dct["Date"].strftime("%Y-%m-%d")
        data_js.append(dj)
    pubs_json = json.dumps(data_js, ensure_ascii=False)
    edges_json = json.dumps(citation_edges, ensure_ascii=False)
    tag_counts_json = json.dumps(dict(filtered_counts.most_common(200)), ensure_ascii=False)
    plot_div = pio.to_html(fig, include_plotlyjs="inline", full_html=False, config={"displaylogo": False})
    main_count = sum(1 for d in data if not d["IsCitationOnly"])
    citation_count = sum(1 for d in data if d["IsCitationOnly"])

    # --- Write final HTML with fixed interactive behavior ---
    html = build_final_html(
        TERM, RETMAX, main_count, citation_count, plot_div,
        trend_table_html, new_trends_table_html, new_dominant_table_html, 
        tables_html, pubs_json, edges_json, tag_counts_json, min_year, max_year
    )

    with open(OUTPUT_HTML, "w", encoding="utf-8") as f:
        f.write(html)
    print(f" Saved single-file HTML to {OUTPUT_HTML}")

def build_final_html(term, retmax, main_count, citation_count,
                     plot_div, trend_table_html, new_trends_table_html, new_dominant_table_html, tables_html,
                     pubs_json, edges_json, tag_counts_json,
                     min_year, max_year):
    """Return full HTML with independent multi-select yearly tags + global filters."""
    return f"""<!DOCTYPE html>
<html lang="en">
<meta charset="utf-8">
<title>PubMed Timeline — {term}</title>
<style>
body {{
  margin: 1.25rem;
  font-family: system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif;
}}
h1 {{ margin-bottom: 0.5rem; }}
.sub {{ color: #555; margin-bottom: 1rem; }}
.tagbar {{
  display: flex;
  flex-wrap: wrap;
  gap: .5rem .75rem;
  margin: .75rem 0 1rem;
  user-select: none;
}}
.tag {{
  cursor: pointer;
  color: #075985;
  border-bottom: 1px dotted rgba(2,132,199,.35);
}}
.tag.active {{
  color: #b45309;
  background: rgba(251,191,36,.15);
  border-bottom-color: transparent;
  border-radius: .25rem;
  padding: .1rem .25rem;
}}
button {{
  margin-right: .5rem;
  padding: .2rem .5rem;
}}
</style>
<body>
<h1>PubMed Search: "{term}"</h1>
<div class="sub">{main_count} publications (limit {retmax}), {citation_count} associated publications</div>
<div id="timeline-container"
     style="overflow-x:auto; white-space:nowrap; border:1px solid rgba(0,0,0,0.1); padding-bottom:1rem;">
  <div style="display:inline-block; vertical-align:top;">
    {plot_div}
    <div id="tag-trends-table">
        {trend_table_html}
        {new_trends_table_html}
        {new_dominant_table_html}
    </div>
    <div id="pub-tables" style="white-space:normal; width:100%;">
      {tables_html}
    </div>
  </div>
</div>
<script>
document.addEventListener("DOMContentLoaded", function() {{
  const container = document.getElementById("timeline-container");
  container.scrollLeft = container.scrollWidth;
}});
</script>
<div style="margin-top:1rem;">
  <button id="hideCitations">Hide all citations</button>
</div>
<h2>Interactive Tags</h2>
<div class="tagbar" id="tagbar"></div>
<div>
  <button id="showAll">Show all</button>
  <button id="hideAll">Hide all</button>
</div>

<script id="pubs-data" type="application/json">{pubs_json}</script>
<script id="edges-data" type="application/json">{edges_json}</script>
<script id="tag-counts" type="application/json">{tag_counts_json}</script>

<script>
(function() {{
  const pubs = JSON.parse(document.getElementById('pubs-data').textContent);
  const edges = JSON.parse(document.getElementById('edges-data').textContent);
  const tagCounts = JSON.parse(document.getElementById('tag-counts').textContent);
  const figDiv = document.querySelector('.plotly-graph-div');
  const origLayout = JSON.parse(JSON.stringify(figDiv.layout));
  let activePMID = null;

  const tagbar = document.getElementById('tagbar');
  const entries = Object.entries(tagCounts).sort((a,b)=>b[1]-a[1]);

  // Global tags: all active initially
  const selectedGlobal = new Set(entries.map(([t])=>t));

  // Yearly tags: map of year -> Set of tags
  const selectedYearly = new Map();

  const minYear = {min_year};
  const maxYear = {max_year};

  function fontSize(c) {{
    const vals = entries.map(e=>e[1]);
    const minC = Math.min(...vals), maxC = Math.max(...vals);
    return 12 + (c - minC)/(maxC - minC + 1e-9)*16;
  }}

  function renderTags() {{
    tagbar.innerHTML='';
    entries.forEach(([tag,count]) => {{
      const s = document.createElement('span');
      s.className = 'tag' + (selectedGlobal.has(tag)?' active':'');
      s.textContent = tag;
      s.style.fontSize = fontSize(count)+'px';
      s.onclick = () => {{
        if (selectedGlobal.has(tag)) selectedGlobal.delete(tag);
        else selectedGlobal.add(tag);
        renderTags();
        applyFilter();
      }};
      tagbar.appendChild(s);
    }});
  }}

  // --- Year trend tag logic (multi-select per year) ---
  function bindYearTags() {{
    document.querySelectorAll('.tagbar[data-year] .tag').forEach(el => {{
      el.onclick = () => {{
        const tag = el.getAttribute('data-tag');
        const year = el.closest('.tagbar').getAttribute('data-year');
        const yearSet = selectedYearly.get(year) || new Set();

        if (yearSet.has(tag)) {{
          yearSet.delete(tag);
        }} else {{
          yearSet.add(tag);
        }}

        if (yearSet.size === 0) selectedYearly.delete(year);
        else selectedYearly.set(year, yearSet);

        // refresh visuals for this year's tags
        document.querySelectorAll(`.tagbar[data-year="${{year}}"] .tag`).forEach(t => {{
          const ttag = t.getAttribute('data-tag');
          t.classList.toggle('active', yearSet.has(ttag));
        }});

        applyFilter();
      }};
    }});
  }}

  function redraw(mainList, citeList, linePairs) {{
    const lineTraces = (linePairs || []).map(([a,b]) => ({{
      x: [new Date(a.Date), new Date(b.Date)],
      y: [a.y, b.y],
      mode: "lines",
      line: {{ color: "orange", width: 1.5, dash: "dash" }},
      hoverinfo: "skip",
      showlegend: false
    }}));

    const mainTrace = {{
      x: mainList.map(d=>new Date(d.Date)),
      y: mainList.map(d=>d.y),
      mode:"markers+text",
      text: mainList.map(d => `${{d.Rank}}`),
      textposition:"middle center",
      marker:{{
        size:mainList.map(d=>Math.max({MIN_BUBBLE_SIZE}, d.Citations/{SCALE_FACTOR})),
        color:"skyblue",
        line:{{width:1,color:"black"}}
      }},
      textfont:{{size:10,color:"black"}},
      hovertext: mainList.map(d => `<b>${{d.Title}}</b><br>PMID:${{d.PMID}}<br>Citations:${{d.Citations}}`),
      hoverinfo:"text",
      customdata: mainList.map(d => d.PMID)
    }};

    const citeTrace = citeList.length ? {{
      x: citeList.map(d => new Date(d.Date)),
      y: citeList.map(d => d.y),
      mode: "markers+text",
      text: citeList.map(d => d.Rank),
      textposition: "middle center",
      textfont: {{size:10, color:"orange"}},
      marker: {{
        size: citeList.map(d => Math.max({MIN_BUBBLE_SIZE}, d.Citations/{SCALE_FACTOR})),
        color: "white",
        line: {{width:2, color:"gray", dash: "dot"}},
        symbol: "circle"
      }},
      hovertext: citeList.map(d => `<b>${{d.Title}}</b><br>PMID:${{d.PMID}} (citation)`),
      hoverinfo: "text"
    }} : null;

    const layout = JSON.parse(JSON.stringify(origLayout));
    const traces = [mainTrace, ...lineTraces];
    if (citeTrace) traces.push(citeTrace);
    Plotly.react(figDiv, traces, layout);
  }}

  function applyFilter() {{
    const allGlobal = selectedGlobal.size === entries.length;
    const noYearFilters = selectedYearly.size === 0;

    let mainList;
    if (selectedGlobal.size === 0) {{
      mainList = [];
    }} else if (allGlobal && noYearFilters) {{
      mainList = pubs.filter(p => !p.IsCitationOnly);
    }} else {{
      mainList = pubs.filter(p => {{
        if (p.IsCitationOnly) return false;
        const y = String(new Date(p.Date).getFullYear());
        const globalOK = (p.Tags || []).some(t => selectedGlobal.has(t));
        const yearSet = selectedYearly.get(y);
        const yearOK = !yearSet || (p.Tags || []).some(t => yearSet.has(t));
        return globalOK && yearOK;
      }});
    }}

    let citeList = [];
    let linePairs = [];

    if (activePMID) {{
      const anchor = pubs.find(p => p.PMID === activePMID);
      if (anchor) {{
        const outgoing = edges.filter(([child]) => child === activePMID);
        const incoming = edges.filter(([child, parent]) => parent === activePMID);
        const related = [...new Set([...outgoing.map(e=>e[1]), ...incoming.map(e=>e[0])])];
        citeList = pubs.filter(p => related.includes(p.PMID));
        linePairs = citeList.map(c => [anchor, c]);
      }}
    }}

    redraw(mainList, citeList, linePairs);
  }}

  figDiv.on('plotly_click', function(ev) {{
    if (!ev || !ev.points.length) return;
    const curve = ev.points[0].curveNumber;
    if (curve !== 0) return;
    const pmid = ev.points[0].customdata;
    const clicked = pubs.find(p => p.PMID === pmid && !p.IsCitationOnly);
    if (!clicked) return;
    activePMID = (activePMID === pmid) ? null : pmid;
    applyFilter();
  }});

  document.getElementById('hideCitations').onclick = () => {{
    activePMID = null;
    applyFilter();
  }};

  document.getElementById('showAll').onclick = () => {{
    entries.forEach(([t])=>selectedGlobal.add(t));
    renderTags();
    applyFilter();
  }};
  document.getElementById('hideAll').onclick = () => {{
    selectedGlobal.clear();
    renderTags();
    applyFilter();
  }};

  renderTags();
  bindYearTags();
  applyFilter();
}})();
</script>
</body>
</html>"""

if __name__ == "__main__":
    main()