tuned-org-uk · Mec-iS · May 7, 2026 · May 7, 2026
diff --git a/neurips/Trec-covid/README.md b/neurips/Trec-covid/README.md
@@ -0,0 +1,70 @@
+# Trec-covid Reproducibility Pack
+
+This folder contains a reproducible setup for TREC-COVID experiments focused on:
+- `k=10` tau sweep analysis
+- `eps/build_k` validation sweep (nDCG@10)
+- plot generation and copied reference outputs
+
+## Folder layout
+
+- `scripts/`: runnable scripts for download, embedding, experiments, and plotting.
+- `data/`: dataset location (`trec-covid/` created here).
+- `embeddings/`: embedding cache outputs.
+- `results/k10_tau/`: logs, plots, and LaTeX artifacts for `k=10` tau sweep.
+- `results/k10_eps_buildk/`: logs and plots for `eps/build_k` sweep.
+
+## Dataset source (TREC-COVID)
+
+Dataset is from BEIR:
+- URL: `https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/trec-covid.zip`
+- Download script: `scripts/01_download_trec_covid.sh`
+
+Expected extracted files:
+- `data/trec-covid/corpus.jsonl`
+- `data/trec-covid/queries.jsonl`
+- `data/trec-covid/qrels/`
+
+## Reproduce
+
+Run from repo root:
+
+```bash
+bash Trec-covid/scripts/00_run_all.sh
+```
+
+Or step-by-step:
+
+```bash
+bash Trec-covid/scripts/01_download_trec_covid.sh
+bash Trec-covid/scripts/02_embed_minilm.sh
+bash Trec-covid/scripts/03_run_k10_tau_report.sh
+bash Trec-covid/scripts/04_make_neurips_style_plots.sh
+bash Trec-covid/scripts/05_run_k10_eps_buildk_sweep.sh
+```
+
+## Script details
+
+- `01_download_trec_covid.sh`
+  - Downloads and unzips BEIR TREC-COVID into `Trec-covid/data/`.
+
+- `02_embed_minilm.sh`
+  - Uses pretrained `sentence-transformers/all-MiniLM-L6-v2`.
+  - Writes embedding cache into `Trec-covid/embeddings/no_finetune_minilm/cache/`.
+
+- `03_run_k10_tau_report.sh`
+  - Runs full tau sweep at `k=10`.
+  - Writes outputs to `Trec-covid/results/k10_tau/`.
+
+- `04_make_neurips_style_plots.sh`
+  - Generates NeurIPS-style plots/tables from `k=10` logs.
+
+- `05_run_k10_eps_buildk_sweep.sh`
+  - Runs manual grid over `eps` and `build_k` for `nDCG@10`.
+  - Writes outputs to `Trec-covid/results/k10_eps_buildk/`.
+
+## Notes
+
+- Default device for embedding is `mps`. Override with:
+  - `DEVICE=cpu bash Trec-covid/scripts/02_embed_minilm.sh`
+- If matplotlib cache warnings appear, set:
+  - `export MPLCONFIGDIR=/tmp/mplcache`
diff --git a/neurips/Trec-covid/results/k10_tau/latex/trec_qrels_table_k10.tex b/neurips/Trec-covid/results/k10_tau/latex/trec_qrels_table_k10.tex
@@ -0,0 +1,13 @@
+\begin{table}[h]
+\centering
+\caption{Qrels-based quality at $k=10$ (mean $\pm$ std over queries).}
+\begin{tabular}{lrr}
+\toprule
+Method & nDCG@k & Relevance Recall@k \\
+\midrule
+Cosine (tau=1.0) & $0.455\pm0.304$ & $0.508\pm0.313$ \\
+hybrid (tau=0.7) & $\mathbf{0.467\pm0.313}$ & $\mathbf{0.520\pm0.312}$ \\
+taumode (tau=0.4) & $0.420\pm0.309$ & $0.478\pm0.309$ \\
+\bottomrule
+\end{tabular}
+\end{table}
diff --git a/neurips/Trec-covid/results/k10_tau/latex/trec_rank_corr_table_k10.tex b/neurips/Trec-covid/results/k10_tau/latex/trec_rank_corr_table_k10.tex
@@ -0,0 +1,13 @@
+\begin{table}[h]
+\centering
+\caption{Pairwise Spearman/Kendall on per-query nDCG at $k=10$.}
+\begin{tabular}{lrr}
+\toprule
+Method Pair & Spearman & Kendall \\
+\midrule
+Cosine vs Hybrid & 0.973 & 0.890 \\
+Cosine vs Taumode & 0.869 & 0.744 \\
+Hybrid vs Taumode & 0.913 & 0.830 \\
+\bottomrule
+\end{tabular}
+\end{table}
diff --git a/neurips/Trec-covid/results/k10_tau/latex/trec_recall_table_k10.tex b/neurips/Trec-covid/results/k10_tau/latex/trec_recall_table_k10.tex
@@ -0,0 +1,13 @@
+\begin{table}[h]
+\centering
+\caption{Recall metrics at $k=10$ (mean $\pm$ std over queries).}
+\begin{tabular}{lrrr}
+\toprule
+Recall variant & Cosine & hybrid & taumode \\
+\midrule
+Traditional Recall@10 & $\mathbf{1.000\pm0.000}$ & $0.806\pm0.186$ & $0.666\pm0.280$ \\
+Semantic Recall@10 & $\mathbf{1.000\pm0.000}$ & $0.860\pm0.253$ & $0.647\pm0.377$ \\
+Tolerant Recall@10 & $\mathbf{1.000\pm0.000}$ & $0.990\pm0.071$ & $\mathbf{1.000\pm0.000}$ \\
+\bottomrule
+\end{tabular}
+\end{table}
diff --git a/neurips/Trec-covid/results/k10_tau/latex/trec_tables_k10.tex b/neurips/Trec-covid/results/k10_tau/latex/trec_tables_k10.tex
@@ -0,0 +1,69 @@
+\begin{table}[h]
+\centering
+\caption{Mean tail-shape metrics at $k=10$ from TREC logs.}
+\begin{tabular}{lrrrrr}
+\toprule
+Method & Head mean & Tail mean & T/H Ratio & Tail CV & Tail Decay \\
+\midrule
+Cosine (tau=1.0) & 0.802 & 0.761 & 0.94862 & 0.01362 & 0.004203 \\
+hybrid (tau=0.7) & 0.849 & 0.820 & 0.96616 & 0.00886 & 0.002964 \\
+taumode (tau=0.4) & 0.908 & 0.890 & \textbf{0.98009} & \textbf{0.00459} & \textbf{0.001647} \\
+\bottomrule
+\end{tabular}
+\end{table}
+
+\begin{table}[h]
+\centering
+\caption{Per-query winners over 50 queries at $k=10$.}
+\begin{tabular}{lrrr}
+\toprule
+Metric & Cosine wins & hybrid wins & taumode wins \\
+\midrule
+T/H Ratio & 0 & 1 & \textbf{49} \\
+Tail CV & 0 & 2 & \textbf{48} \\
+Tail Decay & 0 & 2 & \textbf{48} \\
+\bottomrule
+\end{tabular}
+\end{table}
+
+\begin{table}[h]
+\centering
+\caption{Recall metrics at $k=10$ (mean $\pm$ std over queries).}
+\begin{tabular}{lrrr}
+\toprule
+Recall variant & Cosine & hybrid & taumode \\
+\midrule
+Traditional Recall@10 & $\mathbf{1.000\pm0.000}$ & $0.806\pm0.186$ & $0.666\pm0.280$ \\
+Semantic Recall@10 & $\mathbf{1.000\pm0.000}$ & $0.860\pm0.253$ & $0.647\pm0.377$ \\
+Tolerant Recall@10 & $\mathbf{1.000\pm0.000}$ & $0.990\pm0.071$ & $\mathbf{1.000\pm0.000}$ \\
+\bottomrule
+\end{tabular}
+\end{table}
+
+\begin{table}[h]
+\centering
+\caption{Qrels-based quality at $k=10$ (mean $\pm$ std over queries).}
+\begin{tabular}{lrr}
+\toprule
+Method & nDCG@k & Relevance Recall@k \\
+\midrule
+Cosine (tau=1.0) & $0.455\pm0.304$ & $0.508\pm0.313$ \\
+hybrid (tau=0.7) & $\mathbf{0.467\pm0.313}$ & $\mathbf{0.520\pm0.312}$ \\
+taumode (tau=0.4) & $0.420\pm0.309$ & $0.478\pm0.309$ \\
+\bottomrule
+\end{tabular}
+\end{table}
+
+\begin{table}[h]
+\centering
+\caption{Pairwise Spearman/Kendall on per-query nDCG at $k=10$.}
+\begin{tabular}{lrr}
+\toprule
+Method Pair & Spearman & Kendall \\
+\midrule
+Cosine vs Hybrid & 0.973 & 0.890 \\
+Cosine vs Taumode & 0.869 & 0.744 \\
+Hybrid vs Taumode & 0.913 & 0.830 \\
+\bottomrule
+\end{tabular}
+\end{table}
diff --git a/neurips/Trec-covid/results/k10_tau/latex/trec_tail_table_k10.tex b/neurips/Trec-covid/results/k10_tau/latex/trec_tail_table_k10.tex
@@ -0,0 +1,13 @@
+\begin{table}[h]
+\centering
+\caption{Mean tail-shape metrics at $k=10$ from TREC logs.}
+\begin{tabular}{lrrrrr}
+\toprule
+Method & Head mean & Tail mean & T/H Ratio & Tail CV & Tail Decay \\
+\midrule
+Cosine (tau=1.0) & 0.802 & 0.761 & 0.94862 & 0.01362 & 0.004203 \\
+hybrid (tau=0.7) & 0.849 & 0.820 & 0.96616 & 0.00886 & 0.002964 \\
+taumode (tau=0.4) & 0.908 & 0.890 & \textbf{0.98009} & \textbf{0.00459} & \textbf{0.001647} \\
+\bottomrule
+\end{tabular}
+\end{table}
diff --git a/neurips/Trec-covid/results/k10_tau/latex/trec_winners_table_k10.tex b/neurips/Trec-covid/results/k10_tau/latex/trec_winners_table_k10.tex
@@ -0,0 +1,13 @@
+\begin{table}[h]
+\centering
+\caption{Per-query winners over 50 queries at $k=10$.}
+\begin{tabular}{lrrr}
+\toprule
+Metric & Cosine wins & hybrid wins & taumode wins \\
+\midrule
+T/H Ratio & 0 & 1 & \textbf{49} \\
+Tail CV & 0 & 2 & \textbf{48} \\
+Tail Decay & 0 & 2 & \textbf{48} \\
+\bottomrule
+\end{tabular}
+\end{table}
diff --git a/neurips/Trec-covid/results/k10_tau/logs/report_config.json b/neurips/Trec-covid/results/k10_tau/logs/report_config.json
@@ -0,0 +1,11 @@
+{
+  "eval_k": 10,
+  "head_k": 3,
+  "reference_tau": 1.0,
+  "semantic_percentile": 25.0,
+  "relevance_recall_definition": "qrel_hits_at_k_divided_by_k",
+  "semantic_uplift_internal": "tolerant_recall_minus_traditional_recall_vs_tau1_reference",
+  "semantic_uplift_qrel": "tolerant_recall_minus_qrel_hits_at_k_divided_by_k",
+  "csv_dir": "Test-clean/trec-covid/no_finetune_minilm/scale100_full_tau_report/k_10/logs",
+  "plot_dir": "Test-clean/trec-covid/no_finetune_minilm/scale100_full_tau_report/k_10/plots"
+}
diff --git a/neurips/Trec-covid/results/k10_tau/logs/trec_headk_sweep.csv b/neurips/Trec-covid/results/k10_tau/logs/trec_headk_sweep.csv
@@ -0,0 +1,17 @@
+tau,head_k,eval_k,tail_to_head_ratio_mean,tail_to_head_ratio_std,tail_to_head_ratio_ci95,n_queries
+0.4,3,10,0.9800895687177058,0.0124984120418682,0.0034643830682764944,50
+0.4,5,10,0.9834664271661887,0.009064738298039611,0.0025126172647282835,50
+0.5,3,10,0.975317151787956,0.0153885916265571,0.004265499976883227,50
+0.5,5,10,0.9792231823398794,0.011319010460375195,0.003137469628718284,50
+0.55,3,10,0.9731918256007874,0.016634087211239754,0.004610733739439095,50
+0.55,5,10,0.9771918506958005,0.012125148597746322,0.0033609197201735046,50
+0.6,3,10,0.9708592641445312,0.017959786852872313,0.004978198932353758,50
+0.6,5,10,0.9749484502335171,0.012973595671633004,0.003596097250507382,50
+0.65,3,10,0.968569495203471,0.019430630291895526,0.005385895933314204,50
+0.65,5,10,0.9730787405614554,0.013937066304412824,0.0038631576847291626,50
+0.7,3,10,0.9661578774118559,0.02080738694740698,0.005767513408439301,50
+0.7,5,10,0.9711014234049012,0.014912368826393158,0.0041334977513134115,50
+0.8,3,10,0.9607023602469624,0.02337351444161576,0.006478807660236655,50
+0.8,5,10,0.9662049983841007,0.016906917446209067,0.004686358422268826,50
+1.0,3,10,0.9486224857953602,0.02881096057852758,0.007985990834249458,50
+1.0,5,10,0.9558078442986491,0.021175800321909587,0.005869632385832421,50