Skip to content

Commit 76a7291

Browse files
acesnikacesnik
authored
Merge branch 'master' into guard/no-x64-only-thermo-reference
2 parents d1cf9ff + 5ba1315 commit 76a7291

18 files changed

Lines changed: 3207 additions & 1 deletion

‎.gitattributes‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,15 @@
6262
#*.rtf diff=astextplain
6363
#*.RTF diff=astextplain
6464

65+
###############################################################################
66+
# SDRF-Proteomics test fixtures are byte-exact regression inputs.
67+
# `* text=auto` above would store them LF-normalized and hand them back CRLF on
68+
# a checkout with core.autocrlf=true, so a byte-identical round-trip test would
69+
# be validating git's conversion rather than the reader, and would fail on any
70+
# clone with autocrlf=false or input. Keep these bytes exactly as committed.
71+
###############################################################################
72+
*.sdrf.tsv -text
73+
6574
###############################################################################
6675
# Byte-exact ontology and vocabulary files
6776
#

‎mzLib/Readers/Sdrf/SdrfCell.cs‎

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
using System.Diagnostics.CodeAnalysis;
2+
using MzLibUtil;
3+
4+
namespace Readers
5+
{
6+
/// <summary>
7+
/// Interprets an SDRF cell's key=value grammar, e.g.
8+
/// "NT=Oxidation;AC=UNIMOD:35;TA=M;MT=Variable".
9+
///
10+
/// This is the opt-in projection that <see cref="SdrfDocument"/> deliberately does not perform
11+
/// at read time. Cells are stored raw so that documents round-trip losslessly; anything that
12+
/// wants to reason about a cell's meaning asks for it here.
13+
///
14+
/// Detection is by LEADING KEY, never by shape. A cell is only treated as a term if it starts
15+
/// with one of the recognised SDRF keys. Testing for "contains '=' and ';'" instead would
16+
/// misfire on real data: comment[file uri] cells in the curated corpus hold pre-signed download
17+
/// URLs whose query strings carry "Signature=", "Expires=" and "Id=" -- 5,750 occurrences each
18+
/// -- and a shape-based parser decodes those as controlled-vocabulary descriptors.
19+
/// </summary>
20+
internal static class SdrfCell
21+
{
22+
/// <summary>
23+
/// The keys the SDRF-Proteomics specification defines. This is the SOURCE OF TRUTH:
24+
/// <see cref="KnownLeadingKeys"/> is a subset of <see cref="KnownKeys"/>, pinned by
25+
/// LeadingKeysAreASubsetOfKnownKeys in the test project, so a key cannot enter the grammar
26+
/// through the leading-key set without first being justified here.
27+
///
28+
/// NT name, AC accession, MT modification type, TA target amino acid, PP position,
29+
/// VV version value, CT compound type, QY quantity, PS peptide sequence, SP species,
30+
/// CN common name, CV vendor, CL cleavable, MH/ML stub mass high/low, SN source name.
31+
///
32+
/// SN is genuinely specified, which is easy to doubt because it is absent from the
33+
/// modification-style key list: it belongs to characteristics[pooled sample], where
34+
/// "SN=sample1;SN=sample2" lists the source names of a pool (specification README.adoc
35+
/// lines 352 and 362, TERMS.tsv "pooled sample"). The corpus agrees -- all 383 leading SN
36+
/// cells sit in characteristics[pooled sample].
37+
/// </summary>
38+
private static readonly HashSet<string> SpecKeys = new(StringComparer.OrdinalIgnoreCase)
39+
{
40+
"NT", "AC", "MT", "TA", "PP", "VV", "CT", "QY",
41+
"PS", "SP", "CN", "CV", "CL", "MH", "ML", "SN"
42+
};
43+
44+
/// <summary>
45+
/// Keys the corpus uses that the specification does not define, kept because they are
46+
/// widespread and unambiguous in context -- MM monoisotopic mass (26,828 occurrences),
47+
/// CS cleavage site (3,810), CF chemical formula (2,602), all inside modification and
48+
/// cleavage-agent cells.
49+
///
50+
/// Separated from <see cref="SpecKeys"/> rather than merged so that the distinction between
51+
/// "the format says so" and "the community writes it anyway" stays visible. Anything added
52+
/// here is a tolerance, and tolerances belong on the reading side only -- never in authored
53+
/// output.
54+
/// </summary>
55+
private static readonly HashSet<string> ObservedNonSpecKeys = new(StringComparer.OrdinalIgnoreCase)
56+
{
57+
"MM", "CS", "CF"
58+
};
59+
60+
/// <summary>
61+
/// Every key this reader will interpret: the specified ones plus the tolerated ones.
62+
///
63+
/// "PMID" was removed. It is not a key in any position -- the specification uses PMID only
64+
/// as a VALUE format for reference columns ("URL, DOI, or PMID"), and the 3 corpus
65+
/// occurrences are of that kind. "N" was removed for the reasons given on
66+
/// <see cref="KnownLeadingKeys"/>. "PS" was added: it is specified (spiked-compound cells,
67+
/// "CT=peptide;PS=PEPTIDESEQ;QY=10 fmol") though no corpus file uses it yet.
68+
/// </summary>
69+
internal static readonly HashSet<string> KnownKeys =
70+
new(SpecKeys.Concat(ObservedNonSpecKeys), StringComparer.OrdinalIgnoreCase);
71+
72+
/// <summary>
73+
/// Keys that actually appear FIRST in a corpus cell: NT (1,459,508 cells in all 1,236
74+
/// files), AC (190,369 in 323), CT (389 in 6) and SN (383 in 4). Everything else in
75+
/// <see cref="KnownKeys"/> only ever appears after a semicolon.
76+
///
77+
/// "A" was removed from the set entirely. It is a single character, it never leads a real
78+
/// cell, and as a leading key it can only ever produce false positives -- any free-text
79+
/// value beginning "A=" would have been decoded as a controlled-vocabulary term.
80+
///
81+
/// "N" was removed for exactly the same reasons, and the specification settles it: the key
82+
/// grammar has no "N", and NT is the only name key. It leads 45 corpus cells -- 27 in
83+
/// PXD039582 and 18 in PXD039585, every one of them "N=Orbitrap" in
84+
/// comment[ms2 analyzer type] -- which is a typo for NT= in two files by one submitter, not
85+
/// a spelling of the grammar. Accepting it decoded drift AS the grammar and so hid it from
86+
/// the one component whose job is to report it; SdrfDriftLint now sees those cells as the
87+
/// free text they are. An earlier revision of this comment listed the leading keys as
88+
/// "NT, AC, CT, SN, and N" with a count against every key except N, which is what made the
89+
/// reviewer look.
90+
/// </summary>
91+
internal static readonly HashSet<string> KnownLeadingKeys = new(StringComparer.OrdinalIgnoreCase)
92+
{
93+
"NT", "AC", "CT", "SN", "MT", "TA", "PP", "CS", "CF", "CL", "CV", "MM", "VV"
94+
};
95+
96+
/// <summary>
97+
/// Formats a controlled-vocabulary term as an SDRF cell: "NT=Oxidation;AC=UNIMOD:35".
98+
///
99+
/// NT first, then AC, then any extra keys in the order given. This is a specification MUST,
100+
/// not merely a convention read off the corpus: "The key order MUST be NT (name) first,
101+
/// followed by AC (accession), then any additional keys" (README.adoc:261). The corpus
102+
/// agrees overwhelmingly anyway -- NT leads 1,459,508 cells, AC 190,369.
103+
///
104+
/// The accession is written EXACTLY as supplied. This deliberately does not "helpfully"
105+
/// upper-case or add a missing prefix: the corpus is full of drift the caller should not be
106+
/// able to launder through here -- bare "4" for UNIMOD:4 in 39 documents, "Unimod:35" in 22,
107+
/// bare "1001251" for Trypsin in 39. Authored terms are meant to come from the pinned
108+
/// vocabulary already correct; silently repairing a wrong one here would hide the bug and
109+
/// make the drift lint's job impossible.
110+
/// </summary>
111+
/// <param name="term">The term. Name and Accession may not both be empty.</param>
112+
/// <param name="extras">
113+
/// Additional key=value pairs in document order, e.g. ("TA","M"), ("MT","Variable").
114+
/// Keys are emitted as given; a null or empty value is skipped.
115+
/// </param>
116+
public static string ToCell(CvParam term, params (string Key, string Value)[] extras)
117+
{
118+
if (term is null) throw new ArgumentNullException(nameof(term));
119+
if (string.IsNullOrEmpty(term.Name) && string.IsNullOrEmpty(term.Accession))
120+
throw new ArgumentException(
121+
"A controlled-vocabulary term needs at least a name or an accession.", nameof(term));
122+
123+
var parts = new List<string>(2 + (extras?.Length ?? 0));
124+
if (!string.IsNullOrEmpty(term.Name)) parts.Add("NT=" + term.Name);
125+
if (!string.IsNullOrEmpty(term.Accession)) parts.Add("AC=" + term.Accession);
126+
127+
foreach (var (key, value) in extras ?? Array.Empty<(string, string)>())
128+
{
129+
if (string.IsNullOrEmpty(key) || string.IsNullOrEmpty(value)) continue;
130+
parts.Add(key + "=" + value);
131+
}
132+
133+
string cell = string.Join(";", parts);
134+
135+
// A separator inside a value would silently invent or merge keys on read. There is no
136+
// escape mechanism, so the only honest response is to refuse.
137+
if (cell.IndexOf('\t') >= 0 || cell.IndexOf('\n') >= 0 || cell.IndexOf('\r') >= 0)
138+
throw new ArgumentException(
139+
$"An SDRF cell cannot contain a tab or newline; the format defines no escape " +
140+
$"mechanism. Offending term: '{cell}'.", nameof(term));
141+
142+
return cell;
143+
}
144+
145+
/// <summary>
146+
/// True when the cell is written in the key=value grammar rather than being free text.
147+
/// </summary>
148+
public static bool IsTerm(string cell)
149+
{
150+
if (string.IsNullOrEmpty(cell)) return false;
151+
int equals = cell.IndexOf('=');
152+
if (equals <= 0) return false;
153+
int semicolon = cell.IndexOf(';');
154+
if (semicolon >= 0 && semicolon < equals) return false;
155+
156+
// Case-insensitive, matching ParseKeyValues. These disagreed before: the leading key was
157+
// compared Ordinal while every later key was compared OrdinalIgnoreCase, so
158+
// "NT=Ox;ac=UNIMOD:35" parsed but "nt=Ox;AC=UNIMOD:35" was silently free text.
159+
return KnownLeadingKeys.Contains(cell.Substring(0, equals).Trim());
160+
}
161+
162+
/// <summary>
163+
/// Splits a cell into its key=value pairs, preserving order and duplicates-last-wins.
164+
/// Returns an empty dictionary for free text. Keys are upper-cased for lookup; values keep
165+
/// their original casing and inner whitespace.
166+
/// </summary>
167+
public static IReadOnlyDictionary<string, string> ParseKeyValues(string cell)
168+
{
169+
var pairs = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
170+
if (!IsTerm(cell)) return pairs;
171+
172+
foreach (var part in cell.Split(';'))
173+
{
174+
int equals = part.IndexOf('=');
175+
if (equals <= 0) continue;
176+
string key = part.Substring(0, equals).Trim();
177+
if (key.Length == 0) continue;
178+
pairs[key] = part.Substring(equals + 1).Trim();
179+
}
180+
return pairs;
181+
}
182+
183+
/// <summary>
184+
/// Reads the cell as a controlled-vocabulary term. False for free text.
185+
///
186+
/// The CV label is derived from the accession prefix ("MS:1001911" -> "MS"), because SDRF
187+
/// cells do not carry one separately the way an mzML cvParam does.
188+
/// </summary>
189+
public static bool TryParseTerm(string cell, [MaybeNullWhen(false)] out CvParam term)
190+
{
191+
term = null;
192+
var pairs = ParseKeyValues(cell);
193+
if (pairs.Count == 0) return false;
194+
195+
// NT is the ONLY name key. An earlier revision also accepted "N" as an alternative
196+
// spelling, on the strength of 45 corpus cells that write "N=Orbitrap"; the
197+
// specification defines no such key, so that was decoding one submitter's typo as
198+
// grammar. See KnownLeadingKeys.
199+
pairs.TryGetValue("NT", out string? name);
200+
pairs.TryGetValue("AC", out string? accession);
201+
202+
// A cell in the key=value grammar with neither NT nor AC is NOT a controlled-vocabulary
203+
// term, and must not be promoted to one.
204+
//
205+
// An earlier revision fell back to CN/SN/CT/SP here, to stop such cells being reported
206+
// as free text. That was worse than the problem. ParseKeyValues is last-wins, and the
207+
// dominant real case is characteristics[pooled sample], where a cell carries up to 45
208+
// repeated SN= keys ("SN=OSL.53E;SN=OSL.567;..."): the fallback produced a Name that was
209+
// one arbitrary member of a list, with an empty accession. Worse, callers branch on this
210+
// method, so promoted cells left the free-text index without entering the accession
211+
// index -- 538 cells became invisible to every kind of drift analysis at once.
212+
//
213+
// Returning false routes them to the free-text side, where their values are still
214+
// compared. IsTerm still reports true, so a caller that wants the raw pairs can ask
215+
// ParseKeyValues for them.
216+
if (string.IsNullOrEmpty(name) && string.IsNullOrEmpty(accession)) return false;
217+
218+
accession ??= "";
219+
name ??= "";
220+
int colon = accession.IndexOf(':');
221+
string cvLabel = colon > 0 ? accession.Substring(0, colon) : "";
222+
223+
term = new CvParam(cvLabel, accession, name, "");
224+
return true;
225+
}
226+
}
227+
}

0 commit comments

Comments
 (0)