diff --git a/+exploreFNIRS/processMethods.m b/+exploreFNIRS/processMethods.m index 1b423331..6ef873ca 100644 --- a/+exploreFNIRS/processMethods.m +++ b/+exploreFNIRS/processMethods.m @@ -15,7 +15,7 @@ function processMethods(rawMethodStr,oxyMethodStr) % oxyMethodStr - Name of the oxy processing method % % Example: -% exploreFNIRS.processMethods('x5_TDDR', 'takizawa_easy'); +% exploreFNIRS.processMethods('OD_TDDR', 'takizawa_easy'); % exploreFNIRS.processMethods([], 'None'); % oxy-only reprocessing % % See also: processFNIRS2, pf2.methods.raw.list, pf2.methods.oxy.list, diff --git a/+pf2/+GUI/configureOxyMethods.m b/+pf2/+GUI/configureOxyMethods.m deleted file mode 100644 index 85c25951..00000000 --- a/+pf2/+GUI/configureOxyMethods.m +++ /dev/null @@ -1,32 +0,0 @@ -function varargout=configureOxyMethods(varargin) -% CONFIGUREOXYMETHODS Open the GUI to configure oxy (Stage 3) methods -% -% Wrapper that launches the processFNIRS2 method-configuration GUI scoped to -% the oxy (hemoglobin, Stage 3) processing stage. All arguments and outputs -% are forwarded to processFNIRS2_configureMethods with the 'oxy' stage -% pre-selected. -% -% Syntax: -% pf2.GUI.configureOxyMethods() -% out = pf2.GUI.configureOxyMethods(...) -% -% Inputs: -% varargin - Any arguments accepted by processFNIRS2_configureMethods -% (after the implicit 'oxy' stage argument). -% -% Outputs: -% varargout - Whatever processFNIRS2_configureMethods returns when an -% output is requested (e.g. the app handle). -% -% Example: -% % Open the oxy method configuration GUI -% pf2.GUI.configureOxyMethods(); -% -% See also: pf2.GUI.configureRawMethods, pf2.methods.oxy.configureMethods, -% processFNIRS2_configureMethods - -if(nargout>0) - varargout{1:nargout}=processFNIRS2_configureMethods('oxy',varargin{:}); -else - processFNIRS2_configureMethods('oxy',varargin{:}); -end \ No newline at end of file diff --git a/+pf2/+GUI/configureRawMethods.m b/+pf2/+GUI/configureRawMethods.m deleted file mode 100644 index bbbcacbb..00000000 --- a/+pf2/+GUI/configureRawMethods.m +++ /dev/null @@ -1,32 +0,0 @@ -function varargout=configureRawMethods(varargin) -% CONFIGURERAWMETHODS Open the GUI to configure raw (Stage 1) methods -% -% Wrapper that launches the processFNIRS2 method-configuration GUI scoped to -% the raw (light-to-optical-density, Stage 1) processing stage. All arguments -% and outputs are forwarded to processFNIRS2_configureMethods with the 'raw' -% stage pre-selected. -% -% Syntax: -% pf2.GUI.configureRawMethods() -% out = pf2.GUI.configureRawMethods(...) -% -% Inputs: -% varargin - Any arguments accepted by processFNIRS2_configureMethods -% (after the implicit 'raw' stage argument). -% -% Outputs: -% varargout - Whatever processFNIRS2_configureMethods returns when an -% output is requested (e.g. the app handle). -% -% Example: -% % Open the raw method configuration GUI -% pf2.GUI.configureRawMethods(); -% -% See also: pf2.GUI.configureOxyMethods, pf2.methods.raw.configureMethods, -% processFNIRS2_configureMethods - -if(nargout>0) - varargout{1:nargout}=processFNIRS2_configureMethods('raw',varargin{:}); -else - processFNIRS2_configureMethods('raw',varargin{:}); -end \ No newline at end of file diff --git a/+pf2/+GUI/functions.m b/+pf2/+GUI/functions.m deleted file mode 100644 index eeb3e91b..00000000 --- a/+pf2/+GUI/functions.m +++ /dev/null @@ -1,23 +0,0 @@ -function functions() -% FUNCTIONS Display the processing functions available to the method GUI -% -% Placeholder entry point for listing the processing functions that can be -% added to raw/oxy methods through the configuration GUI. Currently prints a -% notice to the console; reserved for the interactive function browser. -% -% Syntax: -% pf2.GUI.functions() -% -% Inputs: -% None -% -% Outputs: -% None. A notice is printed to the command window. -% -% Example: -% % Show the available-functions notice -% pf2.GUI.functions(); -% -% See also: pf2.GUI.functions.add, pf2.GUI.functions.edit, -% pf2.GUI.configureRawMethods, pf2.GUI.configureOxyMethods -disp('This will display currently available functions'); \ No newline at end of file diff --git a/+pf2/+methods/+raw/describeMethod.m b/+pf2/+methods/+raw/describeMethod.m index a81237d6..92fdca03 100644 --- a/+pf2/+methods/+raw/describeMethod.m +++ b/+pf2/+methods/+raw/describeMethod.m @@ -35,7 +35,7 @@ % pf2.methods.raw.describeMethod('x2_lpf_smar'); % % % Get method details as variables for programmatic use -% [desc, funcs] = pf2.methods.raw.describeMethod('x5_TDDR'); +% [desc, funcs] = pf2.methods.raw.describeMethod('OD_TDDR'); % fprintf('Method has %d processing functions\n', length(funcs)); % % % Describe method by index diff --git a/+pf2/+methods/+raw/setMethod.m b/+pf2/+methods/+raw/setMethod.m index 8ceadb9e..a9b07056 100644 --- a/+pf2/+methods/+raw/setMethod.m +++ b/+pf2/+methods/+raw/setMethod.m @@ -30,7 +30,7 @@ function setMethod(raw_method, ctx) % % % Typical workflow % pf2.methods.raw.list(); % View available methods -% pf2.methods.raw.setMethod('x5_TDDR'); % Select TDDR method +% pf2.methods.raw.setMethod('OD_TDDR'); % Select TDDR method % data = processFNIRS2(rawData); % Process with selected method % % See also: pf2.methods.raw.list, pf2.methods.raw.describeMethod, diff --git a/+pf2/+methods/+seeds/+raw/OD_TDDR_lpf.m b/+pf2/+methods/+seeds/+raw/OD_TDDR_lpf.m deleted file mode 100644 index e5c932eb..00000000 --- a/+pf2/+methods/+seeds/+raw/OD_TDDR_lpf.m +++ /dev/null @@ -1,37 +0,0 @@ -function p = OD_TDDR_lpf() -% OD_TDDR_LPF Factory for the default raw method (OD, TDDR, low-pass) -% -% Builds the shipped "OD_TDDR_lpf" raw (Stage 1) processing method as a -% RawPipeline: a log transform to optical density, Temporal Derivative -% Distribution Repair (TDDR) motion correction, then a 0.1 Hz low-pass -% filter. Used by pf2_initialize and pf2.methods.resetDefaults to (re-)seed -% the default raw methods. Returns a pipeline object to save() or run(). -% -% Reference: -% Fishburn, F. A. et al. (2019). Temporal Derivative Distribution Repair -% (TDDR): A motion correction method for fNIRS. NeuroImage, 184, 171-179. -% DOI: 10.1016/j.neuroimage.2018.09.025 -% -% Syntax: -% p = pf2.methods.seeds.raw.OD_TDDR_lpf() -% -% Inputs: -% None -% -% Outputs: -% p - pf2_base.RawPipeline named 'OD_TDDR_lpf', ready for save() or run() -% -% Example: -% % Build and register the default OD_TDDR_lpf raw method -% p = pf2.methods.seeds.raw.OD_TDDR_lpf(); -% p.save(); -% -% See also: pf2.methods.seeds.raw.OD_TDDR, pf2.methods.resetDefaults, -% pf2_base.RawPipeline, pf2_MotionCorrectTDDR, pf2_lpf - -p = pf2_base.RawPipeline('OD_TDDR_lpf', ... - 'Description', 'Log transform, TDDR motion correction, low-pass filter at 0.1 Hz'); -p = p.add('pf2_Intensity2OD'); -p = p.add('pf2_MotionCorrectTDDR'); -p = p.add('pf2_lpf', 'freq_cut', 0.1); -end diff --git a/+pf2/+methods/Editor.m b/+pf2/+methods/Editor.m index a72fa9d4..c32eba62 100644 --- a/+pf2/+methods/Editor.m +++ b/+pf2/+methods/Editor.m @@ -862,7 +862,7 @@ function onSeedDefaults(app) sel = uiconfirm(app.UIFigure, ... sprintf(['Install repo seed methods for the %s stage? ' ... 'This will overwrite any existing methods with the ' ... - 'same names (OD_TDDR, OD_TDDR_lpf for raw; ' ... + 'same names (OD_TDDR, OD_SMAR for raw; ' ... 'LPF, LPF_ROI for oxy). Other user methods are kept.'], ... app.Stage), ... 'Install seed methods', ... @@ -870,10 +870,10 @@ function onSeedDefaults(app) 'DefaultOption', 1, 'CancelOption', 2); if ~strcmp(sel, 'Install'), return; end try - seeds = pf2.methods.seeds.list(app.Stage); + seeds = pf2_base.methods.seeds.list(app.Stage); for k = 1:numel(seeds) s = seeds(k); - factory = ['pf2.methods.seeds.' s.stage '.' s.name]; + factory = ['pf2_base.methods.seeds.' s.stage '.' s.name]; p = feval(factory); p.save(s.stage, 'Replace', true); end diff --git a/+pf2/+methods/resetDefaults.m b/+pf2/+methods/resetDefaults.m index 8ee1c57d..6d2bdad1 100644 --- a/+pf2/+methods/resetDefaults.m +++ b/+pf2/+methods/resetDefaults.m @@ -12,7 +12,7 @@ function resetDefaults(varargin) % 2. Deletes the prefdir cfg file(s) for the requested stage(s). % 3. Clears cached globals so pf2_initialize re-seeds on next call. % 4. Calls pf2_initialize(), which now applies all repo seeds via -% the factory functions in +pf2/+methods/+seeds/+raw and +oxy. +% the factory functions in +pf2_base/+methods/+seeds/+raw and +oxy. % % Use this when the stored methods cfg has become corrupted, or after % upgrading pf2 and wanting the latest shipped defaults. @@ -24,7 +24,7 @@ function resetDefaults(varargin) % % Reset only the raw stage without a confirmation prompt % pf2.methods.resetDefaults('Stage', 'raw', 'Confirm', false); % -% See also: pf2.methods.seeds.list +% See also: pf2_base.methods.seeds.list ip = inputParser; ip.addParameter('Stage', 'both', @(x) ismember(lower(char(x)), {'raw','oxy','both'})); @@ -76,12 +76,12 @@ function resetDefaults(varargin) pf2_base.pf2_initialize(); % Now apply the seeds. -seeds = pf2.methods.seeds.list(); +seeds = pf2_base.methods.seeds.list(); for k = 1:numel(seeds) s = seeds(k); if strcmp(stage, 'raw') && ~strcmp(s.stage, 'raw'), continue; end if strcmp(stage, 'oxy') && ~strcmp(s.stage, 'oxy'), continue; end - factory = ['pf2.methods.seeds.' s.stage '.' s.name]; + factory = ['pf2_base.methods.seeds.' s.stage '.' s.name]; try p = feval(factory); p.save(s.stage, 'Replace', true); diff --git a/+pf2/+process/processRaw.m b/+pf2/+process/processRaw.m index 9d83575c..f64e5135 100644 --- a/+pf2/+process/processRaw.m +++ b/+pf2/+process/processRaw.m @@ -56,7 +56,7 @@ % % % Compare different raw methods before committing to oxy processing % result1 = pf2.process.processRaw(data, 'x2_lpf_smar'); -% result2 = pf2.process.processRaw(data, 'x5_TDDR'); +% result2 = pf2.process.processRaw(data, 'OD_TDDR'); % % Notes: % - This function sets 'SkipOxy' to true internally diff --git a/+pf2/ProcessingContext.m b/+pf2/ProcessingContext.m index f4d7cd58..94fa85c5 100644 --- a/+pf2/ProcessingContext.m +++ b/+pf2/ProcessingContext.m @@ -19,7 +19,7 @@ % ctx = pf2.ProcessingContext(Name, Value, ...) % % Inputs (Name-Value pairs; all optional): - % 'RawMethod' - Name of a raw-stage method (e.g. 'x5_TDDR') + % 'RawMethod' - Name of a raw-stage method (e.g. 'OD_TDDR') % 'OxyMethod' - Name of an oxy-stage method (e.g. 'takizawa_easy') % 'DPFmode' - 'None' | 'Fixed' | 'Calc' % 'FixedDPF' - Fixed DPF value (used when DPFmode is 'Fixed') @@ -36,18 +36,12 @@ % % Examples: % % Configure once, in one call, without touching globals - % ctx = pf2.ProcessingContext('RawMethod','x5_TDDR', ... + % ctx = pf2.ProcessingContext('RawMethod','OD_TDDR', ... % 'DPFmode','Calc', 'SubjectAge',25, 'blLength',10); % out = processFNIRS2(data, 'Context', ctx); % % ...or let the context be the receiver: % out = ctx.process(data); % - % % Save an analysis "recipe" and reload it in a fresh session - % recipe = ctx.toStruct(); - % save('study_protocol.mat', '-struct', 'recipe'); - % s = load('study_protocol.mat'); - % ctx = pf2.ProcessingContext.fromRecipe(s); % fully usable again - % % % Parallel, one independent context per worker (note copy(), not '='): % parfor i = 1:numel(allData) % c = ctx.copy(); @@ -156,23 +150,4 @@ function applySetting(obj, key, val) end end - methods (Static) - function obj = fromRecipe(s) - % FROMRECIPE Rebuild a usable context from a saved settings struct - % - % Delegates deserialization to the inherited (base) fromStruct -- - % the single source of truth for which fields round-trip, and which - % also reloads the method libraries -- then returns the settings as - % a public pf2.ProcessingContext. - % - % Syntax: - % ctx = pf2.ProcessingContext.fromRecipe(s) - - base = pf2_base.ProcessingContext.fromStruct(s); - obj = pf2.ProcessingContext(); - for p = properties(base)' - obj.(p{1}) = base.(p{1}); - end - end - end end diff --git a/+pf2/+import/+sampleData/addDemoPipelines.m b/+pf2_base/+examples/addDemoPipelines.m similarity index 92% rename from +pf2/+import/+sampleData/addDemoPipelines.m rename to +pf2_base/+examples/addDemoPipelines.m index 4dd96241..65b044c1 100644 --- a/+pf2/+import/+sampleData/addDemoPipelines.m +++ b/+pf2_base/+examples/addDemoPipelines.m @@ -17,9 +17,9 @@ % 2. pf2_build_pca_ROI - Build ROI signals via PCA 1st component (if ROIs defined) % % Syntax: -% pf2.import.sampleData.addDemoPipelines() -% [rawName, oxyName] = pf2.import.sampleData.addDemoPipelines() -% [rawName, oxyName, oxyNamePCA] = pf2.import.sampleData.addDemoPipelines() +% pf2_base.examples.addDemoPipelines() +% [rawName, oxyName] = pf2_base.examples.addDemoPipelines() +% [rawName, oxyName, oxyNamePCA] = pf2_base.examples.addDemoPipelines() % % Outputs: % rawName - Name of the raw method ('demo_TDDR') @@ -27,7 +27,7 @@ % oxyNamePCA - Name of the PCA oxy method ('demo_lpf_pca') % % Example: -% [rawM, oxyM] = pf2.import.sampleData.addDemoPipelines(); +% [rawM, oxyM] = pf2_base.examples.addDemoPipelines(); % ex.settings.rawMethod = rawM; % ex.settings.oxyMethod = oxyM; % ex.aggregate(); diff --git a/+pf2_base/+identity/Contents.m b/+pf2_base/+identity/Contents.m new file mode 100644 index 00000000..84065728 --- /dev/null +++ b/+pf2_base/+identity/Contents.m @@ -0,0 +1,24 @@ +% PF2_BASE.IDENTITY Internal deterministic semantic-identity primitives. +% +% These functions support the future Recipe/RunSpec executor. They are not a +% user-facing authoring API; public clients should eventually construct typed +% artifacts and let the canonical executor apply identity policy. +% +% Semantic identity: +% canonicalBytes - Canonically encode schema-neutral MATLAB values. +% hashProjection - Hash a canonical value in an explicit artifact domain. +% +% Input identity and fail-closed verification: +% importedContentFingerprint - Hash normalized computation-visible input. +% preflightImport - Verify source/imported identity before Layer-1. +% sha256File - Stream and hash exact file bytes. +% sourceFingerprint - Hash a portable multi-file source manifest. +% verifyExpected - Enforce an expected digest without warning fallback. +% +% Phase-0 proof carrier (not the future human-facing Recipe JSON format): +% decodeJsonTransport - Decode a lossless tagged canonical-value carrier. +% encodeJsonTransport - Encode a lossless tagged canonical-value carrier. +% +% Implementation support: +% normalizeText - Normalize valid text to Unicode NFC and UTF-8. +% sha256Bytes - Compute a lowercase prefixed SHA-256 byte digest. diff --git a/+pf2_base/+identity/canonicalBytes.m b/+pf2_base/+identity/canonicalBytes.m new file mode 100644 index 00000000..b2ff150c --- /dev/null +++ b/+pf2_base/+identity/canonicalBytes.m @@ -0,0 +1,391 @@ +function bytes = canonicalBytes(value, varargin) +% CANONICALBYTES Encode schema-neutral MATLAB values deterministically. +% +% bytes = pf2_base.identity.canonicalBytes(value) +% bytes = pf2_base.identity.canonicalBytes(value, ... +% 'AllowNonFinite', true) +% +% Produces a row vector of UINT8 bytes using the +% "pf2-canonical-binary-v1" profile. The encoding is independent of struct +% field insertion order and host byte order. It preserves MATLAB value type +% and visible array dimensions, visits array elements in column-major order, +% and normalizes all text to Unicode NFC before UTF-8 encoding. +% +% Supported values may be nested arbitrarily and are limited to: +% - scalar or array structs; +% - cell arrays; +% - real, full arrays of the built-in numeric classes; +% - logical arrays; +% - character vectors; and +% - string arrays (including missing string elements). A character vector +% and an equivalent nonmissing scalar string share one text identity. +% +% Name-value options: +% AllowNonFinite - Scalar logical, default false. When false, NaN and +% positive/negative Inf are rejected. When true, each is +% encoded with one profile-defined IEEE bit pattern. +% +% Floating-point negative zero is always normalized to positive zero. +% Sparse and complex arrays, character matrices, tables, other objects, and +% function handles are rejected. This deliberately small value language is +% the common semantic carrier for durable MAT and JSON representations; it +% is not a serialization of arbitrary MATLAB runtime state. +% +% Binary profile (version 1): +% document := ASCII "PF2-CANONICAL", NUL, version-byte, framed-value +% framed-value := type-tag, uint64(payload-byte-count), payload +% +% All integer metadata and numeric payloads use big-endian/network byte +% order. Array payloads begin with a uint64 dimension count followed by +% uint64 dimension lengths. Struct field names are sorted by normalized +% UTF-8 bytes; values are then emitted element-first and field-second. +% +% See also: pf2_base.identity.hashProjection, +% pf2_base.identity.sha256Bytes + +allowNonFinite = parseOptions(varargin{:}); + +profileHeader = [uint8('PF2-CANONICAL'), uint8(0), uint8(1)]; +bytes = [profileHeader, encodeValue(value, allowNonFinite)]; + +end + + +function allowNonFinite = parseOptions(varargin) +allowNonFinite = false; + +if mod(numel(varargin), 2) ~= 0 + error('pf2:identity:invalidArguments', ... + 'Options must be supplied as name-value pairs.'); +end + +seen = false; +for i = 1:2:numel(varargin) + name = optionName(varargin{i}); + if ~strcmpi(name, 'AllowNonFinite') + error('pf2:identity:unknownOption', ... + 'Unknown option "%s".', name); + end + if seen + error('pf2:identity:duplicateOption', ... + 'Option "AllowNonFinite" may be supplied only once.'); + end + + candidate = varargin{i + 1}; + if ~islogical(candidate) || ~isscalar(candidate) + error('pf2:identity:invalidAllowNonFinite', ... + 'AllowNonFinite must be a scalar logical.'); + end + allowNonFinite = candidate; + seen = true; +end +end + + +function name = optionName(value) +if ischar(value) && isrow(value) + name = value; +elseif isstring(value) && isscalar(value) && ~ismissing(value) + name = char(value); +else + error('pf2:identity:invalidOptionName', ... + 'Option names must be character vectors or nonmissing string scalars.'); +end +end + + +function bytes = encodeValue(value, allowNonFinite) +if issparse(value) + error('pf2:identity:unsupportedType', ... + 'Sparse arrays are not supported by the canonical value language.'); +end + +if isa(value, 'function_handle') + error('pf2:identity:unsupportedType', ... + 'Function handles are not supported by the canonical value language.'); +end + +if isstruct(value) + bytes = encodeStruct(value, allowNonFinite); +elseif iscell(value) + bytes = encodeCell(value, allowNonFinite); +elseif islogical(value) + bytes = frame(uint8(3), ... + [encodeDimensions(size(value)), uint8(value(:).')]); +elseif ischar(value) + bytes = encodeChar(value); +elseif isstring(value) + bytes = encodeString(value); +elseif isnumeric(value) + if ~isreal(value) + error('pf2:identity:unsupportedType', ... + 'Complex numeric arrays are not supported by the canonical value language.'); + end + bytes = encodeNumeric(value, allowNonFinite); +elseif isa(value, 'table') || isa(value, 'timetable') + error('pf2:identity:unsupportedType', ... + 'Tables and timetables are not supported by the canonical value language.'); +elseif isobject(value) + error('pf2:identity:unsupportedType', ... + 'Objects of class "%s" are not supported by the canonical value language.', ... + class(value)); +else + error('pf2:identity:unsupportedType', ... + 'Values of class "%s" are not supported by the canonical value language.', ... + class(value)); +end +end + + +function bytes = encodeStruct(value, allowNonFinite) +% Tag 1: dimensions, sorted field-name table, then framed field values. +originalNames = fieldnames(value); +[originalNames, normalizedNames, normalizedUtf8] = ... + sortFieldNames(originalNames); + +parts = cell(1, 2 + numel(normalizedNames) + ... + numel(value) * numel(normalizedNames)); +cursor = 1; +parts{cursor} = encodeDimensions(size(value)); +cursor = cursor + 1; +parts{cursor} = encodeUint64(numel(normalizedNames)); +cursor = cursor + 1; + +for fieldIndex = 1:numel(normalizedNames) + parts{cursor} = lengthDelimited(normalizedUtf8{fieldIndex}); + cursor = cursor + 1; +end + +for elementIndex = 1:numel(value) + for fieldIndex = 1:numel(originalNames) + fieldValue = value(elementIndex).(originalNames{fieldIndex}); + parts{cursor} = encodeValue(fieldValue, allowNonFinite); + cursor = cursor + 1; + end +end + +payload = concatenate(parts); +bytes = frame(uint8(1), payload); +end + + +function [originalNames, normalizedNames, normalizedUtf8] = ... + sortFieldNames(originalNames) +fieldCount = numel(originalNames); +normalizedNames = cell(fieldCount, 1); +normalizedUtf8 = cell(fieldCount, 1); +sortKeys = cell(fieldCount, 1); + +for i = 1:fieldCount + [normalizedNames{i}, normalizedUtf8{i}] = ... + pf2_base.identity.normalizeText(originalNames{i}); + sortKeys{i} = lower(reshape(dec2hex(normalizedUtf8{i}, 2).', 1, [])); +end + +for i = 1:fieldCount + for j = i + 1:fieldCount + if strcmp(normalizedNames{i}, normalizedNames{j}) + error('pf2:identity:invalidText', ... + ['Struct fields "%s" and "%s" have the same Unicode NFC ' ... + 'spelling.'], originalNames{i}, originalNames{j}); + end + end +end + +[~, order] = sort(sortKeys); +originalNames = originalNames(order); +normalizedNames = normalizedNames(order); +normalizedUtf8 = normalizedUtf8(order); +end + + +function bytes = encodeCell(value, allowNonFinite) +% Tag 2: dimensions followed by framed values in column-major order. +parts = cell(1, numel(value) + 1); +parts{1} = encodeDimensions(size(value)); +for i = 1:numel(value) + parts{i + 1} = encodeValue(value{i}, allowNonFinite); +end +bytes = frame(uint8(2), concatenate(parts)); +end + + +function bytes = encodeChar(value) +% Tag 4: one schema-level text scalar, independent of MATLAB char shape. +if ndims(value) ~= 2 || ... + ~(size(value, 1) == 1 || size(value, 2) == 1 || ... + isequal(size(value), [0 0])) + error('pf2:identity:unsupportedType', ... + 'Character inputs must be vectors; character matrices are unsupported.'); +end + +bytes = encodeTextScalar(value(:).'); +end + + +function bytes = encodeString(value) +% A nonmissing scalar string is the same schema text scalar as char. +if isscalar(value) && ~ismissing(value) + bytes = encodeTextScalar(char(value)); + return; +end + +% Tag 5: dimensions and column-major missing/present string-array elements. +parts = cell(1, numel(value) + 1); +parts{1} = encodeDimensions(size(value)); +for i = 1:numel(value) + if ismissing(value(i)) + parts{i + 1} = uint8(0); + else + [~, utf8] = pf2_base.identity.normalizeText(char(value(i))); + parts{i + 1} = [uint8(1), lengthDelimited(utf8)]; + end +end +bytes = frame(uint8(5), concatenate(parts)); +end + + +function bytes = encodeTextScalar(value) +[~, utf8] = pf2_base.identity.normalizeText(value); +bytes = frame(uint8(4), lengthDelimited(utf8)); +end + + +function bytes = encodeNumeric(value, allowNonFinite) +[tag, bytesPerElement] = numericProfile(class(value)); +linear = value(:); + +if isfloat(linear) + nonFinite = isnan(linear) | isinf(linear); + if any(nonFinite) && ~allowNonFinite + error('pf2:identity:nonFinite', ... + ['NaN and Inf require the explicit name-value option ' ... + '''AllowNonFinite'', true.']); + end + + % Assignment of a typed +0 removes the IEEE sign bit from both zero forms. + linear(linear == 0) = cast(0, class(linear)); +end + +if isLittleEndian() + networkValues = swapbytes(linear); +else + networkValues = linear; +end +raw = reshape(typecast(networkValues, 'uint8'), bytesPerElement, []); + +if isfloat(linear) && any(isnan(linear) | isinf(linear)) + raw = normalizeNonFiniteBytes(raw, linear); +end + +payload = [encodeDimensions(size(value)), reshape(raw, 1, [])]; +bytes = frame(tag, payload); +end + + +function [tag, bytesPerElement] = numericProfile(className) +switch className + case 'double' + tag = uint8(16); bytesPerElement = 8; + case 'single' + tag = uint8(17); bytesPerElement = 4; + case 'int8' + tag = uint8(18); bytesPerElement = 1; + case 'uint8' + tag = uint8(19); bytesPerElement = 1; + case 'int16' + tag = uint8(20); bytesPerElement = 2; + case 'uint16' + tag = uint8(21); bytesPerElement = 2; + case 'int32' + tag = uint8(22); bytesPerElement = 4; + case 'uint32' + tag = uint8(23); bytesPerElement = 4; + case 'int64' + tag = uint8(24); bytesPerElement = 8; + case 'uint64' + tag = uint8(25); bytesPerElement = 8; + otherwise + error('pf2:identity:unsupportedType', ... + 'Numeric class "%s" is not a supported built-in numeric type.', ... + className); +end +end + + +function raw = normalizeNonFiniteBytes(raw, values) +% Fixed quiet-NaN and infinity encodings from IEEE 754, in network order. +if isa(values, 'double') + nanBytes = uint8([127 248 0 0 0 0 0 0]); + positiveInfBytes = uint8([127 240 0 0 0 0 0 0]); + negativeInfBytes = uint8([255 240 0 0 0 0 0 0]); +else + nanBytes = uint8([127 192 0 0]); + positiveInfBytes = uint8([127 128 0 0]); + negativeInfBytes = uint8([255 128 0 0]); +end + +for i = 1:numel(values) + if isnan(values(i)) + raw(:, i) = nanBytes(:); + elseif isinf(values(i)) && values(i) > 0 + raw(:, i) = positiveInfBytes(:); + elseif isinf(values(i)) + raw(:, i) = negativeInfBytes(:); + end +end +end + + +function bytes = encodeDimensions(dimensions) +parts = cell(1, numel(dimensions) + 1); +parts{1} = encodeUint64(numel(dimensions)); +for i = 1:numel(dimensions) + parts{i + 1} = encodeUint64(dimensions(i)); +end +bytes = concatenate(parts); +end + + +function bytes = frame(tag, payload) +bytes = [tag, encodeUint64(numel(payload)), payload]; +end + + +function bytes = lengthDelimited(payload) +bytes = [encodeUint64(numel(payload)), payload]; +end + + +function bytes = encodeUint64(value) +if ~isscalar(value) || ~isreal(value) || value < 0 || ... + (isfloat(value) && (~isfinite(value) || fix(value) ~= value)) + error('pf2:identity:internalLengthError', ... + 'Canonical lengths and dimensions must be finite nonnegative integers.'); +end + +networkValue = uint64(value); +if isLittleEndian() + networkValue = swapbytes(networkValue); +end +bytes = reshape(typecast(networkValue, 'uint8'), 1, []); +end + + +function tf = isLittleEndian() +persistent littleEndian +if isempty(littleEndian) + [~, ~, endian] = computer; + littleEndian = strcmp(endian, 'L'); +end +tf = littleEndian; +end + + +function bytes = concatenate(parts) +if isempty(parts) + bytes = zeros(1, 0, 'uint8'); +else + bytes = [parts{:}]; +end +end diff --git a/+pf2_base/+identity/decodeJsonTransport.m b/+pf2_base/+identity/decodeJsonTransport.m new file mode 100644 index 00000000..8029471d --- /dev/null +++ b/+pf2_base/+identity/decodeJsonTransport.m @@ -0,0 +1,337 @@ +function value = decodeJsonTransport(json) +%DECODEJSONTRANSPORT Decode the lossless PF2 canonical JSON transport. +% +% value = pf2_base.identity.decodeJsonTransport(json) +% +% JSON must carry the internal `pf2-canonical-json-v1` envelope produced by +% encodeJsonTransport. The decoder validates every node and rejects unknown +% kinds, fields, classes, dimensions, noncanonical hexadecimal, and ignored +% payloads. The result belongs to canonicalBytes' schema-neutral value language. +% +% This function restores semantic values, not artifact classes. Recipe, +% RunSpec, and RunRecord schemas remain responsible for validating their own +% fields and versions after transport decoding. This proof carrier is not the +% future public, human-readable Recipe JSON format. +% +% See also: pf2_base.identity.encodeJsonTransport, +% pf2_base.identity.canonicalBytes + + json = scalarText(json); + try + envelope = jsondecode(json); + validateEnvelope(envelope); + value = decodeNode(envelope.root); + catch cause + if strcmp(cause.identifier, 'pf2:identity:invalidJsonTransport') + rethrow(cause); + end + exception = MException('pf2:identity:invalidJsonTransport', ... + 'The input is not a valid pf2-canonical-json-v1 transport.'); + exception = addCause(exception, cause); + throw(exception); + end +end + +function validateEnvelope(envelope) + if ~isstruct(envelope) || ~isscalar(envelope) || ... + ~sameFields(envelope, {'format', 'root'}) + invalid('The JSON root must contain exactly format and root.'); + end + format = scalarText(envelope.format); + if ~strcmp(format, 'pf2-canonical-json-v1') + invalid('Unsupported canonical JSON transport format "%s".', format); + end +end + +function value = decodeNode(node) + required = {'kind', 'className', 'dimensions', 'text', 'dataHex', ... + 'fieldNames', 'children'}; + if ~isstruct(node) || ~isscalar(node) || ~sameFields(node, required) + invalid('Every transport node must contain exactly the v1 node fields.'); + end + + kind = scalarText(node.kind); + switch kind + case 'struct' + requireEmpty(node.className, 'struct className'); + requireEmpty(node.text, 'struct text'); + requireEmpty(node.dataHex, 'struct dataHex'); + dimensions = decodeDimensions(node.dimensions); + fieldNames = decodeTextList(node.fieldNames); + children = decodeChildren(node.children); + expected = arrayElementCount(dimensions) * numel(fieldNames); + if numel(children) ~= expected + invalid('Struct child count does not match dimensions and fields.'); + end + value = buildStruct(dimensions, fieldNames, children); + + case 'cell' + requireEmpty(node.className, 'cell className'); + requireEmpty(node.text, 'cell text'); + requireEmpty(node.dataHex, 'cell dataHex'); + requireEmpty(node.fieldNames, 'cell fieldNames'); + dimensions = decodeDimensions(node.dimensions); + children = decodeChildren(node.children); + if numel(children) ~= arrayElementCount(dimensions) + invalid('Cell child count does not match its dimensions.'); + end + value = cell(dimensions); + for i = 1:numel(children) + value{i} = decodeNode(children{i}); + end + + case 'logical' + requireEmpty(node.className, 'logical className'); + requireEmpty(node.text, 'logical text'); + requireEmpty(node.fieldNames, 'logical fieldNames'); + requireEmpty(node.children, 'logical children'); + dimensions = decodeDimensions(node.dimensions); + bytes = decodeHex(node.dataHex); + if numel(bytes) ~= arrayElementCount(dimensions) || ... + any(bytes ~= 0 & bytes ~= 1) + invalid('Logical data must contain one 00 or 01 byte per element.'); + end + value = reshape(logical(bytes), dimensions); + + case 'numeric' + requireEmpty(node.text, 'numeric text'); + requireEmpty(node.fieldNames, 'numeric fieldNames'); + requireEmpty(node.children, 'numeric children'); + className = scalarText(node.className); + bytesPerElement = numericClassSize(className); + dimensions = decodeDimensions(node.dimensions); + bytes = decodeHex(node.dataHex); + expected = arrayElementCount(dimensions) * bytesPerElement; + if numel(bytes) ~= expected + invalid('Numeric payload length does not match class and dimensions.'); + end + value = numericFromNetworkBytes(bytes, className, dimensions); + + case 'text' + requireEmpty(node.className, 'text className'); + requireEmpty(node.dimensions, 'text dimensions'); + requireEmpty(node.dataHex, 'text dataHex'); + requireEmpty(node.fieldNames, 'text fieldNames'); + requireEmpty(node.children, 'text children'); + value = pf2_base.identity.normalizeText(scalarText(node.text)); + + case 'string-array' + requireEmpty(node.className, 'string-array className'); + requireEmpty(node.text, 'string-array text'); + requireEmpty(node.dataHex, 'string-array dataHex'); + requireEmpty(node.fieldNames, 'string-array fieldNames'); + dimensions = decodeDimensions(node.dimensions); + children = decodeChildren(node.children); + if numel(children) ~= arrayElementCount(dimensions) + invalid('String-array child count does not match its dimensions.'); + end + value = repmat("", dimensions); + for i = 1:numel(children) + child = children{i}; + childKind = nodeKind(child); + if strcmp(childKind, 'missing-string') + validateMissingString(child); + value(i) = string(missing); + elseif strcmp(childKind, 'text') + decoded = decodeNode(child); + value(i) = string(decoded); + else + invalid('String-array children must be text or missing-string nodes.'); + end + end + + case 'missing-string' + invalid('A missing-string node is valid only inside a string array.'); + + otherwise + invalid('Unknown canonical JSON node kind "%s".', kind); + end +end + +function value = buildStruct(dimensions, fieldNames, children) + for i = 1:numel(fieldNames) + normalized = pf2_base.identity.normalizeText(fieldNames{i}); + if ~strcmp(normalized, fieldNames{i}) + invalid('Struct field names must already be Unicode NFC.'); + end + for j = 1:i-1 + if strcmp(fieldNames{i}, fieldNames{j}) + invalid('Struct field names must be unique.'); + end + end + end + + try + if isempty(fieldNames) + template = struct(); + else + template = cell2struct(cell(1, numel(fieldNames)), ... + fieldNames, 2); + end + value = repmat(template, dimensions); + cursor = 1; + for elementIndex = 1:numel(value) + for fieldIndex = 1:numel(fieldNames) + value(elementIndex).(fieldNames{fieldIndex}) = ... + decodeNode(children{cursor}); + cursor = cursor + 1; + end + end + catch cause + exception = MException('pf2:identity:invalidJsonTransport', ... + 'The transport contains an invalid MATLAB struct representation.'); + exception = addCause(exception, cause); + throw(exception); + end +end + +function validateMissingString(node) + if ~isstruct(node) || ~isscalar(node) || ... + ~sameFields(node, {'kind', 'className', 'dimensions', 'text', ... + 'dataHex', 'fieldNames', 'children'}) + invalid('Invalid missing-string node.'); + end + requireEmpty(node.className, 'missing-string className'); + requireEmpty(node.dimensions, 'missing-string dimensions'); + requireEmpty(node.text, 'missing-string text'); + requireEmpty(node.dataHex, 'missing-string dataHex'); + requireEmpty(node.fieldNames, 'missing-string fieldNames'); + requireEmpty(node.children, 'missing-string children'); +end + +function kind = nodeKind(node) + if ~isstruct(node) || ~isscalar(node) || ~isfield(node, 'kind') + invalid('A child is not a valid transport node.'); + end + kind = scalarText(node.kind); +end + +function dimensions = decodeDimensions(value) + if ~isnumeric(value) || ~isreal(value) || issparse(value) || ... + ~isvector(value) || numel(value) < 2 + invalid('Dimensions must be a JSON numeric vector with at least two values.'); + end + dimensions = double(value(:).'); + if any(~isfinite(dimensions)) || any(dimensions < 0) || ... + any(fix(dimensions) ~= dimensions) || any(dimensions > flintmax) + invalid('Dimensions must be finite nonnegative integers.'); + end +end + +function count = arrayElementCount(dimensions) + count = prod(dimensions); + if ~isfinite(count) || count > flintmax + invalid('The declared dimensions exceed the supported array size.'); + end +end + +function names = decodeTextList(value) + if isempty(value) + names = cell(1, 0); + elseif ischar(value) && isrow(value) + names = {value}; + elseif isstring(value) && all(~ismissing(value(:))) + names = cellstr(value(:)).'; + elseif iscell(value) + names = cell(1, numel(value)); + for i = 1:numel(value) + names{i} = scalarText(value{i}); + end + else + invalid('fieldNames must be a JSON array of strings.'); + end +end + +function children = decodeChildren(value) + if isempty(value) + children = cell(1, 0); + elseif isstruct(value) + children = num2cell(value(:).'); + elseif iscell(value) + children = value(:).'; + if ~all(cellfun(@(x) isstruct(x) && isscalar(x), children)) + invalid('children must contain only JSON node objects.'); + end + else + invalid('children must be a JSON array of node objects.'); + end +end + +function bytes = decodeHex(value) + value = scalarText(value); + if mod(numel(value), 2) ~= 0 || ... + ~isempty(regexp(value, '[^0-9a-f]', 'once')) + invalid('dataHex must contain an even number of lowercase hexadecimal digits.'); + end + if isempty(value) + bytes = zeros(1, 0, 'uint8'); + else + bytes = uint8(sscanf(value, '%2x').'); + if numel(bytes) * 2 ~= numel(value) + invalid('dataHex could not be decoded completely.'); + end + end +end + +function count = numericClassSize(className) + switch className + case {'double', 'int64', 'uint64'} + count = 8; + case {'single', 'int32', 'uint32'} + count = 4; + case {'int16', 'uint16'} + count = 2; + case {'int8', 'uint8'} + count = 1; + otherwise + invalid('Unsupported numeric class "%s".', className); + end +end + +function value = numericFromNetworkBytes(bytes, className, dimensions) + if isempty(bytes) + linear = zeros(0, 1, className); + else + linear = typecast(bytes, className); + if isLittleEndian() + linear = swapbytes(linear); + end + end + value = reshape(linear, dimensions); +end + +function tf = isLittleEndian() + persistent littleEndian + if isempty(littleEndian) + [~, ~, endian] = computer; + littleEndian = strcmp(endian, 'L'); + end + tf = littleEndian; +end + +function requireEmpty(value, label) + if ~isempty(value) + invalid('The %s field must be empty.', label); + end +end + +function tf = sameFields(value, expected) + actual = sort(fieldnames(value)); + expected = sort(expected(:)); + tf = isequal(actual, expected); +end + +function value = scalarText(value) + if isstring(value) + if ~isscalar(value) || ismissing(value) + invalid('Expected a nonmissing JSON string scalar.'); + end + value = char(value); + elseif ~(ischar(value) && (isrow(value) || isequal(size(value), [0 0]))) + invalid('Expected a character vector or nonmissing string scalar.'); + end +end + +function invalid(message, varargin) + error('pf2:identity:invalidJsonTransport', message, varargin{:}); +end diff --git a/+pf2_base/+identity/encodeJsonTransport.m b/+pf2_base/+identity/encodeJsonTransport.m new file mode 100644 index 00000000..a593cdbe --- /dev/null +++ b/+pf2_base/+identity/encodeJsonTransport.m @@ -0,0 +1,232 @@ +function json = encodeJsonTransport(value, varargin) +%ENCODEJSONTRANSPORT Losslessly encode a canonical PF2 value as JSON. +% +% json = pf2_base.identity.encodeJsonTransport(value) +% json = pf2_base.identity.encodeJsonTransport(value, ... +% 'AllowNonFinite', true, 'PrettyPrint', true) +% +% This is the internal `pf2-canonical-json-v1` proof carrier for the deliberately small +% value language accepted by canonicalBytes. It preserves numeric classes, +% exact finite bits, array dimensions, logicals, cells, struct arrays, missing +% strings, and permitted non-finite values. It is a storage/interchange format; +% its JSON bytes are never an artifact identity. Decode and hash the semantic +% value with hashProjection instead. It is not the future public Recipe JSON +% format: that schema-owned format must remain human-readable and restore types +% and shapes from Recipe metadata rather than exposing this tagged hex carrier. +% +% Numeric payloads use lowercase hexadecimal network-order bytes so JSON number +% coercion cannot lose uint64 values or floating-point precision. Scalar +% nonmissing strings and character vectors share the schema-level text form, +% matching canonicalBytes. +% +% See also: pf2_base.identity.decodeJsonTransport, +% pf2_base.identity.canonicalBytes, +% pf2_base.identity.hashProjection + + options = parseOptions(varargin{:}); + + % Validate the complete tree before constructing any transport output. + pf2_base.identity.canonicalBytes(value, ... + 'AllowNonFinite', options.AllowNonFinite); + + envelope = struct( ... + 'format', 'pf2-canonical-json-v1', ... + 'root', encodeNode(value, options.AllowNonFinite)); + json = jsonencode(envelope, 'PrettyPrint', options.PrettyPrint); +end + +function options = parseOptions(varargin) + parser = inputParser; + parser.FunctionName = 'pf2_base.identity.encodeJsonTransport'; + parser.addParameter('AllowNonFinite', false, ... + @(x) islogical(x) && isscalar(x)); + parser.addParameter('PrettyPrint', false, ... + @(x) islogical(x) && isscalar(x)); + parser.parse(varargin{:}); + options = parser.Results; +end + +function node = encodeNode(value, allowNonFinite) + node = emptyNode(); + if isstruct(value) + node.kind = 'struct'; + node.dimensions = double(size(value)); + [originalNames, normalizedNames] = sortedFieldNames(fieldnames(value)); + node.fieldNames = normalizedNames(:).'; + node.children = cell(1, numel(value) * numel(originalNames)); + cursor = 1; + for elementIndex = 1:numel(value) + for fieldIndex = 1:numel(originalNames) + node.children{cursor} = encodeNode( ... + value(elementIndex).(originalNames{fieldIndex}), ... + allowNonFinite); + cursor = cursor + 1; + end + end + elseif iscell(value) + node.kind = 'cell'; + node.dimensions = double(size(value)); + node.children = cell(1, numel(value)); + for i = 1:numel(value) + node.children{i} = encodeNode(value{i}, allowNonFinite); + end + elseif islogical(value) + node.kind = 'logical'; + node.dimensions = double(size(value)); + node.dataHex = bytesToHex(uint8(value(:).')); + elseif ischar(value) + node.kind = 'text'; + node.text = string(pf2_base.identity.normalizeText(value(:).')); + elseif isstring(value) + if isscalar(value) && ~ismissing(value) + node.kind = 'text'; + node.text = string(pf2_base.identity.normalizeText(char(value))); + else + node.kind = 'string-array'; + node.dimensions = double(size(value)); + node.children = cell(1, numel(value)); + for i = 1:numel(value) + child = emptyNode(); + if ismissing(value(i)) + child.kind = 'missing-string'; + else + child.kind = 'text'; + child.text = string( ... + pf2_base.identity.normalizeText(char(value(i)))); + end + node.children{i} = child; + end + end + elseif isnumeric(value) + node.kind = 'numeric'; + node.className = class(value); + node.dimensions = double(size(value)); + node.dataHex = bytesToHex(numericNetworkBytes(value, allowNonFinite)); + else + % canonicalBytes has already rejected this path. Keep a stable guard in + % case encodeNode is changed independently later. + error('pf2:identity:unsupportedType', ... + 'Values of class "%s" are not supported by the JSON transport.', ... + class(value)); + end +end + +function node = emptyNode() + % Every node has the same fields so nested JSON arrays decode predictably. + node = struct( ... + 'kind', '', ... + 'className', '', ... + 'dimensions', [], ... + 'text', '', ... + 'dataHex', '', ... + 'fieldNames', {cell(1, 0)}, ... + 'children', {cell(1, 0)}); +end + +function [originalNames, normalizedNames] = sortedFieldNames(originalNames) + normalizedNames = cell(size(originalNames)); + utf8 = cell(size(originalNames)); + for i = 1:numel(originalNames) + [normalizedNames{i}, utf8{i}] = ... + pf2_base.identity.normalizeText(originalNames{i}); + end + + order = (1:numel(originalNames)).'; + for i = 2:numel(order) + current = order(i); + j = i - 1; + while j >= 1 && bytesGreater(utf8{order(j)}, utf8{current}) + order(j + 1) = order(j); + j = j - 1; + end + order(j + 1) = current; + end + originalNames = originalNames(order); + normalizedNames = normalizedNames(order); +end + +function tf = bytesGreater(left, right) + sharedLength = min(numel(left), numel(right)); + difference = find(left(1:sharedLength) ~= right(1:sharedLength), 1); + if isempty(difference) + tf = numel(left) > numel(right); + else + tf = left(difference) > right(difference); + end +end + +function bytes = numericNetworkBytes(value, allowNonFinite) + linear = value(:); + if isfloat(linear) + nonFinite = isnan(linear) | isinf(linear); + if any(nonFinite) && ~allowNonFinite + error('pf2:identity:nonFinite', ... + 'NaN and Inf are not allowed in this JSON transport.'); + end + linear(linear == 0) = cast(0, class(linear)); + end + + if isLittleEndian() + networkValues = swapbytes(linear); + else + networkValues = linear; + end + bytes = reshape(typecast(networkValues, 'uint8'), 1, []); + + if isfloat(linear) && any(isnan(linear) | isinf(linear)) + bytesPerElement = bytesPerNumericElement(class(linear)); + matrix = reshape(bytes, bytesPerElement, []); + matrix = normalizeNonFiniteBytes(matrix, linear); + bytes = reshape(matrix, 1, []); + end +end + +function raw = normalizeNonFiniteBytes(raw, values) + if isa(values, 'double') + nanBytes = uint8([127 248 0 0 0 0 0 0]); + positiveInfBytes = uint8([127 240 0 0 0 0 0 0]); + negativeInfBytes = uint8([255 240 0 0 0 0 0 0]); + else + nanBytes = uint8([127 192 0 0]); + positiveInfBytes = uint8([127 128 0 0]); + negativeInfBytes = uint8([255 128 0 0]); + end + for i = 1:numel(values) + if isnan(values(i)) + raw(:, i) = nanBytes(:); + elseif isinf(values(i)) && values(i) > 0 + raw(:, i) = positiveInfBytes(:); + elseif isinf(values(i)) + raw(:, i) = negativeInfBytes(:); + end + end +end + +function count = bytesPerNumericElement(className) + switch className + case {'double', 'int64', 'uint64'} + count = 8; + case {'single', 'int32', 'uint32'} + count = 4; + case {'int16', 'uint16'} + count = 2; + case {'int8', 'uint8'} + count = 1; + otherwise + error('pf2:identity:unsupportedType', ... + 'Unsupported numeric class "%s".', className); + end +end + +function text = bytesToHex(bytes) + text = lower(reshape(dec2hex(bytes, 2).', 1, [])); +end + +function tf = isLittleEndian() + persistent littleEndian + if isempty(littleEndian) + [~, ~, endian] = computer; + littleEndian = strcmp(endian, 'L'); + end + tf = littleEndian; +end diff --git a/+pf2_base/+identity/hashProjection.m b/+pf2_base/+identity/hashProjection.m new file mode 100644 index 00000000..b36ad543 --- /dev/null +++ b/+pf2_base/+identity/hashProjection.m @@ -0,0 +1,189 @@ +function hash = hashProjection(value, varargin) +% HASHPROJECTION Hash a canonical value in an explicit PF2 artifact domain. +% +% hash = pf2_base.identity.hashProjection(value, ... +% 'ArtifactKind', kind, ... +% 'SchemaVersion', version, ... +% 'Projection', projection) +% +% hash = pf2_base.identity.hashProjection(value, ..., ... +% 'AllowNonFinite', true) +% +% Required name-value options: +% ArtifactKind - Nonempty character vector or string scalar containing +% the stable wire name, for example +% "pf2.processing.recipe". +% SchemaVersion - Positive integer schema version. +% Projection - Nonempty character vector or string scalar naming the +% semantic projection, for example +% "scientific-content-v1" or "requested-recipe-v1". +% +% Optional name-value option: +% AllowNonFinite - Scalar logical, default false; forwarded to +% canonicalBytes for both the domain envelope and value. +% +% The digest input is a canonical envelope containing a fixed PF2 marker, +% the canonical-encoding profile, ArtifactKind, SchemaVersion, Projection, +% and the supplied value. Consequently, identical payload values in two +% artifact kinds, schema versions, or projections cannot share a digest by +% accident. The output is "sha256:" plus 64 lowercase hexadecimal digits. +% +% Hash semantic projections rather than MAT-file or JSON-file bytes. Storage +% formatting, struct insertion order, and host endianness then do not define +% durable scientific identity. +% +% See also: pf2_base.identity.canonicalBytes, +% pf2_base.identity.sha256Bytes + +options = parseOptions(varargin{:}); + +envelope = struct(); +envelope.marker = 'PF2-HASH-PROJECTION'; +envelope.encodingProfile = 'pf2-canonical-binary-v1'; +envelope.artifactKind = options.ArtifactKind; +envelope.schemaVersion = options.SchemaVersion; +envelope.projection = options.Projection; +envelope.value = value; + +bytes = pf2_base.identity.canonicalBytes(envelope, ... + 'AllowNonFinite', options.AllowNonFinite); +hash = pf2_base.identity.sha256Bytes(bytes); + +end + + +function options = parseOptions(varargin) +options = struct( ... + 'ArtifactKind', '', ... + 'SchemaVersion', uint64(0), ... + 'Projection', '', ... + 'AllowNonFinite', false); +seen = struct( ... + 'ArtifactKind', false, ... + 'SchemaVersion', false, ... + 'Projection', false, ... + 'AllowNonFinite', false); + +if mod(numel(varargin), 2) ~= 0 + error('pf2:identity:invalidArguments', ... + 'Options must be supplied as name-value pairs.'); +end + +for i = 1:2:numel(varargin) + suppliedName = optionName(varargin{i}); + fieldName = canonicalOptionName(suppliedName); + if seen.(fieldName) + error('pf2:identity:duplicateOption', ... + 'Option "%s" may be supplied only once.', fieldName); + end + + candidate = varargin{i + 1}; + switch fieldName + case 'ArtifactKind' + options.ArtifactKind = artifactKind(candidate); + case 'SchemaVersion' + options.SchemaVersion = schemaVersion(candidate); + case 'Projection' + options.Projection = projectionName(candidate); + case 'AllowNonFinite' + if ~islogical(candidate) || ~isscalar(candidate) + error('pf2:identity:invalidAllowNonFinite', ... + 'AllowNonFinite must be a scalar logical.'); + end + options.AllowNonFinite = candidate; + end + seen.(fieldName) = true; +end + +required = {'ArtifactKind', 'SchemaVersion', 'Projection'}; +for i = 1:numel(required) + if ~seen.(required{i}) + error('pf2:identity:missingOption', ... + 'Required option "%s" was not supplied.', required{i}); + end +end +end + + +function name = optionName(value) +if ischar(value) && isrow(value) + name = value; +elseif isstring(value) && isscalar(value) && ~ismissing(value) + name = char(value); +else + error('pf2:identity:invalidOptionName', ... + 'Option names must be character vectors or nonmissing string scalars.'); +end +end + + +function canonical = canonicalOptionName(name) +known = {'ArtifactKind', 'SchemaVersion', 'Projection', 'AllowNonFinite'}; +match = find(strcmpi(name, known), 1); +if isempty(match) + error('pf2:identity:unknownOption', 'Unknown option "%s".', name); +end +canonical = known{match}; +end + + +function text = artifactKind(value) +text = requiredText(value, ... + 'pf2:identity:invalidArtifactKind', 'ArtifactKind'); +if isempty(regexp(text, ... + '^[a-z][a-z0-9]*(\.[a-z][a-z0-9-]*)+$', 'once')) + error('pf2:identity:invalidArtifactKind', ... + ['ArtifactKind must be a lowercase dotted wire name such as ', ... + '"pf2.processing.recipe".']); +end +end + + +function text = projectionName(value) +text = requiredText(value, ... + 'pf2:identity:invalidProjection', 'Projection'); +if isempty(regexp(text, '^[a-z][a-z0-9]*([.-][a-z0-9]+)*$', 'once')) + error('pf2:identity:invalidProjection', ... + ['Projection must be a lowercase stable name using letters, digits, ', ... + 'dots, and hyphens.']); +end +end + + +function text = requiredText(value, identifier, label) +if ischar(value) && isrow(value) + text = value; +elseif isstring(value) && isscalar(value) && ~ismissing(value) + text = char(value); +else + error(identifier, ... + '%s must be a character vector or nonmissing string scalar.', label); +end + +if isempty(text) + error(identifier, '%s must not be empty.', label); +end +end + + +function version = schemaVersion(value) +validClass = isnumeric(value) && ~islogical(value); +validShape = isscalar(value) && isreal(value) && ~issparse(value); +if ~validClass || ~validShape + error('pf2:identity:invalidSchemaVersion', ... + 'SchemaVersion must be a positive integer scalar.'); +end + +if isfloat(value) + validValue = isfinite(value) && value >= 1 && fix(value) == value && ... + value <= flintmax(class(value)); +else + validValue = value >= 1; +end +if ~validValue + error('pf2:identity:invalidSchemaVersion', ... + 'SchemaVersion must be a positive integer scalar.'); +end + +version = uint64(value); +end diff --git a/+pf2_base/+identity/importedContentFingerprint.m b/+pf2_base/+identity/importedContentFingerprint.m new file mode 100644 index 00000000..384ea89f --- /dev/null +++ b/+pf2_base/+identity/importedContentFingerprint.m @@ -0,0 +1,47 @@ +function fingerprint = importedContentFingerprint(projection, varargin) +%IMPORTEDCONTENTFINGERPRINT Fingerprint a schema-normalized imported input. +% +% fingerprint = pf2_base.identity.importedContentFingerprint(projection) +% computes the Decision-5 imported-content identity for an explicit, +% canonicalizable projection. PROJECTION must already contain every value +% visible to Layer-1 processing, expressed only with the types accepted by +% pf2_base.identity.canonicalBytes. +% +% This function deliberately does not guess which fields of an arbitrary +% fNIRS struct matter. Processing functions may request the complete +% fNIRstruct, so the Recipe/import schema or canonical executor must build +% the projection and either include that complete computation-visible state +% or enforce declared field dependencies before calling this function. +% +% Name-value options: +% 'AllowNonFinite' - Permit canonical NaN/+Inf/-Inf values (default true). +% Imported arrays commonly use NaN masks; the owning +% schema remains responsible for deciding where such +% values are valid. +% +% The returned struct contains: +% .profile - 'pf2.input.imported.v1' +% .algorithm - 'sha256' +% .digest - domain-separated sha256: identity +% +% See also pf2_base.identity.preflightImport, +% pf2_base.identity.hashProjection, +% pf2_base.identity.canonicalBytes + + parser = inputParser; + parser.FunctionName = 'pf2_base.identity.importedContentFingerprint'; + parser.addParameter('AllowNonFinite', true, ... + @(x) islogical(x) && isscalar(x)); + parser.parse(varargin{:}); + + digest = pf2_base.identity.hashProjection(projection, ... + 'ArtifactKind', 'pf2.input.imported', ... + 'SchemaVersion', 1, ... + 'Projection', 'imported-content-v1', ... + 'AllowNonFinite', parser.Results.AllowNonFinite); + + fingerprint = struct( ... + 'profile', 'pf2.input.imported.v1', ... + 'algorithm', 'sha256', ... + 'digest', digest); +end diff --git a/+pf2_base/+identity/normalizeText.m b/+pf2_base/+identity/normalizeText.m new file mode 100644 index 00000000..4edc32fa --- /dev/null +++ b/+pf2_base/+identity/normalizeText.m @@ -0,0 +1,63 @@ +function [normalized, utf8] = normalizeText(value, varargin) +%NORMALIZETEXT Normalize one MATLAB text scalar to Unicode NFC and UTF-8. +% +% normalized = pf2_base.identity.normalizeText(value) +% [normalized, utf8] = pf2_base.identity.normalizeText(value) +% ... = normalizeText(value, 'ErrorIdentifier', identifier) +% +% VALUE must be a character row vector or a nonmissing scalar string. Invalid +% UTF-16, including unpaired surrogate code units, is rejected before Java can +% silently substitute a replacement character. UTF8 is a row vector of UINT8. + + parser = inputParser; + parser.FunctionName = 'pf2_base.identity.normalizeText'; + parser.addParameter('ErrorIdentifier', 'pf2:identity:invalidText', ... + @(x) ischar(x) && isrow(x) && ~isempty(x)); + parser.parse(varargin{:}); + identifier = parser.Results.ErrorIdentifier; + + if isstring(value) + if ~isscalar(value) || ismissing(value) + error(identifier, ... + 'Text must be a character vector or nonmissing scalar string.'); + end + value = char(value); + elseif ~(ischar(value) && (isrow(value) || isequal(size(value), [0 0]))) + error(identifier, ... + 'Text must be a character vector or nonmissing scalar string.'); + end + + validateUtf16(value, identifier); + try + form = javaMethod('valueOf', 'java.text.Normalizer$Form', 'NFC'); + javaText = javaObject('java.lang.String', value); + normalized = char(javaMethod('normalize', 'java.text.Normalizer', ... + javaText, form)); + utf8 = reshape(unicode2native(normalized, 'UTF-8'), 1, []); + catch cause + exception = MException(identifier, ... + 'Text could not be normalized to Unicode NFC UTF-8.'); + exception = addCause(exception, cause); + throw(exception); + end +end + +function validateUtf16(value, identifier) + units = uint16(value); + i = 1; + while i <= numel(units) + if units(i) >= hex2dec('D800') && units(i) <= hex2dec('DBFF') + if i == numel(units) || units(i + 1) < hex2dec('DC00') || ... + units(i + 1) > hex2dec('DFFF') + error(identifier, ... + 'Text contains an unpaired UTF-16 high surrogate.'); + end + i = i + 2; + elseif units(i) >= hex2dec('DC00') && units(i) <= hex2dec('DFFF') + error(identifier, ... + 'Text contains an unpaired UTF-16 low surrogate.'); + else + i = i + 1; + end + end +end diff --git a/+pf2_base/+identity/preflightImport.m b/+pf2_base/+identity/preflightImport.m new file mode 100644 index 00000000..6ae3cf3b --- /dev/null +++ b/+pf2_base/+identity/preflightImport.m @@ -0,0 +1,148 @@ +function [data, identity] = preflightImport(importer, sourcePaths, projectionFcn, varargin) +%PREFLIGHTIMPORT Fail-closed source/imported identity gate before Layer-1. +% +% [data, identity] = pf2_base.identity.preflightImport( ... +% importer, sourcePaths, projectionFcn) +% +% performs the canonical input-identity sequence around an existing importer: +% 1. Fingerprint the explicit source manifest. +% 2. Verify an expected source fingerprint, if supplied. +% 3. Invoke IMPORTER exactly once. +% 4. Re-fingerprint the manifest and fail if any source changed mid-import. +% 5. Convert imported DATA with PROJECTIONFCN. +% 6. Fingerprint and verify the normalized imported projection. +% +% The function returns only after both identity gates pass. A canonical +% executor can therefore call Layer-1 processing after this function without +% offering a warning-only escape hatch. +% +% Inputs: +% importer - Zero-argument function handle returning imported data. +% sourcePaths - File path or cell/string array of every consumed source. +% Pass [] for an in-memory/streaming source with no stable +% byte representation. +% projectionFcn - Function handle mapping DATA to a schema-normalized, +% canonicalizable computation-visible projection. +% +% Name-value options: +% 'LogicalNames' - Portable logical relative names corresponding to +% sourcePaths. Absolute roots never enter identity. +% 'ExpectedSource' - Expected source digest or fingerprint struct. +% 'ExpectedImported' - Expected imported digest or fingerprint struct. +% 'AllowNonFinite' - Permit canonical non-finite values in the imported +% projection (default true). +% +% Output IDENTITY contains the observed source/imported fingerprints and +% validation outcomes. For an in-memory source, identity.source.available is +% false with a structured reason; imported identity remains mandatory. +% +% IMPORTANT: sourcePaths must be the complete set actually consumed by the +% importer, including marker, mask, BIDS, NIRx, device, or other sidecars. +% Omitting a consumed source creates incomplete provenance and is an error in +% the calling import adapter, even though this generic function cannot detect +% an undeclared file access itself. +% +% See also pf2_base.identity.sourceFingerprint, +% pf2_base.identity.importedContentFingerprint, +% pf2_base.identity.verifyExpected + + if ~isa(importer, 'function_handle') + error('pf2:identity:invalidImporter', ... + 'Importer must be a zero-argument function handle.'); + end + if ~isa(projectionFcn, 'function_handle') + error('pf2:identity:invalidProjectionFunction', ... + 'ProjectionFcn must be a function handle.'); + end + + parser = inputParser; + parser.FunctionName = 'pf2_base.identity.preflightImport'; + parser.addParameter('LogicalNames', {}, ... + @(x) ischar(x) || isstring(x) || iscellstr(x)); + parser.addParameter('ExpectedSource', '', ... + @(x) isempty(x) || ischar(x) || (isstring(x) && isscalar(x)) || isstruct(x)); + parser.addParameter('ExpectedImported', '', ... + @(x) isempty(x) || ischar(x) || (isstring(x) && isscalar(x)) || isstruct(x)); + parser.addParameter('AllowNonFinite', true, ... + @(x) islogical(x) && isscalar(x)); + parser.parse(varargin{:}); + + expectedSource = parser.Results.ExpectedSource; + expectedImported = parser.Results.ExpectedImported; + logicalNames = parser.Results.LogicalNames; + + hasStableSource = ~isempty(sourcePaths); + if hasStableSource + before = fingerprintSources(sourcePaths, logicalNames); + sourceValidation = pf2_base.identity.verifyExpected( ... + expectedSource, before, 'Kind', 'source'); + else + if hasExpectation(expectedSource) + error('pf2:identity:sourceUnavailable', ... + ['An expected source fingerprint was supplied, but this input ', ... + 'has no stable byte source to verify.']); + end + before = unavailableSource(); + sourceValidation = struct( ... + 'kind', 'source', ... + 'expected', '', ... + 'observed', '', ... + 'status', 'unavailable'); + end + + % The source expectation is verified before the importer can run. + data = importer(); + + if hasStableSource + after = fingerprintSources(sourcePaths, logicalNames); + pf2_base.identity.verifyExpected(before, after, ... + 'Kind', 'source-stability'); + sourceFingerprint = after; + else + sourceFingerprint = before; + end + + projection = projectionFcn(data); + importedFingerprint = pf2_base.identity.importedContentFingerprint( ... + projection, 'AllowNonFinite', parser.Results.AllowNonFinite); + importedValidation = pf2_base.identity.verifyExpected( ... + expectedImported, importedFingerprint, 'Kind', 'imported-content'); + + identity = struct(); + identity.profile = 'pf2.input.preflight.v1'; + identity.source = sourceFingerprint; + identity.sourceValidation = sourceValidation; + identity.imported = importedFingerprint; + identity.importedValidation = importedValidation; +end + +function tf = hasExpectation(value) + % Match verifyExpected's definition of an omitted expectation. + tf = true; + if (isnumeric(value) || islogical(value) || ischar(value)) && isempty(value) + tf = false; + elseif isstring(value) && (isempty(value) || ... + (isscalar(value) && ~ismissing(value) && strlength(value) == 0)) + tf = false; + end +end + +function fingerprint = fingerprintSources(sourcePaths, logicalNames) + if isempty(logicalNames) + fingerprint = pf2_base.identity.sourceFingerprint(sourcePaths); + else + fingerprint = pf2_base.identity.sourceFingerprint(sourcePaths, ... + 'LogicalNames', logicalNames); + end +end + +function source = unavailableSource() + source = struct(); + source.profile = 'pf2.input.source.v1'; + source.algorithm = 'sha256'; + source.available = false; + source.reason = 'no-stable-byte-source'; + source.digest = ''; + source.entries = struct( ... + 'logicalName', {}, 'byteLength', {}, 'byteDigest', {}); +end diff --git a/+pf2_base/+identity/sha256Bytes.m b/+pf2_base/+identity/sha256Bytes.m new file mode 100644 index 00000000..bde4c68e --- /dev/null +++ b/+pf2_base/+identity/sha256Bytes.m @@ -0,0 +1,41 @@ +function hash = sha256Bytes(bytes) +% SHA256BYTES Compute a lowercase, prefixed SHA-256 digest of bytes. +% +% hash = pf2_base.identity.sha256Bytes(bytes) +% +% Input: +% bytes - Real, full UINT8 vector. Row and column vectors are equivalent. +% An empty UINT8 array computes the standard SHA-256 empty digest. +% +% Output: +% hash - Character row vector of the form "sha256:" followed by exactly +% 64 lowercase hexadecimal digits. +% +% This function hashes the supplied bytes exactly. For durable PF2 artifact +% identity, use hashProjection, which adds artifact-domain separation and +% canonical value encoding. +% +% See also: pf2_base.identity.canonicalBytes, +% pf2_base.identity.hashProjection + +if ~isa(bytes, 'uint8') || issparse(bytes) || ~isreal(bytes) || ... + ndims(bytes) ~= 2 || (~isempty(bytes) && ~isvector(bytes)) + error('pf2:identity:invalidBytes', ... + 'Input must be a real, full UINT8 vector.'); +end + +try + digestEngine = javaMethod('getInstance', ... + 'java.security.MessageDigest', 'SHA-256'); + if ~isempty(bytes) + digestEngine.update(typecast(bytes(:), 'int8')); + end + digest = reshape(typecast(digestEngine.digest(), 'uint8'), 1, []); +catch cause + error('pf2:identity:hashUnavailable', ... + 'The Java SHA-256 implementation is unavailable: %s', cause.message); +end + +hash = ['sha256:', lower(reshape(dec2hex(digest, 2).', 1, []))]; + +end diff --git a/+pf2_base/+identity/sha256File.m b/+pf2_base/+identity/sha256File.m new file mode 100644 index 00000000..983a89eb --- /dev/null +++ b/+pf2_base/+identity/sha256File.m @@ -0,0 +1,94 @@ +function [digest, byteLength] = sha256File(filePath) +% SHA256FILE Compute the SHA-256 digest of a file's exact bytes. +% +% Syntax: +% digest = pf2_base.identity.sha256File(filePath) +% [digest, byteLength] = pf2_base.identity.sha256File(filePath) +% +% Inputs: +% filePath - Explicit path to one existing regular file, supplied as a +% character vector or scalar string. +% +% Outputs: +% digest - Lowercase wire digest, `sha256:<64 hexadecimal digits>`. +% byteLength - Number of bytes read, as a uint64 scalar. +% +% The file is streamed in bounded chunks. Its path, timestamps, permissions, +% and other filesystem metadata are deliberately excluded from the digest. +% Missing, unreadable, and interrupted files fail with stable identifiers in +% the `pf2:identity:*` namespace; no partial digest is returned. +% Errors: `invalidSource`, `fileNotFound`, `fileReadFailed`, and +% `hashUnavailable`. +% +% See also: pf2_base.identity.sourceFingerprint, +% pf2_base.identity.sha256Bytes + +filePath = localScalarText(filePath, 'file path'); +if isempty(filePath) + error('pf2:identity:invalidSource', ... + 'The file path must not be empty.'); +end +if exist(filePath, 'file') ~= 2 + error('pf2:identity:fileNotFound', ... + 'The fingerprint source is not an existing regular file: %s', ... + filePath); +end + +[fid, openMessage] = fopen(filePath, 'rb'); +if fid < 0 + error('pf2:identity:fileReadFailed', ... + 'Could not open fingerprint source "%s": %s', ... + filePath, openMessage); +end +closeFile = onCleanup(@() fclose(fid)); %#ok + +try + md = javaMethod('getInstance', ... + 'java.security.MessageDigest', 'SHA-256'); +catch cause + exception = MException('pf2:identity:hashUnavailable', ... + 'A SHA-256 implementation is unavailable.'); + exception = addCause(exception, cause); + throw(exception); +end + +chunkSize = 1024 * 1024; +byteLength = uint64(0); +while true + [bytes, count] = fread(fid, chunkSize, '*uint8'); + if count == 0 + break; + end + md.update(typecast(bytes(:), 'int8')); + byteLength = byteLength + uint64(count); +end + +[readMessage, readError] = ferror(fid); +% MATLAB reports EOF as status -1; only positive status values are errors. +if readError > 0 + error('pf2:identity:fileReadFailed', ... + 'Could not read fingerprint source "%s": %s', ... + filePath, readMessage); +end + +hashBytes = reshape(typecast(md.digest(), 'uint8'), 1, []); +digest = ['sha256:' sprintf('%02x', hashBytes)]; + +end + +function value = localScalarText(value, label) +% Convert one character vector or scalar string without lossy coercion. + +if isstring(value) + if ~isscalar(value) || ismissing(value) + error('pf2:identity:invalidSource', ... + 'The %s must be a character vector or nonmissing scalar string.', ... + label); + end + value = char(value); +elseif ~(ischar(value) && isrow(value)) + error('pf2:identity:invalidSource', ... + 'The %s must be a character vector or nonmissing scalar string.', ... + label); +end +end diff --git a/+pf2_base/+identity/sourceFingerprint.m b/+pf2_base/+identity/sourceFingerprint.m new file mode 100644 index 00000000..c4377d19 --- /dev/null +++ b/+pf2_base/+identity/sourceFingerprint.m @@ -0,0 +1,252 @@ +function fingerprint = sourceFingerprint(paths, varargin) +% SOURCEFINGERPRINT Fingerprint the exact bytes of an input source manifest. +% +% Syntax: +% fingerprint = pf2_base.identity.sourceFingerprint(paths) +% fingerprint = pf2_base.identity.sourceFingerprint( ... +% paths, 'LogicalNames', names) +% +% Inputs: +% paths - One or more explicit paths to existing regular files. Accepts a +% character vector, string array, or cell array of text scalars. +% +% Name-value Parameters: +% LogicalNames - Portable relative names for the files. Defaults to each +% path's basename, including its extension. Backslashes are +% canonicalized to `/` and text is normalized to Unicode +% NFC. Names must be unique and may not be absolute, empty, +% contain empty or `.` components, or traverse with `..`. +% +% Outputs: +% fingerprint - Scalar struct with fields: +% profile - `pf2.input.source.v1` +% algorithm - `sha256` +% available - Always true here; a matching field is present so this +% struct concatenates with the unavailable-source form +% emitted for in-memory inputs (see preflightImport). +% reason - Empty for an available source; carries a structured +% reason string only in the unavailable-source form. +% digest - Hash of the canonical source-manifest projection +% entries - Logical name, uint64 byte length, and byte digest for +% every file, sorted deterministically by logical name +% +% Physical roots, enumeration order, modification times, permissions, and +% all other filesystem metadata are excluded. Changing a logical name, byte +% length, or file byte changes the manifest identity. Invalid paths, unsafe +% logical names, and duplicate normalized names fail with stable identifiers +% in the `pf2:identity:*` namespace; incomplete manifests are never returned. +% Validation errors use `invalidSource`, `invalidLogicalName`, or +% `duplicateLogicalName`; file-reading errors are forwarded by sha256File. +% +% See also: pf2_base.identity.sha256File, +% pf2_base.identity.hashProjection, +% pf2_base.identity.verifyExpected + +pathList = localTextList(paths, 'paths', 'pf2:identity:invalidSource'); +if isempty(pathList) + error('pf2:identity:invalidSource', ... + 'At least one fingerprint source path is required.'); +end + +[logicalNames, logicalNamesSupplied] = localParseOptions(varargin{:}); +if logicalNamesSupplied + logicalNames = localTextList(logicalNames, 'LogicalNames', ... + 'pf2:identity:invalidLogicalName'); + if numel(logicalNames) ~= numel(pathList) + error('pf2:identity:invalidLogicalName', ... + ['LogicalNames must contain exactly one name for each source ' ... + 'path.']); + end +else + logicalNames = cell(size(pathList)); + for i = 1:numel(pathList) + [~, base, extension] = fileparts(pathList{i}); + logicalNames{i} = [base extension]; + end +end + +for i = 1:numel(logicalNames) + logicalNames{i} = localNormalizeLogicalName(logicalNames{i}); +end +localRejectDuplicates(logicalNames); + +order = localUtf8SortOrder(logicalNames); +pathList = pathList(order); +logicalNames = logicalNames(order); + +template = struct( ... + 'logicalName', '', ... + 'byteLength', uint64(0), ... + 'byteDigest', ''); +entries = repmat(template, numel(pathList), 1); +for i = 1:numel(pathList) + [byteDigest, byteLength] = ... + pf2_base.identity.sha256File(pathList{i}); + entries(i).logicalName = logicalNames{i}; + entries(i).byteLength = byteLength; + entries(i).byteDigest = byteDigest; +end + +digest = pf2_base.identity.hashProjection(entries, ... + 'ArtifactKind', 'pf2.input.source', ... + 'SchemaVersion', 1, ... + 'Projection', 'source-manifest-v1'); + +fingerprint = struct( ... + 'profile', 'pf2.input.source.v1', ... + 'algorithm', 'sha256', ... + 'available', true, ... + 'reason', '', ... + 'digest', digest, ... + 'entries', entries); + +end + +function [logicalNames, supplied] = localParseOptions(varargin) +% Parse the deliberately small public name-value surface with stable errors. + +logicalNames = []; +supplied = false; +if mod(numel(varargin), 2) ~= 0 + error('pf2:identity:invalidSource', ... + 'Options must be supplied as name-value pairs.'); +end +for i = 1:2:numel(varargin) + optionName = localTextScalar(varargin{i}, 'option name', ... + 'pf2:identity:invalidSource'); + if strcmpi(optionName, 'LogicalNames') + if supplied + error('pf2:identity:invalidSource', ... + 'LogicalNames may be supplied only once.'); + end + logicalNames = varargin{i + 1}; + supplied = true; + else + error('pf2:identity:invalidSource', ... + 'Unknown sourceFingerprint option: %s', optionName); + end +end +end + +function values = localTextList(value, label, identifier) +% Normalize supported text-container shapes to a column cell array. + +if ischar(value) + if ~isrow(value) + error(identifier, ... + '%s must contain character vectors or scalar strings.', label); + end + values = {value}; +elseif isstring(value) + if any(ismissing(value(:))) + error(identifier, '%s must not contain missing strings.', label); + end + values = cellstr(value(:)); +elseif iscell(value) + values = cell(numel(value), 1); + for i = 1:numel(value) + values{i} = localTextScalar(value{i}, label, identifier); + end +else + error(identifier, ... + '%s must be text or a cell array of text scalars.', label); +end +values = values(:); +end + +function value = localTextScalar(value, label, identifier) +% Convert one character vector or scalar string without lossy coercion. + +if isstring(value) + if ~isscalar(value) || ismissing(value) + error(identifier, ... + '%s values must be character vectors or nonmissing scalar strings.', ... + label); + end + value = char(value); +elseif ~(ischar(value) && isrow(value)) + error(identifier, ... + '%s values must be character vectors or nonmissing scalar strings.', ... + label); +end +end + +function name = localNormalizeLogicalName(name) +% Produce one portable NFC logical path and reject aliases/traversal. + +name = localTextScalar(name, 'LogicalNames', ... + 'pf2:identity:invalidLogicalName'); +try + name = pf2_base.identity.normalizeText(name, ... + 'ErrorIdentifier', 'pf2:identity:invalidLogicalName'); +catch cause + exception = MException('pf2:identity:invalidLogicalName', ... + 'Unicode NFC normalization is unavailable.'); + exception = addCause(exception, cause); + throw(exception); +end + +name = strrep(name, '\', '/'); +if isempty(name) + error('pf2:identity:invalidLogicalName', ... + 'Logical source names must not be empty.'); +end +if name(1) == '/' || ~isempty(regexp(name, '^[A-Za-z]:', 'once')) + error('pf2:identity:invalidLogicalName', ... + 'Logical source names must be relative: %s', name); +end +if any(uint16(name) < 32) || any(uint16(name) == 127) + error('pf2:identity:invalidLogicalName', ... + 'Logical source names must not contain control characters.'); +end + +components = regexp(name, '/', 'split'); +if any(cellfun('isempty', components)) || ... + any(strcmp(components, '.')) || any(strcmp(components, '..')) + error('pf2:identity:invalidLogicalName', ... + ['Logical source names must not contain empty, `.`, or `..` path ' ... + 'components: %s'], name); +end +end + +function localRejectDuplicates(names) +% Compare normalized names exactly; logical names are case-sensitive. + +for i = 1:numel(names) + for j = 1:i-1 + if strcmp(names{i}, names{j}) + error('pf2:identity:duplicateLogicalName', ... + 'Logical source name appears more than once: %s', names{i}); + end + end +end +end + +function order = localUtf8SortOrder(names) +% Sort by NFC UTF-8 bytes, independent of locale and filesystem collation. + +order = (1:numel(names)).'; +encoded = cellfun(@(x) unicode2native(x, 'UTF-8'), names, ... + 'UniformOutput', false); +for i = 2:numel(order) + current = order(i); + j = i - 1; + while j >= 1 && localBytesGreater(encoded{order(j)}, encoded{current}) + order(j + 1) = order(j); + j = j - 1; + end + order(j + 1) = current; +end +end + +function tf = localBytesGreater(left, right) +% True when left follows right under unsigned lexicographic byte ordering. + +sharedLength = min(numel(left), numel(right)); +difference = find(left(1:sharedLength) ~= right(1:sharedLength), 1); +if isempty(difference) + tf = numel(left) > numel(right); +else + tf = left(difference) > right(difference); +end +end diff --git a/+pf2_base/+identity/verifyExpected.m b/+pf2_base/+identity/verifyExpected.m new file mode 100644 index 00000000..51e5050a --- /dev/null +++ b/+pf2_base/+identity/verifyExpected.m @@ -0,0 +1,140 @@ +function status = verifyExpected(expected, observed, varargin) +% VERIFYEXPECTED Enforce an expected identity digest without warning fallback. +% +% Syntax: +% status = pf2_base.identity.verifyExpected(expected, observed) +% status = pf2_base.identity.verifyExpected( ... +% expected, observed, 'Kind', kind) +% +% Inputs: +% expected - Empty when no expectation was supplied, or a lowercase SHA-256 +% wire digest / scalar struct containing a `digest` field. +% observed - Lowercase SHA-256 wire digest or scalar struct containing a +% `digest` field. The observed value is always required. +% +% Name-value Parameters: +% Kind - Human-readable fingerprint kind used in status and diagnostics. +% Default: `input`. +% +% Outputs: +% status - Scalar struct with `kind`, `expected`, `observed`, and `status` +% fields. Status is `not-requested` when expected is empty and +% `matched` when the two supplied digests are equal. +% +% A supplied mismatch always throws `pf2:identity:fingerprintMismatch`. +% Callers that intentionally accept changed input must create a new RunSpec; +% this function has no warning or proceed-on-mismatch mode. +% Malformed digests or options throw `pf2:identity:invalidHash`. +% +% See also: pf2_base.identity.sourceFingerprint + +kind = localParseKind(varargin{:}); +observedDigest = localDigest(observed, 'observed', false); + +expectationProvided = ~localIsEmptyExpectation(expected); +if expectationProvided + expectedDigest = localDigest(expected, 'expected', false); +else + expectedDigest = ''; +end + +status = struct( ... + 'kind', kind, ... + 'expected', expectedDigest, ... + 'observed', observedDigest, ... + 'status', 'not-requested'); + +if ~expectationProvided + return; +end +if ~strcmp(expectedDigest, observedDigest) + error('pf2:identity:fingerprintMismatch', ... + ['The expected %s fingerprint does not match the observed input. ' ... + 'Expected %s; observed %s. Processing must not continue with this ' ... + 'RunSpec.'], kind, expectedDigest, observedDigest); +end +status.status = 'matched'; + +end + +function kind = localParseKind(varargin) +% Parse the single optional name-value pair with stable errors. + +kind = 'input'; +if mod(numel(varargin), 2) ~= 0 + error('pf2:identity:invalidHash', ... + 'Options must be supplied as name-value pairs.'); +end +supplied = false; +for i = 1:2:numel(varargin) + optionName = localScalarText(varargin{i}, 'option name', ... + 'pf2:identity:invalidHash'); + if strcmpi(optionName, 'Kind') + if supplied + error('pf2:identity:invalidHash', ... + 'Kind may be supplied only once.'); + end + kind = localScalarText(varargin{i + 1}, 'Kind', ... + 'pf2:identity:invalidHash'); + if isempty(kind) || any(uint16(kind) < 32) || any(uint16(kind) == 127) + error('pf2:identity:invalidHash', ... + 'Kind must be nonempty text without control characters.'); + end + supplied = true; + else + error('pf2:identity:invalidHash', ... + 'Unknown verifyExpected option: %s', optionName); + end +end +end + +function digest = localDigest(value, label, allowEmpty) +% Extract and validate one exact lowercase SHA-256 wire digest. + +if isstruct(value) + if ~isscalar(value) || ~isfield(value, 'digest') + error('pf2:identity:invalidHash', ... + 'The %s fingerprint struct must be scalar and contain digest.', ... + label); + end + value = value.digest; +end +digest = localScalarText(value, [label ' digest'], ... + 'pf2:identity:invalidHash'); +if allowEmpty && isempty(digest) + return; +end +if isempty(regexp(digest, '^sha256:[0-9a-f]{64}$', 'once')) + error('pf2:identity:invalidHash', ... + ['The %s fingerprint must use the exact lowercase wire form ' ... + '`sha256:<64 hexadecimal digits>`.'], label); +end +end + +function tf = localIsEmptyExpectation(value) +% Only a genuinely absent top-level value means "not requested". + +tf = (isnumeric(value) || islogical(value)) && isempty(value); +if ischar(value) + tf = isempty(value); +elseif isstring(value) + tf = isempty(value) || ... + (isscalar(value) && ~ismissing(value) && strlength(value) == 0); +end +end + +function value = localScalarText(value, label, identifier) +% Convert one character vector or scalar string without lossy coercion. + +if isstring(value) + if ~isscalar(value) || ismissing(value) + error(identifier, ... + '%s must be a character vector or nonmissing scalar string.', ... + label); + end + value = char(value); +elseif ~(ischar(value) && isrow(value)) + error(identifier, ... + '%s must be a character vector or nonmissing scalar string.', label); +end +end diff --git a/+pf2/+GUI/+functions/add.m b/+pf2_base/+methods/+functions/add.m similarity index 85% rename from +pf2/+GUI/+functions/add.m rename to +pf2_base/+methods/+functions/add.m index 5728c2f2..4e975100 100644 --- a/+pf2/+GUI/+functions/add.m +++ b/+pf2_base/+methods/+functions/add.m @@ -7,8 +7,8 @@ % processFNIRS2_configureMethods_functionAddEdit. % % Syntax: -% pf2.GUI.functions.add() -% out = pf2.GUI.functions.add() +% pf2_base.methods.functions.add() +% out = pf2_base.methods.functions.add() % % Inputs: % varargin - Reserved; the underlying add/edit GUI is invoked with no @@ -20,9 +20,9 @@ % % Example: % % Open the GUI to define a new processing function -% pf2.GUI.functions.add(); +% pf2_base.methods.functions.add(); % -% See also: pf2.GUI.functions.edit, pf2.GUI.functions, +% See also: pf2_base.methods.functions.edit, % processFNIRS2_configureMethods_functionAddEdit if(nargout>0) diff --git a/+pf2/+GUI/+functions/edit.m b/+pf2_base/+methods/+functions/edit.m similarity index 78% rename from +pf2/+GUI/+functions/edit.m rename to +pf2_base/+methods/+functions/edit.m index 6b3a8ca4..62e53bf0 100644 --- a/+pf2/+GUI/+functions/edit.m +++ b/+pf2_base/+methods/+functions/edit.m @@ -7,8 +7,8 @@ % are forwarded to processFNIRS2_configureMethods_functionAddEdit. % % Syntax: -% pf2.GUI.functions.edit(funcName) -% out = pf2.GUI.functions.edit(funcName, ...) +% pf2_base.methods.functions.edit(funcName) +% out = pf2_base.methods.functions.edit(funcName, ...) % % Inputs: % funcName - Name of the processing function to edit (e.g. 'pf2_lpf') @@ -20,13 +20,13 @@ % % Example: % % Edit the definition of the low-pass filter function -% pf2.GUI.functions.edit('pf2_lpf'); +% pf2_base.methods.functions.edit('pf2_lpf'); % -% See also: pf2.GUI.functions.add, pf2.GUI.functions, +% See also: pf2_base.methods.functions.add, % processFNIRS2_configureMethods_functionAddEdit if(nargin<1) - error('pf2:GUI:functions:edit:noFunctionName', 'Please provide function name to edit'); + error('pf2:methods:functions:edit:noFunctionName', 'Please provide function name to edit'); end if(nargout>0) diff --git a/+pf2_base/+methods/+seeds/+oxy/BPF.m b/+pf2_base/+methods/+seeds/+oxy/BPF.m new file mode 100644 index 00000000..b379b89a --- /dev/null +++ b/+pf2_base/+methods/+seeds/+oxy/BPF.m @@ -0,0 +1,30 @@ +function p = BPF() +% BPF Factory for the "BPF" oxy method (0.01-0.1 Hz Butterworth band-pass) +% +% Builds the shipped "BPF" oxy (Stage 3) processing method as an OxyPipeline: a +% single Butterworth band-pass filter (0.01-0.1 Hz) applied to hemoglobin +% signals, isolating the typical task-hemodynamic band while removing slow drift +% and high-frequency noise. Used by pf2_initialize and pf2.methods.resetDefaults +% to (re-)seed the default oxy methods. Returns a pipeline object you can save() +% to register or run() directly. +% +% Syntax: +% p = pf2_base.methods.seeds.oxy.BPF() +% +% Inputs: +% None +% +% Outputs: +% p - pf2_base.OxyPipeline named 'BPF', ready for save() or run() +% +% Example: +% p = pf2_base.methods.seeds.oxy.BPF(); +% p.save(); +% +% See also: pf2_base.methods.seeds.oxy.LPF, pf2_base.methods.seeds.oxy.HPF, +% pf2.methods.resetDefaults, pf2_base.OxyPipeline + +p = pf2_base.OxyPipeline('BPF', ... + 'Description', 'Butterworth band-pass filter (0.01-0.1 Hz) on hemoglobin signals'); +p = p.add('pf2_bpf_butter', 'lowF', 0.01, 'highF', 0.1); +end diff --git a/+pf2_base/+methods/+seeds/+oxy/HPF.m b/+pf2_base/+methods/+seeds/+oxy/HPF.m new file mode 100644 index 00000000..7034b758 --- /dev/null +++ b/+pf2_base/+methods/+seeds/+oxy/HPF.m @@ -0,0 +1,29 @@ +function p = HPF() +% HPF Factory for the "HPF" oxy method (0.01 Hz high-pass filter) +% +% Builds the shipped "HPF" oxy (Stage 3) processing method as an OxyPipeline: a +% single 0.01 Hz high-pass filter applied to hemoglobin signals to remove slow +% drift. Used by pf2_initialize and pf2.methods.resetDefaults to (re-)seed the +% default oxy methods. Returns a pipeline object you can save() to register or +% run() directly. +% +% Syntax: +% p = pf2_base.methods.seeds.oxy.HPF() +% +% Inputs: +% None +% +% Outputs: +% p - pf2_base.OxyPipeline named 'HPF', ready for save() or run() +% +% Example: +% p = pf2_base.methods.seeds.oxy.HPF(); +% p.save(); +% +% See also: pf2_base.methods.seeds.oxy.LPF, pf2_base.methods.seeds.oxy.BPF, +% pf2.methods.resetDefaults, pf2_base.OxyPipeline + +p = pf2_base.OxyPipeline('HPF', ... + 'Description', 'High-pass filter at 0.01 Hz on hemoglobin signals (drift removal)'); +p = p.add('pf2_hpf', 'freq_cut', 0.01); +end diff --git a/+pf2/+methods/+seeds/+oxy/LPF.m b/+pf2_base/+methods/+seeds/+oxy/LPF.m similarity index 83% rename from +pf2/+methods/+seeds/+oxy/LPF.m rename to +pf2_base/+methods/+seeds/+oxy/LPF.m index 0c38316d..037b5e63 100644 --- a/+pf2/+methods/+seeds/+oxy/LPF.m +++ b/+pf2_base/+methods/+seeds/+oxy/LPF.m @@ -8,7 +8,7 @@ % directly. % % Syntax: -% p = pf2.methods.seeds.oxy.LPF() +% p = pf2_base.methods.seeds.oxy.LPF() % % Inputs: % None @@ -18,10 +18,10 @@ % % Example: % % Build and register the default LPF oxy method -% p = pf2.methods.seeds.oxy.LPF(); +% p = pf2_base.methods.seeds.oxy.LPF(); % p.save(); % -% See also: pf2.methods.seeds.oxy.LPF_ROI, pf2.methods.resetDefaults, +% See also: pf2_base.methods.seeds.oxy.LPF_ROI, pf2.methods.resetDefaults, % pf2_base.OxyPipeline p = pf2_base.OxyPipeline('LPF', ... diff --git a/+pf2/+methods/+seeds/+oxy/LPF_ROI.m b/+pf2_base/+methods/+seeds/+oxy/LPF_ROI.m similarity index 84% rename from +pf2/+methods/+seeds/+oxy/LPF_ROI.m rename to +pf2_base/+methods/+seeds/+oxy/LPF_ROI.m index fc065b4d..28a033ef 100644 --- a/+pf2/+methods/+seeds/+oxy/LPF_ROI.m +++ b/+pf2_base/+methods/+seeds/+oxy/LPF_ROI.m @@ -8,7 +8,7 @@ % methods. Returns a pipeline object you can save() to register or run(). % % Syntax: -% p = pf2.methods.seeds.oxy.LPF_ROI() +% p = pf2_base.methods.seeds.oxy.LPF_ROI() % % Inputs: % None @@ -18,10 +18,10 @@ % % Example: % % Build and register the default LPF_ROI oxy method -% p = pf2.methods.seeds.oxy.LPF_ROI(); +% p = pf2_base.methods.seeds.oxy.LPF_ROI(); % p.save(); % -% See also: pf2.methods.seeds.oxy.LPF, pf2.methods.resetDefaults, +% See also: pf2_base.methods.seeds.oxy.LPF, pf2.methods.resetDefaults, % pf2_base.OxyPipeline p = pf2_base.OxyPipeline('LPF_ROI', ... diff --git a/+pf2_base/+methods/+seeds/+oxy/takizawa_easy.m b/+pf2_base/+methods/+seeds/+oxy/takizawa_easy.m new file mode 100644 index 00000000..8bc177ae --- /dev/null +++ b/+pf2_base/+methods/+seeds/+oxy/takizawa_easy.m @@ -0,0 +1,44 @@ +function p = takizawa_easy() +% TAKIZAWA_EASY Factory for the "takizawa_easy" oxy method (lenient rejection) +% +% Builds the shipped "takizawa_easy" oxy (Stage 3) processing method as an +% OxyPipeline: automatic channel rejection using the Takizawa criteria with the +% lenient (non-strict) rule combination. Rejected channels are marked in the +% data's fchMask rather than having their hemoglobin values transformed. Used by +% pf2_initialize and pf2.methods.resetDefaults to (re-)seed the default oxy +% methods. Returns a pipeline object you can save() to register or run() +% directly. +% +% The strict/lenient switch is the pf2_TakizawaRejection strictCriteria flag: +% lenient (strictCriteria = 0) combines the rejection rules with AND, so a +% channel is dropped only when it fails every rule; the strict variant +% (takizawa_hard) uses OR. +% +% Syntax: +% p = pf2_base.methods.seeds.oxy.takizawa_easy() +% +% Inputs: +% None +% +% Outputs: +% p - pf2_base.OxyPipeline named 'takizawa_easy', ready for save() or run() +% +% Example: +% % Build and register the takizawa_easy oxy method +% p = pf2_base.methods.seeds.oxy.takizawa_easy(); +% p.save(); +% +% References: +% Takizawa, R., Kasai, K., Kawakubo, Y., Marumo, K., Kawasaki, S., +% Yamasue, H., & Fukuda, M. (2008). Reduced frontopolar activation during +% verbal fluency task in schizophrenia: A multi-channel near-infrared +% spectroscopy study. Schizophrenia Research, 99(1-3), 250-262. +% DOI: 10.1016/j.schres.2007.10.025 +% +% See also: pf2_base.methods.seeds.oxy.takizawa_hard, pf2_base.methods.seeds.oxy.LPF, +% pf2.methods.resetDefaults, pf2_TakizawaRejection, pf2_base.OxyPipeline + +p = pf2_base.OxyPipeline('takizawa_easy', ... + 'Description', 'Takizawa automatic channel rejection, lenient criteria (Takizawa et al. 2008)'); +p = p.add('pf2_TakizawaRejection', 'strictCriteria', 0); +end diff --git a/+pf2_base/+methods/+seeds/+oxy/takizawa_hard.m b/+pf2_base/+methods/+seeds/+oxy/takizawa_hard.m new file mode 100644 index 00000000..2f4f99a3 --- /dev/null +++ b/+pf2_base/+methods/+seeds/+oxy/takizawa_hard.m @@ -0,0 +1,44 @@ +function p = takizawa_hard() +% TAKIZAWA_HARD Factory for the "takizawa_hard" oxy method (strict rejection) +% +% Builds the shipped "takizawa_hard" oxy (Stage 3) processing method as an +% OxyPipeline: automatic channel rejection using the Takizawa criteria with the +% strict rule combination. Rejected channels are marked in the data's fchMask +% rather than having their hemoglobin values transformed. Used by +% pf2_initialize and pf2.methods.resetDefaults to (re-)seed the default oxy +% methods. Returns a pipeline object you can save() to register or run() +% directly. +% +% The strict/lenient switch is the pf2_TakizawaRejection strictCriteria flag: +% strict (strictCriteria = 1) combines the rejection rules with OR, so a channel +% is dropped when it fails any rule; the lenient variant (takizawa_easy) uses +% AND. +% +% Syntax: +% p = pf2_base.methods.seeds.oxy.takizawa_hard() +% +% Inputs: +% None +% +% Outputs: +% p - pf2_base.OxyPipeline named 'takizawa_hard', ready for save() or run() +% +% Example: +% % Build and register the takizawa_hard oxy method +% p = pf2_base.methods.seeds.oxy.takizawa_hard(); +% p.save(); +% +% References: +% Takizawa, R., Kasai, K., Kawakubo, Y., Marumo, K., Kawasaki, S., +% Yamasue, H., & Fukuda, M. (2008). Reduced frontopolar activation during +% verbal fluency task in schizophrenia: A multi-channel near-infrared +% spectroscopy study. Schizophrenia Research, 99(1-3), 250-262. +% DOI: 10.1016/j.schres.2007.10.025 +% +% See also: pf2_base.methods.seeds.oxy.takizawa_easy, pf2_base.methods.seeds.oxy.LPF, +% pf2.methods.resetDefaults, pf2_TakizawaRejection, pf2_base.OxyPipeline + +p = pf2_base.OxyPipeline('takizawa_hard', ... + 'Description', 'Takizawa automatic channel rejection, strict criteria (Takizawa et al. 2008)'); +p = p.add('pf2_TakizawaRejection', 'strictCriteria', 1); +end diff --git a/+pf2_base/+methods/+seeds/+raw/OD_SMAR.m b/+pf2_base/+methods/+seeds/+raw/OD_SMAR.m new file mode 100644 index 00000000..b9d19ed5 --- /dev/null +++ b/+pf2_base/+methods/+seeds/+raw/OD_SMAR.m @@ -0,0 +1,31 @@ +function p = OD_SMAR() +% OD_SMAR Factory for the "OD_SMAR" raw method (SMAR motion correction) +% +% Builds the shipped "OD_SMAR" raw (Stage 1) processing method as a RawPipeline: +% log transform to optical density, then Sliding-window Motion Artifact +% Rejection (SMAR), which rejects samples whose windowed coefficient of +% variation falls outside acceptable bounds. Used by pf2_initialize and +% pf2.methods.resetDefaults to (re-)seed the default raw methods. Returns a +% pipeline object you can save() to register or run() directly. +% +% Syntax: +% p = pf2_base.methods.seeds.raw.OD_SMAR() +% +% Inputs: +% None +% +% Outputs: +% p - pf2_base.RawPipeline named 'OD_SMAR', ready for save() or run() +% +% Example: +% p = pf2_base.methods.seeds.raw.OD_SMAR(); +% p.save(); +% +% See also: pf2_base.methods.seeds.raw.OD_TDDR, pf2.methods.resetDefaults, +% pf2_base.RawPipeline, pf2_SMAR + +p = pf2_base.RawPipeline('OD_SMAR', ... + 'Description', 'Log transform then SMAR (sliding-window motion artifact rejection)'); +p = p.add('pf2_Intensity2OD'); +p = p.add('pf2_SMAR'); +end diff --git a/+pf2/+methods/+seeds/+raw/OD_TDDR.m b/+pf2_base/+methods/+seeds/+raw/OD_TDDR.m similarity index 86% rename from +pf2/+methods/+seeds/+raw/OD_TDDR.m rename to +pf2_base/+methods/+seeds/+raw/OD_TDDR.m index f37ae0da..4c5f1d3a 100644 --- a/+pf2/+methods/+seeds/+raw/OD_TDDR.m +++ b/+pf2_base/+methods/+seeds/+raw/OD_TDDR.m @@ -13,7 +13,7 @@ % DOI: 10.1016/j.neuroimage.2018.09.025 % % Syntax: -% p = pf2.methods.seeds.raw.OD_TDDR() +% p = pf2_base.methods.seeds.raw.OD_TDDR() % % Inputs: % None @@ -23,10 +23,10 @@ % % Example: % % Build and register the default OD_TDDR raw method -% p = pf2.methods.seeds.raw.OD_TDDR(); +% p = pf2_base.methods.seeds.raw.OD_TDDR(); % p.save(); % -% See also: pf2.methods.seeds.raw.OD_TDDR_lpf, pf2.methods.resetDefaults, +% See also: pf2_base.methods.seeds.raw.OD_SMAR, pf2.methods.resetDefaults, % pf2_base.RawPipeline, pf2_MotionCorrectTDDR p = pf2_base.RawPipeline('OD_TDDR', ... diff --git a/+pf2/+methods/+seeds/list.m b/+pf2_base/+methods/+seeds/list.m similarity index 76% rename from +pf2/+methods/+seeds/list.m rename to +pf2_base/+methods/+seeds/list.m index c864cdb2..11a36aae 100644 --- a/+pf2/+methods/+seeds/list.m +++ b/+pf2_base/+methods/+seeds/list.m @@ -2,9 +2,9 @@ % LIST Enumerate available seed methods for the given stage. % % Syntax: -% info = pf2.methods.seeds.list() % both stages -% info = pf2.methods.seeds.list('raw') % raw stage only -% info = pf2.methods.seeds.list('oxy') % oxy stage only +% info = pf2_base.methods.seeds.list() % both stages +% info = pf2_base.methods.seeds.list('raw') % raw stage only +% info = pf2_base.methods.seeds.list('oxy') % oxy stage only % % Inputs: % stage - (optional) 'raw' | 'oxy' (default: '' = both) @@ -15,14 +15,14 @@ % .stage - 'raw' | 'oxy' % .file - absolute path to the seed factory function % -% Discovery: each seed package (+pf2/+methods/+seeds/+raw and .../+oxy) +% Discovery: each seed package (+pf2_base/+methods/+seeds/+raw and .../+oxy) % contributes one entry per .m file found there. The .m file must be a % zero-argument function returning a Pipeline / RawPipeline / OxyPipeline. % % Example: -% info = pf2.methods.seeds.list('raw'); +% info = pf2_base.methods.seeds.list('raw'); % for k = 1:numel(info) -% p = feval(['pf2.methods.seeds.raw.' info(k).name]); +% p = feval(['pf2_base.methods.seeds.raw.' info(k).name]); % disp(p); % end % diff --git a/+pf2/+methods/validateFunction.m b/+pf2_base/+methods/validateFunction.m similarity index 87% rename from +pf2/+methods/validateFunction.m rename to +pf2_base/+methods/validateFunction.m index c3beb24a..5eaad6d8 100644 --- a/+pf2/+methods/validateFunction.m +++ b/+pf2_base/+methods/validateFunction.m @@ -6,8 +6,8 @@ function validateFunction(funcName, args, argvals, varargin) % consistency, and output variable names. % % Syntax: -% pf2.methods.validateFunction(funcName, args, argvals) -% pf2.methods.validateFunction(funcName, args, argvals, 'Output', outputVar) +% pf2_base.methods.validateFunction(funcName, args, argvals) +% pf2_base.methods.validateFunction(funcName, args, argvals, 'Output', outputVar) % % Inputs: % funcName - Name of the MATLAB function (e.g., 'pf2_TDDR') @@ -19,15 +19,15 @@ function validateFunction(funcName, args, argvals, varargin) % % Example: % % Validate a valid function -% pf2.methods.validateFunction('pf2_TDDR', {'x','fs'}, {'x','fs'}); +% pf2_base.methods.validateFunction('pf2_TDDR', {'x','fs'}, {'x','fs'}); % % % Validate with output specification -% pf2.methods.validateFunction('pf2_TakizawaRejection', ... +% pf2_base.methods.validateFunction('pf2_TakizawaRejection', ... % {'x','fchMask','threshold'}, {'x','fchMask',0.75}, ... % 'Output', 'fchMask'); % % % This will error - function does not exist -% pf2.methods.validateFunction('nonexistent_func', {'x'}, {'x'}); +% pf2_base.methods.validateFunction('nonexistent_func', {'x'}, {'x'}); % % See also: pf2.methods.raw.addFunction, pf2.methods.oxy.addFunction, % pf2.methods.raw.editFunction, pf2.methods.oxy.editFunction diff --git a/+pf2_base/+tests/+data/characterization_golden.mat b/+pf2_base/+tests/+data/characterization_golden.mat index 2d9b0646..27c5c10e 100644 Binary files a/+pf2_base/+tests/+data/characterization_golden.mat and b/+pf2_base/+tests/+data/characterization_golden.mat differ diff --git a/+pf2_base/+tests/+fixtures/IdentityImportSpy.m b/+pf2_base/+tests/+fixtures/IdentityImportSpy.m new file mode 100644 index 00000000..980cbd6e --- /dev/null +++ b/+pf2_base/+tests/+fixtures/IdentityImportSpy.m @@ -0,0 +1,25 @@ +classdef IdentityImportSpy < handle +%IDENTITYIMPORTSPY Observable importer used by identity preflight tests. + + properties + CallCount = 0 + Data = [] + OnRun = [] + end + + methods + function obj = IdentityImportSpy(data) + if nargin > 0 + obj.Data = data; + end + end + + function data = invoke(obj) + obj.CallCount = obj.CallCount + 1; + if ~isempty(obj.OnRun) + obj.OnRun(); + end + data = obj.Data; + end + end +end diff --git a/+pf2_base/+tests/+fixtures/identityRecipeSpike.m b/+pf2_base/+tests/+fixtures/identityRecipeSpike.m new file mode 100644 index 00000000..c0394fc8 --- /dev/null +++ b/+pf2_base/+tests/+fixtures/identityRecipeSpike.m @@ -0,0 +1,70 @@ +function recipe = identityRecipeSpike() +%IDENTITYRECIPESPIKE Representative schema tree for identity proof tests. +% +% This is deliberately a Phase-0 fixture, not the frozen Recipe v1 schema. It +% mirrors the shipped OD_TDDR raw pipeline and LPF oxy pipeline without storing +% runtime Pipeline objects or function handles. Stable production function IDs +% and the final Recipe field catalog still require their metadata ADR. + + emptyParameter = struct( ... + 'parameterId', {}, ... + 'valueType', {}, ... + 'unit', {}, ... + 'shape', {}, ... + 'value', {}); + cutoffParameter = struct( ... + 'parameterId', 'freq-cut', ... + 'valueType', 'double', ... + 'unit', 'Hz', ... + 'shape', uint64([1 1]), ... + 'value', 0.1); + + stepTemplate = struct( ... + 'functionId', '', ... + 'functionVersion', uint64(1), ... + 'enabled', true, ... + 'parameters', emptyParameter); + rawSteps = repmat(stepTemplate, 2, 1); + rawSteps(1).functionId = 'spike.pf2.raw.intensity-to-od'; + rawSteps(2).functionId = 'spike.pf2.raw.motion-correct-tddr'; + oxySteps = stepTemplate; + oxySteps.functionId = 'spike.pf2.oxy.low-pass-filter'; + oxySteps.parameters = cutoffParameter; + + scientific = struct(); + scientific.inputContract = struct( ... + 'modality', 'fnirs-cw', ... + 'domain', 'intensity', ... + 'wavelengthsNm', uint16([690 830]), ... + 'signalUnit', 'arbitrary-unit'); + scientific.context = struct( ... + 'dpfPolicy', 'fixed', ... + 'fixedDpf', 5.93, ... + 'baselineStartSeconds', 0.0, ... + 'baselineLengthSeconds', 10.0, ... + 'dirtyBaseline', false, ... + 'processRejectedChannels', false); + scientific.rawSteps = rawSteps; + scientific.oxySteps = oxySteps; + scientific.qcPolicy = struct( ... + 'policyId', 'spike.pf2.qc.channel-threshold', ... + 'enabled', true, ... + 'rejectLevel', 0.0); + scientific.outputs = struct( ... + 'biomarkers', {{'hbo', 'hbr'}}, ... + 'includeOpticalDensity', true); + + recipe = struct(); + recipe.artifactKind = 'pf2.processing.recipe'; + recipe.schemaVersion = uint64(1); + recipe.artifactId = 'identity-spike-od-tddr-lpf'; + recipe.displayName = ['OD-TDDR + LPF ' char(hex2dec('03B1'))]; + recipe.scientific = scientific; + recipe.authoredFrom = struct( ... + 'kind', 'shipped-methods', ... + 'rawName', 'OD_TDDR', ... + 'oxyName', 'LPF'); + recipe.authoringDelta = {'Phase-0 identity fixture'}; + recipe.citations = { ... + 'Fishburn et al. (2019), doi:10.1016/j.neuroimage.2018.09.025'}; +end diff --git a/+pf2_base/+tests/+integration/PipelineEditorWorkflowTest.m b/+pf2_base/+tests/+integration/PipelineEditorWorkflowTest.m index b1308529..6ee42516 100644 --- a/+pf2_base/+tests/+integration/PipelineEditorWorkflowTest.m +++ b/+pf2_base/+tests/+integration/PipelineEditorWorkflowTest.m @@ -97,7 +97,7 @@ function testA4FirstInstallSeeds(tc) sections = PF2.myRawMethods.cfg.Sections; tc.verifyTrue(any(strcmp(sections, 'OD_TDDR'))); - tc.verifyTrue(any(strcmp(sections, 'OD_TDDR_lpf'))); + tc.verifyTrue(any(strcmp(sections, 'OD_SMAR'))); sectionsOxy = PF2.myOxyMethods.cfg.Sections; tc.verifyTrue(any(strcmp(sectionsOxy, 'LPF'))); tc.verifyTrue(any(strcmp(sectionsOxy, 'LPF_ROI'))); @@ -111,10 +111,10 @@ function restoreCfgs(r, o, br, bo, p2, sf) %% A4 every seed factory returns a runnable Pipeline function testA4SeedsAreRunnable(tc) - seeds = pf2.methods.seeds.list(); + seeds = pf2_base.methods.seeds.list(); for k = 1:numel(seeds) s = seeds(k); - fact = ['pf2.methods.seeds.' s.stage '.' s.name]; + fact = ['pf2_base.methods.seeds.' s.stage '.' s.name]; p = feval(fact); tc.verifyTrue(isa(p, 'pf2_base.Pipeline')); tc.verifyEqual(p.numSteps(), p.numSteps()); % sanity @@ -126,7 +126,7 @@ function testA4SeedsAreRunnable(tc) %% A5 PipelineModel round-trip with a runnable pipeline at the end function testA5ModelRoundTripAndRun(tc) - p0 = pf2.methods.seeds.raw.OD_TDDR(); + p0 = pf2_base.methods.seeds.raw.OD_TDDR(); m = pf2_base.PipelineModel(p0); % Mutate via the model: add LPF, tweak its cutoff m.addStep('pf2_lpf', 'freq_cut', 0.05); diff --git a/+pf2_base/+tests/+unit/CharacterizationTest.m b/+pf2_base/+tests/+unit/CharacterizationTest.m index 7e0ab34e..89dd3d7a 100644 --- a/+pf2_base/+tests/+unit/CharacterizationTest.m +++ b/+pf2_base/+tests/+unit/CharacterizationTest.m @@ -117,7 +117,7 @@ 'f2000_calc_bl_5_15' 'fNIR2000' 'None' 'None' 'Calc' 5.93 25 5 15 false 'f2000_calc_lpf' 'fNIR2000' 'None' 'LPF' 'Calc' 5.93 25 0 10 false 'f2000_tddr_calc' 'fNIR2000' 'OD_TDDR' 'None' 'Calc' 5.93 25 0 10 false - 'f2000_tddr_lpf_fixed' 'fNIR2000' 'OD_TDDR_lpf' 'LPF' 'Fixed' 5.50 25 0 10 false + 'f2000_tddr_lpf_fixed' 'fNIR2000' 'OD_TDDR' 'LPF' 'Fixed' 5.50 25 0 10 false 'f1200_none_default' 'fNIR1200' 'None' 'None' 'None' 5.93 25 0 10 false 'f1200_calc_default' 'fNIR1200' 'None' 'None' 'Calc' 5.93 25 0 10 false 'f1200_fixed_bl_2_8' 'fNIR1200' 'None' 'None' 'Fixed' 6.00 25 2 8 false diff --git a/+pf2_base/+tests/+unit/IdentityCanonicalEncodingTest.m b/+pf2_base/+tests/+unit/IdentityCanonicalEncodingTest.m new file mode 100644 index 00000000..1c059bb8 --- /dev/null +++ b/+pf2_base/+tests/+unit/IdentityCanonicalEncodingTest.m @@ -0,0 +1,192 @@ +classdef IdentityCanonicalEncodingTest < matlab.unittest.TestCase +% IDENTITYCANONICALENCODINGTEST Contract tests for canonical identity bytes. +% +% These tests deliberately exercise semantic properties rather than MAT or +% JSON transport bytes. The encoder receives an already schema-normalized +% value and must produce release- and platform-independent bytes. +% +% Run with: +% runtests('pf2_base.tests.unit.IdentityCanonicalEncodingTest') +% +% See also: pf2_base.identity.canonicalBytes + + methods (Test) + function testReturnsBytes(testCase) + bytes = pf2_base.identity.canonicalBytes(struct('value', 1)); + + testCase.verifyClass(bytes, 'uint8'); + testCase.verifyNotEmpty(bytes); + end + + function testKnownCanonicalVector(testCase) + % pf2-canonical-binary-v1: profile header, UINT8 tag, framed + % payload length, [1 x 1] dimensions, then scalar value 42. + expectedHex = [ ... + '5046322d43414e4f4e4943414c0001' ... + '13' ... + '0000000000000019' ... + '0000000000000002' ... + '0000000000000001' ... + '0000000000000001' ... + '2a']; + expected = uint8(sscanf(expectedHex, '%2x').'); + + testCase.verifyEqual(encode(uint8(42)), expected); + end + + function testFieldOrderIsInsignificant(testCase) + first = struct(); + first.zeta = uint16(2); + first.alpha = 'near infrared'; + first.nested = struct('right', true, 'left', single(0.5)); + + second = struct(); + second.nested = struct('left', single(0.5), 'right', true); + second.alpha = 'near infrared'; + second.zeta = uint16(2); + + testCase.verifyEqual(encode(first), encode(second)); + end + + function testArrayOrderAndShapeAreSignificant(testCase) + row = uint16([1 2 3]); + column = row.'; + reordered = uint16([1 3 2]); + + testCase.verifyNotEqual(encode(row), encode(column)); + testCase.verifyNotEqual(encode(row), encode(reordered)); + testCase.verifyNotEqual(encode(reshape(1:6, [1 2 3])), ... + encode(reshape(1:6, [1 3 2]))); + end + + function testCellShapeIsSignificant(testCase) + row = {uint8(1), 'two'}; + column = row.'; + + testCase.verifyNotEqual(encode(row), encode(column)); + end + + function testStringArrayShapeAndContentAreSignificant(testCase) + row = ["one", "two"]; + column = row.'; + changed = ["one", "too"]; + + testCase.verifyNotEqual(encode(row), encode(column)); + testCase.verifyNotEqual(encode(row), encode(changed)); + end + + function testMissingAndEmptyStringsAreDistinct(testCase) + emptyString = ""; + missingString = string(missing); + + testCase.verifyNotEqual(encode(emptyString), encode(missingString)); + testCase.verifyNotEqual(encode(["", "value"]), ... + encode([missingString, "value"])); + end + + function testNumericAndLogicalTypesAreSignificant(testCase) + values = { ... + logical(1), uint8(1), int8(1), uint16(1), int16(1), ... + uint32(1), int32(1), uint64(1), int64(1), ... + single(1), double(1)}; + encodings = cellfun(@encode, values, 'UniformOutput', false); + + for i = 1:numel(encodings) + for j = i + 1:numel(encodings) + testCase.verifyNotEqual(encodings{i}, encodings{j}, ... + sprintf('Types %s and %s were conflated.', ... + class(values{i}), class(values{j}))); + end + end + end + + function testFiniteBitChangesAreSignificant(testCase) + testCase.verifyNotEqual(encode(1.0), encode(1.0 + eps(1.0))); + testCase.verifyNotEqual(encode(single(1.0)), ... + encode(single(1.0) + eps(single(1.0)))); + testCase.verifyNotEqual(encode(realmin), encode(realmin * eps)); + end + + function testSignedZeroNormalizes(testCase) + negativeZero = typecast(bitshift(uint64(1), 63), 'double'); + testCase.assertEqual(negativeZero, 0); + testCase.verifyEqual(encode(0.0), encode(negativeZero)); + + negativeZeroSingle = typecast(bitshift(uint32(1), 31), 'single'); + testCase.assertEqual(negativeZeroSingle, single(0)); + testCase.verifyEqual(encode(single(0)), encode(negativeZeroSingle)); + end + + function testNonFiniteRejectedByDefault(testCase) + testCase.verifyError(@() encode(NaN), 'pf2:identity:nonFinite'); + testCase.verifyError(@() encode(Inf), 'pf2:identity:nonFinite'); + testCase.verifyError(@() encode(-Inf), 'pf2:identity:nonFinite'); + end + + function testAllowedNonFiniteHasCanonicalTokens(testCase) + quietNaNBits1 = bitor(bitshift(uint64(2047), 52), ... + bitshift(uint64(1), 51)); + quietNaNBits2 = bitor(quietNaNBits1, uint64(12345)); + nan1 = typecast(quietNaNBits1, 'double'); + nan2 = typecast(quietNaNBits2, 'double'); + + first = encode(nan1, 'AllowNonFinite', true); + second = encode(nan2, 'AllowNonFinite', true); + positiveInfinity = encode(Inf, 'AllowNonFinite', true); + negativeInfinity = encode(-Inf, 'AllowNonFinite', true); + + testCase.verifyEqual(first, second, ... + 'NaN payload bits must not enter canonical identity.'); + testCase.verifyNotEqual(first, positiveInfinity); + testCase.verifyNotEqual(positiveInfinity, negativeInfinity); + end + + function testTextUsesNFCAndUTF8(testCase) + composed = char(hex2dec('00E9')); % e-acute + decomposed = ['e' char(hex2dec('0301'))]; % e + combining acute + chinese = char([hex2dec('8FD1'), hex2dec('7EA2'), hex2dec('5916')]); + japanese = char([hex2dec('8FD1'), hex2dec('8D64'), hex2dec('5916')]); + + testCase.verifyEqual(encode(composed), encode(decomposed)); + testCase.verifyEqual(encode(chinese), encode(string(chinese))); + testCase.verifyEqual(encode(japanese), encode(string(japanese))); + testCase.verifyNotEqual(encode(chinese), encode(japanese)); + end + + function testInvalidUnicodeRejected(testCase) + loneHighSurrogate = char(uint16(hex2dec('D800'))); + testCase.verifyError(@() encode(loneHighSurrogate), ... + 'pf2:identity:invalidText'); + end + + function testEmptyArrayShapesAreDistinct(testCase) + empty00 = zeros(0, 0, 'uint8'); + empty01 = zeros(0, 1, 'uint8'); + empty10 = zeros(1, 0, 'uint8'); + + testCase.verifyNotEqual(encode(empty00), encode(empty01)); + testCase.verifyNotEqual(encode(empty00), encode(empty10)); + testCase.verifyNotEqual(encode(empty01), encode(empty10)); + end + + function testUnsupportedRuntimeTypesFailClosed(testCase) + unsupported = { ... + @sin, ... + table((1:2).'), ... + sparse(eye(2)), ... + 1 + 2i, ... + containers.Map({'a'}, {1})}; + + for i = 1:numel(unsupported) + value = unsupported{i}; + testCase.verifyError(@() encode(value), ... + 'pf2:identity:unsupportedType', ... + sprintf('Unsupported class %s did not fail closed.', class(value))); + end + end + end +end + +function bytes = encode(value, varargin) + bytes = pf2_base.identity.canonicalBytes(value, varargin{:}); +end diff --git a/+pf2_base/+tests/+unit/IdentityNamedHashTest.m b/+pf2_base/+tests/+unit/IdentityNamedHashTest.m new file mode 100644 index 00000000..cb3b6bf6 --- /dev/null +++ b/+pf2_base/+tests/+unit/IdentityNamedHashTest.m @@ -0,0 +1,181 @@ +classdef IdentityNamedHashTest < matlab.unittest.TestCase +% IDENTITYNAMEDHASHTEST Contract tests for SHA-256 and named hash domains. +% +% Run with: +% runtests('pf2_base.tests.unit.IdentityNamedHashTest') +% +% See also: pf2_base.identity.sha256Bytes, +% pf2_base.identity.hashProjection, +% pf2_base.identity.verifyExpected + + methods (Test) + function testKnownSHA256Vectors(testCase) + emptyExpected = "sha256:e3b0c44298fc1c149afbf4c8996fb924" + ... + "27ae41e4649b934ca495991b7852b855"; + abcExpected = "sha256:ba7816bf8f01cfea414140de5dae2223" + ... + "b00361a396177a9cb410ff61f20015ad"; + + testCase.verifyEqual(string(pf2_base.identity.sha256Bytes( ... + zeros(1, 0, 'uint8'))), emptyExpected); + testCase.verifyEqual(string(pf2_base.identity.sha256Bytes( ... + uint8([97 98 99]))), abcExpected); + end + + function testHashProjectionGoldenVector(testCase) + % Frozen known-answer digest for a fixed value in a fixed artifact + % domain. Unlike the self-consistency tests below, this anchors the + % exact canonical envelope and domain-separation format across runs + % and releases: any accidental change to canonicalBytes, the + % hashProjection envelope, or a domain string rewrites this digest + % and fails here. If this changes intentionally, it is a durable + % identity break and the constant must be regenerated deliberately. + value = struct('label', 'golden', 'count', uint32(3)); + expected = "sha256:054b34ea43bd16a032a0d659b4a652ec" + ... + "60fb3b12ee723f06fa828396a2417990"; + + digest = pf2_base.identity.hashProjection(value, ... + 'ArtifactKind', 'pf2.processing.recipe', ... + 'SchemaVersion', uint64(1), ... + 'Projection', 'scientific-content-v1'); + + testCase.verifyEqual(string(digest), expected); + end + + function testDigestWireForm(testCase) + digest = string(project(struct('method', 'lpf'))); + + testCase.verifyTrue(~isempty(regexp(char(digest), ... + '^sha256:[0-9a-f]{64}$', 'once')), ... + 'Digest must use the lowercase, algorithm-prefixed wire form.'); + end + + function testProjectionIgnoresStructFieldOrder(testCase) + first = struct('method', 'lpf', 'cutoff', 0.1); + second = struct('cutoff', 0.1, 'method', 'lpf'); + + testCase.verifyEqual(project(first), project(second)); + end + + function testArtifactKindSeparatesDomains(testCase) + payload = struct('value', uint16(42)); + kinds = { ... + 'pf2.processing.recipe', ... + 'pf2.processing.run-spec', ... + 'pf2.processing.run-record', ... + 'pf2.study.manifest'}; + digests = cellfun(@(kind) project(payload, ... + 'ArtifactKind', kind), kinds, 'UniformOutput', false); + + testCase.verifyEqual(numel(unique(digests)), numel(kinds)); + end + + function testSchemaVersionSeparatesDomains(testCase) + payload = struct('value', uint16(42)); + version1 = project(payload, 'SchemaVersion', uint64(1)); + version2 = project(payload, 'SchemaVersion', uint64(2)); + + testCase.verifyNotEqual(version1, version2); + end + + function testProjectionNameSeparatesDomains(testCase) + payload = struct('value', uint16(42)); + requested = project(payload, 'Projection', 'requested-recipe-v1'); + effective = project(payload, 'Projection', 'effective-recipe-v1'); + + testCase.verifyNotEqual(requested, effective); + testCase.verifyEqual(requested, ... + project(payload, 'Projection', 'requested-recipe-v1')); + testCase.verifyEqual(effective, ... + project(payload, 'Projection', 'effective-recipe-v1')); + end + + function testScientificMutationChangesDigest(testCase) + baseline = struct( ... + 'methodId', 'pf2.raw.tddr', ... + 'methodVersion', uint64(1), ... + 'parameters', struct('cutoffHz', 0.1), ... + 'units', 'Hz'); + changedParameter = baseline; + changedParameter.parameters.cutoffHz = 0.11; + changedMethod = baseline; + changedMethod.methodId = 'pf2.raw.spline'; + changedUnits = baseline; + changedUnits.units = 'rad/s'; + + original = project(baseline); + testCase.verifyNotEqual(original, project(changedParameter)); + testCase.verifyNotEqual(original, project(changedMethod)); + testCase.verifyNotEqual(original, project(changedUnits)); + end + + function testProjectionNormalizesSignedZero(testCase) + negativeZero = typecast(bitshift(uint64(1), 63), 'double'); + testCase.verifyEqual(project(struct('x', 0.0)), ... + project(struct('x', negativeZero))); + end + + function testProjectionControlsNonFiniteValues(testCase) + payload = struct('threshold', Inf); + testCase.verifyError(@() project(payload), ... + 'pf2:identity:nonFinite'); + + allowed = project(payload, 'AllowNonFinite', true); + testCase.verifyTrue(startsWith(string(allowed), "sha256:")); + end + + function testInvalidHashDomainMetadataRejected(testCase) + payload = struct('value', 1); + testCase.verifyError(@() project(payload, ... + 'ArtifactKind', 'bad kind'), ... + 'pf2:identity:invalidArtifactKind'); + testCase.verifyError(@() project(payload, ... + 'SchemaVersion', uint64(0)), ... + 'pf2:identity:invalidSchemaVersion'); + testCase.verifyError(@() project(payload, ... + 'Projection', ''), ... + 'pf2:identity:invalidProjection'); + end + + function testVerifyExpectedAcceptsExactDigest(testCase) + digest = project(struct('value', 1)); + testCase.verifyWarningFree(@() ... + pf2_base.identity.verifyExpected(digest, digest, ... + 'Kind', 'recipe')); + end + + function testVerifyExpectedFailsClosedOnMismatch(testCase) + expected = project(struct('value', 1)); + observed = project(struct('value', 2)); + + testCase.verifyError(@() pf2_base.identity.verifyExpected( ... + expected, observed, 'Kind', 'source'), ... + 'pf2:identity:fingerprintMismatch'); + end + + function testVerifyExpectedRejectsMalformedDigest(testCase) + observed = project(struct('value', 1)); + testCase.verifyError(@() pf2_base.identity.verifyExpected( ... + 'not-a-digest', observed, 'Kind', 'source'), ... + 'pf2:identity:invalidHash'); + end + end +end + +function digest = project(value, varargin) + defaults = { ... + 'ArtifactKind', 'pf2.processing.recipe', ... + 'SchemaVersion', uint64(1), ... + 'Projection', 'scientific-content-v1', ... + 'AllowNonFinite', false}; + names = string(defaults(1:2:end)); + suppliedNames = string(varargin(1:2:end)); + for i = 1:numel(suppliedNames) + idx = find(strcmpi(names, suppliedNames(i)), 1); + if ~isempty(idx) + defaults{2 * idx} = varargin{2 * i}; + else + defaults(end + 1:end + 2) = varargin(2 * i - 1:2 * i); %#ok + end + end + digest = pf2_base.identity.hashProjection(value, defaults{:}); +end diff --git a/+pf2_base/+tests/+unit/IdentityTransportRoundtripTest.m b/+pf2_base/+tests/+unit/IdentityTransportRoundtripTest.m new file mode 100644 index 00000000..ad5f335b --- /dev/null +++ b/+pf2_base/+tests/+unit/IdentityTransportRoundtripTest.m @@ -0,0 +1,175 @@ +classdef IdentityTransportRoundtripTest < matlab.unittest.TestCase +%IDENTITYTRANSPORTROUNDTRIPTEST Phase-0 MAT/JSON identity proof spike. + + properties (Access = private) + TempDir char = '' + end + + methods (TestMethodSetup) + function createTemporaryDirectory(testCase) + testCase.TempDir = tempname; + mkdir(testCase.TempDir); + end + end + + methods (TestMethodTeardown) + function removeTemporaryDirectory(testCase) + if ~isempty(testCase.TempDir) && isfolder(testCase.TempDir) + rmdir(testCase.TempDir, 's'); + end + end + end + + methods (Test) + function testCanonicalLanguageRoundTripsLosslessly(testCase) + missingScalar = string(missing); + value = struct( ... + 'uint64Value', intmax('uint64'), ... + 'singleMatrix', reshape(single([0.1 0.2 0.3 0.4]), [2 2]), ... + 'logicalColumn', logical([1; 0]), ... + 'shapedEmpty', zeros(0, 2, 'int16'), ... + 'cellValue', {{uint8(1), 'two'}}, ... + 'strings', ["", missingScalar, "near infrared"]); + + json = pf2_base.identity.encodeJsonTransport(value); + decoded = pf2_base.identity.decodeJsonTransport(json); + + testCase.verifyEqual(canonical(decoded), canonical(value)); + testCase.verifyClass(decoded.uint64Value, 'uint64'); + testCase.verifyClass(decoded.singleMatrix, 'single'); + testCase.verifySize(decoded.shapedEmpty, [0 2]); + testCase.verifyTrue(ismissing(decoded.strings(2))); + testCase.verifyEqual(decoded.strings(1), ""); + end + + function testCompactAndPrettyJsonHaveOneSemanticIdentity(testCase) + recipe = pf2_base.tests.fixtures.identityRecipeSpike(); + compact = pf2_base.identity.encodeJsonTransport(recipe); + pretty = pf2_base.identity.encodeJsonTransport(recipe, ... + 'PrettyPrint', true); + + testCase.verifyNotEqual(compact, pretty); + compactRecipe = pf2_base.identity.decodeJsonTransport(compact); + prettyRecipe = pf2_base.identity.decodeJsonTransport(pretty); + testCase.verifyEqual(recipeHash(compactRecipe), ... + recipeHash(prettyRecipe)); + testCase.verifyEqual(recipeHash(compactRecipe), recipeHash(recipe)); + end + + function testPermittedNonFiniteValuesRoundTripCanonically(testCase) + value = [NaN, Inf, -Inf]; + + testCase.verifyError(@() ... + pf2_base.identity.encodeJsonTransport(value), ... + 'pf2:identity:nonFinite'); + json = pf2_base.identity.encodeJsonTransport(value, ... + 'AllowNonFinite', true); + decoded = pf2_base.identity.decodeJsonTransport(json); + testCase.verifyEqual( ... + pf2_base.identity.canonicalBytes(decoded, ... + 'AllowNonFinite', true), ... + pf2_base.identity.canonicalBytes(value, ... + 'AllowNonFinite', true)); + end + + function testSignedZeroRoundTripsThroughJsonAsPositiveZero(testCase) + % Negative zero is finite, so it must round-trip through the + % default JSON transport with no AllowNonFinite escape and + % normalize to +0, mirroring canonicalBytes' signed-zero + % contract. This is deliberately separate from the non-finite + % path above, where -0 was previously (and misleadingly) bundled + % in with NaN/Inf despite being finite. + negativeZero = typecast(bitshift(uint64(1), 63), 'double'); + testCase.assertEqual(negativeZero, 0); + + json = pf2_base.identity.encodeJsonTransport(negativeZero); + decoded = pf2_base.identity.decodeJsonTransport(json); + + testCase.verifyEqual( ... + pf2_base.identity.canonicalBytes(decoded), ... + pf2_base.identity.canonicalBytes(0.0)); + end + + function testMatAndJsonRoundTripToSameRecipeHash(testCase) + artifact = pf2_base.tests.fixtures.identityRecipeSpike(); %#ok + compressedPath = fullfile(testCase.TempDir, 'compressed.mat'); + plainPath = fullfile(testCase.TempDir, 'plain.mat'); + save(compressedPath, 'artifact', '-v7'); + save(plainPath, 'artifact', '-v7', '-nocompression'); + + fromCompressed = load(compressedPath, 'artifact'); + fromPlain = load(plainPath, 'artifact'); + fromJson = pf2_base.identity.decodeJsonTransport( ... + pf2_base.identity.encodeJsonTransport(artifact)); + + expected = recipeHash(artifact); + testCase.verifyEqual(recipeHash(fromCompressed.artifact), expected); + testCase.verifyEqual(recipeHash(fromPlain.artifact), expected); + testCase.verifyEqual(recipeHash(fromJson), expected); + testCase.verifyEqual(canonical(fromCompressed.artifact), ... + canonical(fromJson)); + end + + function testAuthorshipAndDisplayMetadataAreOutsideScientificHash(testCase) + recipe = pf2_base.tests.fixtures.identityRecipeSpike(); + changed = recipe; + changed.artifactId = 'different-draft-id'; + changed.displayName = 'Different display name'; + changed.authoredFrom.rawName = 'renamed-origin'; + changed.authoringDelta{end + 1} = 'GUI-only note'; + changed.citations = {'Different descriptive citation text'}; + + testCase.verifyNotEqual(canonical(recipe), canonical(changed)); + testCase.verifyEqual(recipeHash(recipe), recipeHash(changed)); + end + + function testScientificMutationsChangeHash(testCase) + recipe = pf2_base.tests.fixtures.identityRecipeSpike(); + variants = cell(1, 6); + variants{1} = recipe; + variants{1}.scientific.context.fixedDpf = 6.0; + variants{2} = recipe; + variants{2}.scientific.rawSteps = flipud( ... + variants{2}.scientific.rawSteps); + variants{3} = recipe; + variants{3}.scientific.oxySteps.parameters.value = 0.11; + variants{4} = recipe; + variants{4}.scientific.oxySteps.parameters.unit = 'rad/s'; + variants{5} = recipe; + variants{5}.scientific.qcPolicy.enabled = false; + variants{6} = recipe; + variants{6}.scientific.inputContract.wavelengthsNm(2) = uint16(850); + + original = recipeHash(recipe); + for i = 1:numel(variants) + testCase.verifyNotEqual(recipeHash(variants{i}), original, ... + sprintf('Scientific mutation %d did not change identity.', i)); + end + end + + function testInvalidOrTamperedJsonFailsClosed(testCase) + recipe = pf2_base.tests.fixtures.identityRecipeSpike(); + json = pf2_base.identity.encodeJsonTransport(recipe); + decoded = jsondecode(json); + decoded.format = 'unknown-format'; + + testCase.verifyError(@() ... + pf2_base.identity.decodeJsonTransport(jsonencode(decoded)), ... + 'pf2:identity:invalidJsonTransport'); + testCase.verifyError(@() ... + pf2_base.identity.decodeJsonTransport('{"format":"pf2-canonical-json-v1"}'), ... + 'pf2:identity:invalidJsonTransport'); + end + end +end + +function bytes = canonical(value) + bytes = pf2_base.identity.canonicalBytes(value); +end + +function digest = recipeHash(recipe) + digest = pf2_base.identity.hashProjection(recipe.scientific, ... + 'ArtifactKind', 'pf2.processing.recipe', ... + 'SchemaVersion', recipe.schemaVersion, ... + 'Projection', 'scientific-content-v1'); +end diff --git a/+pf2_base/+tests/+unit/InputIdentityPreflightTest.m b/+pf2_base/+tests/+unit/InputIdentityPreflightTest.m new file mode 100644 index 00000000..fa37ec27 --- /dev/null +++ b/+pf2_base/+tests/+unit/InputIdentityPreflightTest.m @@ -0,0 +1,171 @@ +classdef InputIdentityPreflightTest < matlab.unittest.TestCase +%INPUTIDENTITYPREFLIGHTTEST Fail-closed import/content identity contracts. + + properties (Access = private) + TempDir char = '' + end + + methods (TestMethodSetup) + function createTemporaryDirectory(testCase) + testCase.TempDir = tempname; + mkdir(testCase.TempDir); + end + end + + methods (TestMethodTeardown) + function removeTemporaryDirectory(testCase) + if ~isempty(testCase.TempDir) && isfolder(testCase.TempDir) + rmdir(testCase.TempDir, 's'); + end + end + end + + methods (Test) + function testSourceMismatchPreventsImporterInvocation(testCase) + source = writeBytes(testCase.TempDir, 'input.bin', uint8(1:8)); + spy = pf2_base.tests.fixtures.IdentityImportSpy(sampleProjection()); + wrong = ['sha256:' repmat('0', 1, 64)]; + + testCase.verifyError(@() pf2_base.identity.preflightImport( ... + @() spy.invoke(), {source}, @(data) data, ... + 'LogicalNames', {'input.bin'}, ... + 'ExpectedSource', wrong), ... + 'pf2:identity:fingerprintMismatch'); + testCase.verifyEqual(spy.CallCount, 0, ... + 'The importer ran before source identity was verified.'); + end + + function testImportedMismatchStopsAfterOneImport(testCase) + source = writeBytes(testCase.TempDir, 'input.bin', uint8(1:8)); + spy = pf2_base.tests.fixtures.IdentityImportSpy(sampleProjection()); + wrong = ['sha256:' repmat('0', 1, 64)]; + + testCase.verifyError(@() pf2_base.identity.preflightImport( ... + @() spy.invoke(), {source}, @(data) data, ... + 'LogicalNames', {'input.bin'}, ... + 'ExpectedImported', wrong), ... + 'pf2:identity:fingerprintMismatch'); + testCase.verifyEqual(spy.CallCount, 1, ... + 'Imported identity must be checked after exactly one import.'); + end + + function testMatchingExpectationsReturnIdentityEnvelope(testCase) + source = writeBytes(testCase.TempDir, 'input.bin', uint8(1:8)); + data = sampleProjection(); + sourceIdentity = pf2_base.identity.sourceFingerprint({source}, ... + 'LogicalNames', {'input.bin'}); + importedIdentity = ... + pf2_base.identity.importedContentFingerprint(data); + spy = pf2_base.tests.fixtures.IdentityImportSpy(data); + + [actual, identity] = pf2_base.identity.preflightImport( ... + @() spy.invoke(), {source}, @(value) value, ... + 'LogicalNames', {'input.bin'}, ... + 'ExpectedSource', sourceIdentity, ... + 'ExpectedImported', importedIdentity); + + testCase.verifyEqual(actual, data); + testCase.verifyEqual(spy.CallCount, 1); + testCase.verifyEqual(identity.source.digest, sourceIdentity.digest); + testCase.verifyEqual(identity.imported.digest, importedIdentity.digest); + testCase.verifyEqual(identity.sourceValidation.status, 'matched'); + testCase.verifyEqual(identity.importedValidation.status, 'matched'); + end + + function testSourceMutationDuringImportFails(testCase) + source = writeBytes(testCase.TempDir, 'input.bin', uint8(1:8)); + spy = pf2_base.tests.fixtures.IdentityImportSpy(sampleProjection()); + spy.OnRun = @() overwriteBytes(source, uint8(2:9)); + + testCase.verifyError(@() pf2_base.identity.preflightImport( ... + @() spy.invoke(), {source}, @(data) data, ... + 'LogicalNames', {'input.bin'}), ... + 'pf2:identity:fingerprintMismatch'); + testCase.verifyEqual(spy.CallCount, 1); + end + + function testInMemoryInputStillGetsImportedFingerprint(testCase) + data = sampleProjection(); + spy = pf2_base.tests.fixtures.IdentityImportSpy(data); + + [~, identity] = pf2_base.identity.preflightImport( ... + @() spy.invoke(), {}, @(value) value); + + testCase.verifyFalse(identity.source.available); + testCase.verifyEqual(identity.source.reason, ... + 'no-stable-byte-source'); + testCase.verifyTrue(startsWith(string(identity.imported.digest), ... + "sha256:")); + testCase.verifyEqual(spy.CallCount, 1); + end + + function testExpectedSourceCannotBeVerifiedForInMemoryInput(testCase) + spy = pf2_base.tests.fixtures.IdentityImportSpy(sampleProjection()); + expected = ['sha256:' repmat('0', 1, 64)]; + + testCase.verifyError(@() pf2_base.identity.preflightImport( ... + @() spy.invoke(), {}, @(value) value, ... + 'ExpectedSource', expected), ... + 'pf2:identity:sourceUnavailable'); + testCase.verifyEqual(spy.CallCount, 0); + end + + function testEmptyStringSourceExpectationMeansNotRequested(testCase) + spy = pf2_base.tests.fixtures.IdentityImportSpy(sampleProjection()); + + [~, identity] = pf2_base.identity.preflightImport( ... + @() spy.invoke(), {}, @(value) value, ... + 'ExpectedSource', ""); + + testCase.verifyEqual(identity.sourceValidation.status, 'unavailable'); + testCase.verifyEqual(spy.CallCount, 1); + end + + function testImportedScientificMutationChangesDigest(testCase) + original = sampleProjection(); + changed = original; + changed.raw(2, 2) = changed.raw(2, 2) + eps(changed.raw(2, 2)); + + first = pf2_base.identity.importedContentFingerprint(original); + second = pf2_base.identity.importedContentFingerprint(changed); + testCase.verifyNotEqual(first.digest, second.digest); + end + + function testImportedFingerprintAllowsCanonicalNaN(testCase) + projection = sampleProjection(); + projection.raw(1) = NaN; + + identity = pf2_base.identity.importedContentFingerprint(projection); + testCase.verifyTrue(startsWith(string(identity.digest), "sha256:")); + testCase.verifyError(@() ... + pf2_base.identity.importedContentFingerprint(projection, ... + 'AllowNonFinite', false), ... + 'pf2:identity:nonFinite'); + end + end +end + +function data = sampleProjection() + data = struct(); + data.raw = reshape(1:12, [4 3]); + data.time = (0:3).'; + data.fs = 1; + data.fchMask = logical([1 1 0]); + data.channels = uint16([1 2 3]); + data.units = 'intensity'; +end + +function path = writeBytes(root, name, bytes) + path = fullfile(root, name); + overwriteBytes(path, bytes); +end + +function overwriteBytes(path, bytes) + fid = fopen(path, 'wb'); + if fid < 0 + error('pf2:tests:identity:cannotCreateFixture', ... + 'Could not write temporary fixture %s.', path); + end + cleanup = onCleanup(@() fclose(fid)); %#ok + fwrite(fid, bytes, 'uint8'); +end diff --git a/+pf2_base/+tests/+unit/PublicProcessingContextTest.m b/+pf2_base/+tests/+unit/PublicProcessingContextTest.m index c6e99c90..c7cf4cb1 100644 --- a/+pf2_base/+tests/+unit/PublicProcessingContextTest.m +++ b/+pf2_base/+tests/+unit/PublicProcessingContextTest.m @@ -3,7 +3,7 @@ % % Covers the user-facing subclass: usable-from-bare-construction (no % fromGlobals bootstrap), Name-Value configuration with aliases, - % independent copy(), the process() convenience, and recipe round-trips. + % independent copy(), and the process() convenience. % % Usage: % results = runtests('pf2_base.tests.unit.PublicProcessingContextTest'); @@ -103,19 +103,7 @@ function testPlainAssignmentAliases(testCase) testCase.verifyEqual(ctx.subjectAge, 77); end - %% Recipe round-trip - - function testRecipeRoundTripIsUsable(testCase) - ctx = pf2.ProcessingContext('DPFmode', 'Fixed', 'FixedDPF', 6.5, ... - 'SubjectAge', 40); - recipe = ctx.toStruct(); - rebuilt = pf2.ProcessingContext.fromRecipe(recipe); - testCase.verifyEqual(rebuilt.dpfMode, 'Fixed'); - testCase.verifyEqual(rebuilt.dpfFixedValue, 6.5, 'AbsTol', 1e-12); - testCase.verifyEqual(rebuilt.subjectAge, 40); - % And it is immediately usable (libraries loaded). - testCase.verifyWarningFree(@() rebuilt.setRawMethod('None')); - end + %% Internal context-struct compatibility function testBaseFromStructReloadsMethods(testCase) % The base fromStruct fix: a deserialized context must be usable. @@ -125,31 +113,6 @@ function testBaseFromStructReloadsMethods(testCase) testCase.verifyWarningFree(@() r.setRawMethod('None')); end - function testRecipeRoundTripRestoresNamedMethod(testCase) - % Exercises the non-'None' branch of loadMethods/fromStruct: a real - % method name must survive serialize -> deserialize and stay set. - ctx = pf2.ProcessingContext(); - sections = ctx.rawMethodsLib.cfg.Sections; - named = sections(~strcmpi(sections, 'None')); - testCase.assumeNotEmpty(named); % need at least one real method - name = named{1}; - ctx.setRawMethod(name); - - rebuilt = pf2.ProcessingContext.fromRecipe(ctx.toStruct()); - testCase.verifyEqual(rebuilt.rawMethodName, name); - testCase.verifyTrue(isfield(rebuilt.rawMethod, 'name')); - testCase.verifyEqual(rebuilt.rawMethod.name, name); - end - - function testFromRecipeRestoresRootPath(testCase) - % fromRecipe delegates to fromStruct, so rootPath round-trips. - ctx = pf2.ProcessingContext(); - recipe = ctx.toStruct(); - recipe.rootPath = fullfile('some', 'custom', 'root'); - rebuilt = pf2.ProcessingContext.fromRecipe(recipe); - testCase.verifyEqual(rebuilt.rootPath, fullfile('some', 'custom', 'root')); - end - %% process() convenience function testProcessProducesHbAndIsolatesAge(testCase) diff --git a/+pf2_base/+tests/+unit/SourceFingerprintTest.m b/+pf2_base/+tests/+unit/SourceFingerprintTest.m new file mode 100644 index 00000000..d76e4653 --- /dev/null +++ b/+pf2_base/+tests/+unit/SourceFingerprintTest.m @@ -0,0 +1,276 @@ +classdef SourceFingerprintTest < matlab.unittest.TestCase +% SOURCEFINGERPRINTTEST Contract tests for exact source-byte identity. +% +% Run with: +% runtests('pf2_base.tests.unit.SourceFingerprintTest') +% +% See also: pf2_base.identity.sha256File, +% pf2_base.identity.sourceFingerprint + + properties (Access = private) + TempDir char = '' + end + + methods (TestMethodSetup) + function createTemporaryDirectory(testCase) + testCase.TempDir = tempname; + mkdir(testCase.TempDir); + end + end + + methods (TestMethodTeardown) + function removeTemporaryDirectory(testCase) + if ~isempty(testCase.TempDir) && isfolder(testCase.TempDir) + rmdir(testCase.TempDir, 's'); + end + end + end + + methods (Test) + function testSHA256FileMatchesByteHash(testCase) + path = writeBytes(testCase.TempDir, 'abc.bin', uint8([97 98 99])); + fromFile = pf2_base.identity.sha256File(path); + fromBytes = pf2_base.identity.sha256Bytes(uint8([97 98 99])); + + testCase.verifyEqual(fromFile, fromBytes); + end + + function testSourceFingerprintStructContract(testCase) + path = writeBytes(testCase.TempDir, 'source.bin', uint8(1:8)); + fp = pf2_base.identity.sourceFingerprint({path}, ... + 'LogicalNames', {'source.bin'}); + + testCase.verifyClass(fp, 'struct'); + for field = {'profile', 'algorithm', 'available', 'reason', ... + 'digest', 'entries'} + testCase.verifyTrue(isfield(fp, field{1}), ... + sprintf('Source fingerprint is missing field %s.', field{1})); + end + testCase.verifyEqual(lower(string(fp.algorithm)), "sha256"); + testCase.verifyTrue(fp.available); + testCase.verifyEqual(fp.reason, ''); + + % An available fingerprint must share the unavailable-source field + % shape so mixed batches concatenate into one struct array. + unavailable = struct( ... + 'profile', 'pf2.input.source.v1', ... + 'algorithm', 'sha256', ... + 'available', false, ... + 'reason', 'no-stable-byte-source', ... + 'digest', '', ... + 'entries', struct('logicalName', {}, ... + 'byteLength', {}, 'byteDigest', {})); + testCase.verifyEqual(numel([fp; unavailable]), 2); + testCase.verifyTrue(~isempty(regexp(char(string(fp.digest)), ... + '^sha256:[0-9a-f]{64}$', 'once'))); + testCase.verifyEqual(numel(fp.entries), 1); + testCase.verifyEqual(fp.entries(1).logicalName, 'source.bin'); + testCase.verifyClass(fp.entries(1).byteLength, 'uint64'); + testCase.verifyEqual(fp.entries(1).byteLength, uint64(8)); + testCase.verifyTrue(~isempty(regexp( ... + char(string(fp.entries(1).byteDigest)), ... + '^sha256:[0-9a-f]{64}$', 'once'))); + end + + function testSourceFingerprintGoldenDigest(testCase) + % Frozen known-answer digest for a fixed file (the NIST "abc" + % bytes) under a fixed logical name. Anchors the durable + % source-manifest identity across runs and releases: a silent + % change to the manifest projection, canonical envelope, or the + % sha256File/hashProjection wiring rewrites this digest and fails + % here. The per-file byte digest is independently the published + % NIST SHA-256("abc") vector, pinning the file-hash layer too. + path = writeBytes(testCase.TempDir, 'abc.bin', uint8([97 98 99])); + fp = pf2_base.identity.sourceFingerprint({path}, ... + 'LogicalNames', {'abc.bin'}); + + manifestExpected = "sha256:e6e2be00fc87b4748cac18b8a5d8df49" + ... + "1bd74df5bd42a96a78fea1aafb59cc0f"; + byteExpected = "sha256:ba7816bf8f01cfea414140de5dae2223" + ... + "b00361a396177a9cb410ff61f20015ad"; + + testCase.verifyEqual(string(fp.digest), manifestExpected); + testCase.verifyEqual(string(fp.entries(1).byteDigest), ... + byteExpected); + end + + function testSingleFileSeparatesManifestAndByteDigests(testCase) + path = writeBytes(testCase.TempDir, 'single.bin', uint8(0:31)); + fp = pf2_base.identity.sourceFingerprint({path}, ... + 'LogicalNames', {'single.bin'}); + rawDigest = pf2_base.identity.sha256File(path); + + testCase.verifyEqual(fp.entries(1).byteDigest, rawDigest); + testCase.verifyNotEqual(fp.digest, rawDigest); + end + + function testMultiFileEnumerationOrderIsInsignificant(testCase) + first = writeBytes(testCase.TempDir, 'physical-a.bin', uint8(1:5)); + second = writeBytes(testCase.TempDir, 'physical-b.bin', uint8(6:10)); + + forward = pf2_base.identity.sourceFingerprint({first, second}, ... + 'LogicalNames', {'inputs/a.bin', 'inputs/b.bin'}); + reverse = pf2_base.identity.sourceFingerprint({second, first}, ... + 'LogicalNames', {'inputs/b.bin', 'inputs/a.bin'}); + + testCase.verifyEqual(forward.digest, reverse.digest); + testCase.verifyEqual({forward.entries.logicalName}, ... + {'inputs/a.bin', 'inputs/b.bin'}); + testCase.verifyEqual({reverse.entries.logicalName}, ... + {'inputs/a.bin', 'inputs/b.bin'}); + end + + function testPhysicalRootAndModificationTimeAreExcluded(testCase) + rootA = fullfile(testCase.TempDir, 'root-a'); + rootB = fullfile(testCase.TempDir, 'root-b'); + mkdir(rootA); + mkdir(rootB); + bytes = uint8([10 20 30 40]); + pathA = writeBytes(rootA, 'data.bin', bytes); + pathB = writeBytes(rootB, 'renamed-physical-file.bin', bytes); + + % Give the second copy deliberately different filesystem metadata. + try + javaFile = javaObject('java.io.File', pathB); + javaFile.setLastModified(javaFile.lastModified() - 60000); + catch + % Identity must remain path/metadata independent even in a + % MATLAB configuration without Java filesystem access. + end + + first = pf2_base.identity.sourceFingerprint({pathA}, ... + 'LogicalNames', {'dataset/data.bin'}); + relocated = pf2_base.identity.sourceFingerprint({pathB}, ... + 'LogicalNames', {'dataset/data.bin'}); + + testCase.verifyEqual(first.digest, relocated.digest); + end + + function testByteMutationChangesFileAndSourceDigests(testCase) + path = writeBytes(testCase.TempDir, 'mutable.bin', uint8([1 2 3])); + beforeFile = pf2_base.identity.sha256File(path); + beforeSource = pf2_base.identity.sourceFingerprint({path}, ... + 'LogicalNames', {'mutable.bin'}); + + writeBytes(testCase.TempDir, 'mutable.bin', uint8([1 2 4])); + afterFile = pf2_base.identity.sha256File(path); + afterSource = pf2_base.identity.sourceFingerprint({path}, ... + 'LogicalNames', {'mutable.bin'}); + + testCase.verifyNotEqual(beforeFile, afterFile); + testCase.verifyNotEqual(beforeSource.digest, afterSource.digest); + end + + function testLogicalNameParticipatesInManifestIdentity(testCase) + path = writeBytes(testCase.TempDir, 'physical.bin', uint8([4 5 6])); + first = pf2_base.identity.sourceFingerprint({path}, ... + 'LogicalNames', {'first.bin'}); + renamed = pf2_base.identity.sourceFingerprint({path}, ... + 'LogicalNames', {'renamed.bin'}); + + testCase.verifyNotEqual(first.digest, renamed.digest); + end + + function testLogicalSeparatorsNormalize(testCase) + path = writeBytes(testCase.TempDir, 'physical.bin', uint8([4 5 6])); + slash = pf2_base.identity.sourceFingerprint({path}, ... + 'LogicalNames', {'folder/data.bin'}); + backslash = pf2_base.identity.sourceFingerprint({path}, ... + 'LogicalNames', {'folder\data.bin'}); + + testCase.verifyEqual(slash.digest, backslash.digest); + end + + function testDuplicateLogicalNamesRejected(testCase) + first = writeBytes(testCase.TempDir, 'one.bin', uint8(1)); + second = writeBytes(testCase.TempDir, 'two.bin', uint8(2)); + + testCase.verifyError(@() pf2_base.identity.sourceFingerprint( ... + {first, second}, 'LogicalNames', {'same.bin', 'same.bin'}), ... + 'pf2:identity:duplicateLogicalName'); + end + + function testDuplicateNamesAfterNormalizationRejected(testCase) + first = writeBytes(testCase.TempDir, 'one.bin', uint8(1)); + second = writeBytes(testCase.TempDir, 'two.bin', uint8(2)); + + testCase.verifyError(@() pf2_base.identity.sourceFingerprint( ... + {first, second}, 'LogicalNames', ... + {'folder/same.bin', 'folder\same.bin'}), ... + 'pf2:identity:duplicateLogicalName'); + end + + function testLogicalNamesUseUnicodeNFC(testCase) + path = writeBytes(testCase.TempDir, 'source.bin', uint8(1)); + composed = ['caf' char(hex2dec('00E9')) '.bin']; + decomposed = ['cafe' char(hex2dec('0301')) '.bin']; + first = pf2_base.identity.sourceFingerprint({path}, ... + 'LogicalNames', {composed}); + second = pf2_base.identity.sourceFingerprint({path}, ... + 'LogicalNames', {decomposed}); + + testCase.verifyEqual(first.digest, second.digest); + end + + function testInvalidLogicalNamesRejected(testCase) + path = writeBytes(testCase.TempDir, 'source.bin', uint8(1)); + invalidNames = {'', '/absolute/source.bin', '../source.bin', ... + 'folder/../source.bin', 'C:\absolute\source.bin'}; + + for i = 1:numel(invalidNames) + name = invalidNames{i}; + identifier = captureError(@() ... + pf2_base.identity.sourceFingerprint({path}, ... + 'LogicalNames', {name})); + testCase.verifyNotEqual(identifier, "", ... + sprintf('Invalid logical name "%s" was accepted.', name)); + testCase.verifyTrue(startsWith(identifier, "pf2:identity:"), ... + sprintf('Unexpected error identifier for "%s": %s', ... + name, identifier)); + end + end + + function testMissingFileRejected(testCase) + missing = fullfile(testCase.TempDir, 'does-not-exist.bin'); + testCase.verifyError(@() pf2_base.identity.sha256File(missing), ... + 'pf2:identity:fileNotFound'); + testCase.verifyError(@() pf2_base.identity.sourceFingerprint( ... + {missing}, 'LogicalNames', {'missing.bin'}), ... + 'pf2:identity:fileNotFound'); + end + + function testPathAndLogicalNameCountsMustMatch(testCase) + path = writeBytes(testCase.TempDir, 'source.bin', uint8(1)); + identifier = captureError(@() ... + pf2_base.identity.sourceFingerprint({path}, ... + 'LogicalNames', {'one.bin', 'two.bin'})); + + testCase.verifyNotEqual(identifier, ""); + testCase.verifyTrue(startsWith(identifier, "pf2:identity:")); + end + end +end + +function path = writeBytes(root, relativePath, bytes) + path = fullfile(root, relativePath); + parent = fileparts(path); + if ~isfolder(parent) + mkdir(parent); + end + fid = fopen(path, 'wb'); + if fid < 0 + error('pf2:tests:identity:cannotCreateFixture', ... + 'Could not create temporary fixture %s.', path); + end + cleanup = onCleanup(@() fclose(fid)); %#ok + fwrite(fid, bytes, 'uint8'); +end + +function identifier = captureError(fcn) + try + fcn(); + identifier = ""; + catch ME + identifier = string(ME.identifier); + end +end diff --git a/+pf2_base/+tests/buildSuite.m b/+pf2_base/+tests/buildSuite.m new file mode 100644 index 00000000..33abbc2c --- /dev/null +++ b/+pf2_base/+tests/buildSuite.m @@ -0,0 +1,130 @@ +function suite = buildSuite(lane) +%BUILDSUITE Authoritative test-suite builder for processFNIRS2. +% +% suite = pf2_base.tests.buildSuite(lane) returns a matlab.unittest.TestSuite +% for the requested execution lane. This is the single source of truth for +% which tests exist: every runner (runAllTests, runCI) and every CI gate +% should build its suite here rather than discovering packages ad hoc. +% +% Syntax: +% suite = pf2_base.tests.buildSuite() % 'full' (default) +% suite = pf2_base.tests.buildSuite(lane) +% +% Inputs: +% lane - (char/string, optional) execution lane. Default 'full'. +% 'full' - every headless-safe TestCase under pf2_base.tests, +% excluding the UI lane. This is the main gate. +% 'ui' - only tests that create app/uifigure surfaces and may +% need a display; run these in their own invocation. +% 'clean' - only the global-state isolation tests, meant to be +% run in a fresh MATLAB process (matlab -batch) so a +% clean-process replay is genuinely clean. This lane is +% a subset of 'full'; it also runs inside 'full'. +% 'identity' - canonical encoding, domain-separated hashing, +% source-manifest fingerprinting, and fail-closed +% identity verification contracts. This lane is a +% subset of 'full' and is intended for the supported- +% release/platform CI matrix. +% 'all' - full + ui (everything discoverable). +% +% Outputs: +% suite - matlab.unittest.TestSuite (possibly empty for a lane). +% +% Description: +% Discovery is authoritative: TestSuite.fromPackage('pf2_base.tests', +% 'IncludingSubpackages', true) sweeps in the root testExperiment class +% AND every unit/integration subpackage. Non-TestCase helpers (fixtures, +% synthetic generators, golden utilities, quick scripts) are ignored by +% fromPackage automatically, so they never leak into a gated run. +% +% Lane membership for the UI and clean-process lanes is declared once, in +% the registries below, keyed by fully-qualified test class name. Moving +% a test between lanes is a one-line edit here; the runners do not encode +% any lane knowledge of their own. +% +% Example: +% % Confirm the root testExperiment class is actually discovered +% s = pf2_base.tests.buildSuite('full'); +% names = unique(regexprep(string({s.Name}), '/.*$', '')); +% assert(any(contains(names, 'testExperiment'))); +% +% See also: pf2_base.tests.runAllTests, pf2_base.tests.runCI, +% pf2_base.tests.runQuickTests, matlab.unittest.TestSuite + + import matlab.unittest.TestSuite; + + if nargin < 1 || isempty(lane) + lane = 'full'; + end + lane = lower(string(lane)); + + % --- Lane registries (single source of truth) ----------------------- + % Tests that build app/uifigure surfaces. Kept out of 'full' so the main + % gate does not depend on a display; run explicitly via the 'ui' lane. + % Add more as a list: ["a.Test", "b.Test"]. + uiClasses = "pf2_base.tests.integration.MethodsEditorTest"; + + % Tests that assert global-state isolation (PF2/setF untouched) or a clean + % replay. Intended to be launched in a fresh process for a true guarantee. + cleanClasses = [ ... + "pf2_base.tests.unit.ProcessingContextTest", ... + "pf2_base.tests.unit.PublicProcessingContextTest", ... + "pf2_base.tests.integration.ProcessingContextIntegrationTest" ]; + + % Durable semantic-identity contract. Keep this explicit so the sparse + % cross-release/platform matrix does not need to run the entire suite. + identityClasses = [ ... + "pf2_base.tests.unit.IdentityCanonicalEncodingTest", ... + "pf2_base.tests.unit.IdentityNamedHashTest", ... + "pf2_base.tests.unit.IdentityTransportRoundtripTest", ... + "pf2_base.tests.unit.InputIdentityPreflightTest", ... + "pf2_base.tests.unit.SourceFingerprintTest" ]; + + % --- Authoritative discovery ---------------------------------------- + full = TestSuite.fromPackage('pf2_base.tests', 'IncludingSubpackages', true); + + if isempty(full) + error('pf2:tests:buildSuite:noTests', ... + ['No tests discovered under pf2_base.tests. Ensure the toolbox is ', ... + 'on the path and test classes inherit from matlab.unittest.TestCase.']); + end + + parents = regexprep(string({full.Name}), '/.*$', ''); + isUI = ismember(parents, uiClasses); + isClean = ismember(parents, cleanClasses); + isIdentity = ismember(parents, identityClasses); + + % Warn if a registry entry never matched a discovered test: a rename or a + % typo would otherwise silently shrink a lane. + warnMissing(uiClasses, parents, 'ui'); + warnMissing(cleanClasses, parents, 'clean'); + warnMissing(identityClasses, parents, 'identity'); + + switch lane + case {"full", "headless"} + suite = full(~isUI); + case "ui" + suite = full(isUI); + case {"clean", "clean-process", "cleanprocess"} + suite = full(isClean); + case "identity" + suite = full(isIdentity); + case "all" + suite = full; + otherwise + error('pf2:tests:buildSuite:unknownLane', ... + ['Unknown lane "%s". Use one of: full | ui | clean | ', ... + 'identity | all.'], lane); + end +end + +function warnMissing(registry, parents, laneName) + known = unique(parents); + for i = 1:numel(registry) + if ~ismember(registry(i), known) + warning('pf2:tests:buildSuite:unknownLaneMember', ... + 'Lane "%s" registers "%s", which matched no discovered test.', ... + laneName, registry(i)); + end + end +end diff --git a/+pf2_base/+tests/generateGoldenFiles.m b/+pf2_base/+tests/generateGoldenFiles.m index c1878ab9..8ea44e4d 100644 --- a/+pf2_base/+tests/generateGoldenFiles.m +++ b/+pf2_base/+tests/generateGoldenFiles.m @@ -71,21 +71,10 @@ function generateGoldenFiles(varargin) % --- Golden 2: TDDR raw, no oxy --- fprintf('Generating: fNIR2000_TDDR_None...\n'); -% Check if x5_TDDR method exists -global PF2 -rawMethods = PF2.myRawMethods.cfg.Sections; -if ismember('x5_TDDR', rawMethods) - pf2.methods.raw.setMethod('x5_TDDR'); - rawMethodName = 'x5_TDDR'; -else - % Create a temporary TDDR method - pf2.methods.raw.create('golden_TDDR', ... - {struct('f', 'pf2_MotionCorrectTDDR', 'args', {{'x', 'fs'}}, ... - 'argvals', {{'x', 'fs'}}, 'output', 'x')}, ... - 'Replace', true); - pf2.methods.raw.setMethod('golden_TDDR'); - rawMethodName = 'golden_TDDR'; -end +% Use the registered OD_TDDR raw method (portable: the golden stores a real +% method name so GoldenFileTest can reproduce it without a temp method). +rawMethodName = 'OD_TDDR'; +pf2.methods.raw.setMethod(rawMethodName); pf2.methods.oxy.setMethod('None'); processed = processFNIRS2(data); @@ -101,11 +90,6 @@ function generateGoldenFiles(varargin) save(outFile, '-struct', 'golden'); fprintf(' Saved: %s\n', outFile); -% Clean up temp method if created -if strcmp(rawMethodName, 'golden_TDDR') - pf2.methods.raw.delete('golden_TDDR'); -end - % --- Golden 3: TDDR function in isolation --- fprintf('Generating: pf2_TDDR_fNIR2000...\n'); od = pf2_Intensity2OD(data.raw); diff --git a/+pf2_base/+tests/runAllTests.m b/+pf2_base/+tests/runAllTests.m index d9d63973..74297091 100644 --- a/+pf2_base/+tests/runAllTests.m +++ b/+pf2_base/+tests/runAllTests.m @@ -18,16 +18,20 @@ % .Duration - Test execution time in seconds % % Description: -% This function discovers and runs all tests in the pf2_base.tests.unit and -% pf2_base.tests.integration packages. Tests are executed with verbose output -% (Verbosity level 3) showing detailed progress information. +% This function runs the 'full' lane from pf2_base.tests.buildSuite: every +% headless-safe TestCase under pf2_base.tests, including the root +% testExperiment class, excluding only the UI lane. Tests execute with +% verbose output (Verbosity level 3). % -% The function prints a summary showing total tests, passed count, -% and failed count. If any tests fail, their names are listed. +% The function REPORTS results (total/passed/failed/incomplete, with the +% names of any failed or incomplete tests) but does not itself signal +% failure to the shell. To gate CI so the process exits non-zero on any +% failure OR incomplete test, use pf2_base.tests.runCI instead. % -% Test Organization: +% Test Organization (see pf2_base.tests.buildSuite for the authoritative map): % pf2_base.tests.unit - Unit tests for individual functions % pf2_base.tests.integration - Integration tests for multi-component workflows +% pf2_base.tests.testExperiment - Experiment class tests (root of the package) % % Example: % % Run all tests and examine results @@ -44,8 +48,8 @@ % disp(failedResults(i).Details); % end % -% See also: pf2_base.tests.runQuickTests, matlab.unittest.TestSuite, -% matlab.unittest.TestRunner +% See also: pf2_base.tests.buildSuite, pf2_base.tests.runCI, +% pf2_base.tests.runQuickTests, matlab.unittest.TestSuite % Author: processFNIRS2 Development Team % Version: 8.1 @@ -60,24 +64,22 @@ fprintf('Running at: %s\n', datestr(now)); fprintf('MATLAB Version: %s\n\n', version); - % Build test suites from packages - fprintf('Discovering tests...\n'); - - unitSuite = TestSuite.fromPackage('pf2_base.tests.unit', 'IncludingSubpackages', true); - integrationSuite = TestSuite.fromPackage('pf2_base.tests.integration', 'IncludingSubpackages', true); - - % Combine suites - suite = [unitSuite, integrationSuite]; + % Build the authoritative headless suite (unit + integration + the root + % testExperiment class, excluding the UI lane). Discovery lives in one + % place, pf2_base.tests.buildSuite, so this runner cannot silently drift + % out of sync with what the test tree actually contains. + fprintf('Discovering tests (lane: full)...\n'); + suite = pf2_base.tests.buildSuite('full'); if isempty(suite) - fprintf('No tests found in pf2_base.tests.unit or pf2_base.tests.integration packages.\n'); + fprintf('No tests discovered under pf2_base.tests.\n'); fprintf('Ensure test classes inherit from matlab.unittest.TestCase.\n'); results = matlab.unittest.TestResult.empty; return; end - fprintf('Found %d unit tests, %d integration tests\n', ... - numel(unitSuite), numel(integrationSuite)); + nClasses = numel(unique(regexprep(string({suite.Name}), '/.*$', ''))); + fprintf('Found %d tests across %d classes\n', numel(suite), nClasses); fprintf('\n'); % Configure runner with verbose output @@ -122,5 +124,10 @@ else fprintf('Some tests FAILED or were INCOMPLETE.\n'); end + + % Other lanes are run separately (they need a display or a fresh process): + fprintf('\nOther lanes: pf2_base.tests.runCI(''ui''), runCI(''clean''), runCI(''quick'').\n'); + fprintf('To gate CI on the result (non-zero exit on any failure/incomplete):\n'); + fprintf(' matlab -batch "pf2_base.tests.runCI(''full'')"\n'); fprintf('=== Test Run Complete ===\n'); end diff --git a/+pf2_base/+tests/runCI.m b/+pf2_base/+tests/runCI.m new file mode 100644 index 00000000..8d8a00a4 --- /dev/null +++ b/+pf2_base/+tests/runCI.m @@ -0,0 +1,78 @@ +function results = runCI(lane) +%RUNCI Run a test lane as a gate: error on any failed OR incomplete test. +% +% pf2_base.tests.runCI(lane) builds the requested lane with +% pf2_base.tests.buildSuite, runs it, prints a one-line summary, and throws +% if any test failed or was incomplete. It is the single command CI and +% automation should use: +% +% matlab -batch "pf2_base.tests.runCI('full')" +% +% Under `matlab -batch`, an uncaught error makes the process exit non-zero, +% so the gate actually fails the build. This replaces the fragile hand-written +% `exit(any([results.Failed]))`, which ignored Incomplete tests (errored in +% fixtures/setup) and let a broken suite report success. +% +% Syntax: +% results = pf2_base.tests.runCI() % 'full' (default) +% results = pf2_base.tests.runCI(lane) +% +% Inputs: +% lane - (char/string, optional) 'full' (default) | 'ui' | 'clean' | +% 'identity' | 'all' | 'quick'. All except 'quick' map to +% pf2_base.tests.buildSuite lanes; 'quick' runs the lightweight +% script validation lane (pf2_base.tests.runQuickTests). +% +% Outputs: +% results - matlab.unittest.TestResult array for the lane (empty for the +% 'quick' lane, which is script-based). Only returned when +% requested; on failure the function errors regardless. +% +% Example: +% % Local pre-push gate over everything, including UI: +% pf2_base.tests.runCI('all'); +% +% See also: pf2_base.tests.buildSuite, pf2_base.tests.runAllTests, +% pf2_base.tests.runQuickTests + + import matlab.unittest.TestRunner; + + if nargin < 1 || isempty(lane) + lane = 'full'; + end + lane = lower(string(lane)); + + % Quick lane is script-based; delegate and gate on its boolean status. + if lane == "quick" + ok = pf2_base.tests.runQuickTests(); + results = matlab.unittest.TestResult.empty; + if ~ok + error('pf2:tests:runCI:quickFailures', ... + 'Quick validation lane reported failures. See output above.'); + end + return; + end + + suite = pf2_base.tests.buildSuite(lane); + if isempty(suite) + error('pf2:tests:runCI:emptyLane', ... + 'Lane "%s" contains no tests. Nothing to gate on.', lane); + end + + runner = TestRunner.withTextOutput('Verbosity', 2); + results = runner.run(suite); + + nTotal = numel(results); + nPassed = sum([results.Passed]); + nFailed = sum([results.Failed]); + nIncomplete = sum([results.Incomplete]); + + fprintf('\n[runCI:%s] total=%d passed=%d failed=%d incomplete=%d duration=%.1fs\n', ... + lane, nTotal, nPassed, nFailed, nIncomplete, sum([results.Duration])); + + if nFailed > 0 || nIncomplete > 0 + error('pf2:tests:runCI:failures', ... + 'Lane "%s" is not green: %d failed, %d incomplete (of %d). See output above.', ... + lane, nFailed, nIncomplete, nTotal); + end +end diff --git a/+pf2_base/+tests/runQuickTests.m b/+pf2_base/+tests/runQuickTests.m index ebd92a69..35445ce2 100644 --- a/+pf2_base/+tests/runQuickTests.m +++ b/+pf2_base/+tests/runQuickTests.m @@ -1,13 +1,21 @@ -function runQuickTests() +function ok = runQuickTests() %RUNQUICKTESTS Run quick validation scripts for processFNIRS2 % -% pf2_base.tests.runQuickTests() executes lightweight validation scripts that -% verify core functionality without the full unit testing framework. +% ok = pf2_base.tests.runQuickTests() executes lightweight validation scripts +% that verify core functionality without the full unit testing framework. % These scripts are designed for rapid sanity checks during development. % % Syntax: +% ok = pf2_base.tests.runQuickTests() % pf2_base.tests.runQuickTests() % +% Outputs: +% ok - logical; true only if every quick test passed. When called with +% NO output captured, a failure instead raises an error so that +% `matlab -batch "pf2_base.tests.runQuickTests()"` exits non-zero +% and CI can gate on it. Capture ok to inspect the status without +% throwing. +% % Description: % This function runs a series of quick validation scripts located in % the pf2_base.tests.quick package. Each script tests a specific functional @@ -107,7 +115,9 @@ function runQuickTests() totalDuration = sum(cellfun(@(r) r.duration, results)); fprintf('Total duration: %.2f seconds\n', totalDuration); - if failCount == 0 + ok = (failCount == 0); + + if ok fprintf('All %d tests passed.\n', passCount); else fprintf('Results: %d passed, %d failed\n', passCount, failCount); @@ -122,5 +132,14 @@ function runQuickTests() fprintf('\nRun individual tests for more details:\n'); fprintf(' pf2_base.tests.quick.testName()\n'); + + % Fail loudly when the caller did not capture the status, so a bare + % `matlab -batch "pf2_base.tests.runQuickTests()"` exits non-zero + % rather than reporting success with the failures merely printed. + if nargout == 0 + error('pf2:tests:runQuickTests:failures', ... + '%d of %d quick validation tests failed. See the list above.', ... + failCount, numTests); + end end end diff --git a/+pf2_base/Pipeline.m b/+pf2_base/Pipeline.m index 203715c7..c77bb77a 100644 --- a/+pf2_base/Pipeline.m +++ b/+pf2_base/Pipeline.m @@ -523,7 +523,7 @@ function disp(obj) function p = fromMethod(methodName, stage) % FROMMETHOD Build a Pipeline from an existing named method. % - % p = pf2_base.Pipeline.fromMethod('x5_TDDR', 'raw') + % p = pf2_base.Pipeline.fromMethod('OD_TDDR', 'raw') % p = pf2_base.Pipeline.fromMethod('takizawa_easy_lpf', 'oxy') % % Requires PF2 to be initialized. diff --git a/+pf2_base/ProcessingContext.m b/+pf2_base/ProcessingContext.m index 3953e59f..6911b7a0 100644 --- a/+pf2_base/ProcessingContext.m +++ b/+pf2_base/ProcessingContext.m @@ -130,7 +130,7 @@ function setRawMethod(obj, methodName) % SETRAWMETHOD Set the raw processing method by name % % Syntax: - % ctx.setRawMethod('x5_TDDR') + % ctx.setRawMethod('OD_TDDR') % % The method must exist in rawMethodsLib. diff --git a/+pf2_base/RawPipeline.m b/+pf2_base/RawPipeline.m index 8315938d..d734ae50 100644 --- a/+pf2_base/RawPipeline.m +++ b/+pf2_base/RawPipeline.m @@ -104,7 +104,7 @@ function warmupSetFDevice(data) function p = fromMethod(methodName) % FROMMETHOD Build a RawPipeline from an existing named method. % - % p = pf2_base.RawPipeline.fromMethod('x5_TDDR') + % p = pf2_base.RawPipeline.fromMethod('OD_TDDR') base = pf2_base.Pipeline.fromMethod(methodName, 'raw'); p = pf2_base.RawPipeline(base.name); diff --git a/+pf2_base/pf2_initialize.m b/+pf2_base/pf2_initialize.m index d0e57207..60749db3 100644 --- a/+pf2_base/pf2_initialize.m +++ b/+pf2_base/pf2_initialize.m @@ -107,13 +107,13 @@ function pf2_initialize() % working defaults out of the box. Failures are non-fatal. if firstTimeRaw || firstTimeOxy try - seeds = pf2.methods.seeds.list(); + seeds = pf2_base.methods.seeds.list(); for k = 1:numel(seeds) s = seeds(k); if strcmp(s.stage,'raw') && ~firstTimeRaw, continue; end if strcmp(s.stage,'oxy') && ~firstTimeOxy, continue; end try - p = feval(['pf2.methods.seeds.' s.stage '.' s.name]); + p = feval(['pf2_base.methods.seeds.' s.stage '.' s.name]); p.save(s.stage); fprintf('Seeded %s method: %s\n', s.stage, s.name); catch ME diff --git a/+pf2_base/pf2version.m b/+pf2_base/pf2version.m index 966cdffe..75135503 100644 --- a/+pf2_base/pf2version.m +++ b/+pf2_base/pf2version.m @@ -29,8 +29,8 @@ % % See also: pf2_initialize, processFNIRS2 -pf2ver='v0.9'; -dateStr='January 23 2026'; +pf2ver='v1.0.1'; +dateStr='July 14 2026'; verString=sprintf('processFNIRS2 Release %s\n',pf2ver); diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..73a0350d --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,70 @@ +name: tests + +# Manual MATLAB validation (run on demand from the Actions tab, not on push/PR +# yet). Runs the fast "quick" lane plus the durable semantic-identity contracts. +# Both gates exit non-zero on any failure or incomplete test. +# +# To enable automatic runs later, add push/pull_request triggers under `on:`. +# +# Notes: +# - matlab-actions/setup-matlab provisions MATLAB on GitHub-hosted runners. +# Public repositories can run batch MATLAB without a license; private repos +# need a MathWorks batch token exposed as the MLM_LICENSE_TOKEN secret +# (add `env: MLM_LICENSE_TOKEN: ${{ secrets.MLM_LICENSE_TOKEN }}` below). +# - To add a heavier lane (nightly), run the full gate: +# pf2_base.tests.runCI('full') +# in a separate scheduled workflow. The quick lane is kept fast on purpose. + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + quick-tests: + name: Quick lane (runCI('quick')) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up MATLAB + uses: matlab-actions/setup-matlab@v3 + with: + release: R2025b + - name: Run quick test lane + uses: matlab-actions/run-command@v3 + with: + command: "addpath(genpath('.')); pf2_base.tests.runCI('quick')" + + identity-contract: + name: Identity (${{ matrix.label }}) + strategy: + fail-fast: false + matrix: + include: + # R2025b is the repository's currently documented recommended release. + - label: R2025b / Linux + os: ubuntu-latest + release: R2025b + # `latest` exercises current MATLAB on every supported runner family + # without hard-coding a release that becomes stale each half-year. + - label: latest / Linux + os: ubuntu-latest + release: latest + - label: latest / Windows + os: windows-latest + release: latest + - label: latest / macOS + os: macos-latest + release: latest + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up MATLAB + uses: matlab-actions/setup-matlab@v3 + with: + release: ${{ matrix.release }} + - name: Run identity contract lane + uses: matlab-actions/run-command@v3 + with: + command: "addpath(genpath('.')); pf2_base.tests.runCI('identity')" diff --git a/.gitignore b/.gitignore index 9dc2b957..92d159d4 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ sampledata/sampleNIR_CH.mat !CONTRIBUTING.md !docs/*.md !examples/scripts/README.md +!mcp/*.md +!tests/golden/README.md internal/ site/ tests/benchmarks/data/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 93b0c275..0d2ed6c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ ### New Features +**Processing Methods:** +- New default oxy (Stage 3) methods, seeded via `pf2_base.methods.seeds.oxy.*`: `HPF` (0.01 Hz high-pass), `BPF` (0.01–0.1 Hz Butterworth band-pass), and `takizawa_easy` / `takizawa_hard` (Takizawa automatic channel rejection, lenient / strict criteria — Takizawa et al. 2008) +- Refreshed the processing-method reference tables to match the live registry + **Toolbox-Free Signal Processing (`+pf2_base/+external/`):** - First-party implementations of `butter`, `fir1`, `sgolay`, `sgolayfilt`, `medfilt1`, `zp2sos`, and the window functions (`hamming`, `hann`, `hanning`, `genCosWin`), so filtering and motion correction no longer require the MATLAB Signal Processing Toolbox - Filtering and motion-correction functions (`bpf`/`hpf`/`lpf`, `pf2_bpf_*`, `pf2_hpf`/`pf2_lpf`, `pf2_bandstop`, `pf2_MotionCorrectTDDR`, `pf2_MotionCorrectSplineSG`) route through the external math and `filtfilt_classic`/`filtfilt_piecewise`/`filtfilt_interp` diff --git a/CITATION.cff b/CITATION.cff index b15378f0..d1af11d9 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -7,8 +7,8 @@ abstract: >- signal processing, hemoglobin conversion, quality control, and group-level statistics. type: software -version: 1.0.0 -date-released: "2026-02-11" +version: 1.0.1 +date-released: "2026-07-14" license: GPL-3.0-or-later repository-code: "https://github.com/AdrianCurtin/processFNIRS2" url: "https://github.com/AdrianCurtin/processFNIRS2" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 41a609e2..c5f5da05 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,10 +37,23 @@ where practical — prefer the `pf2_base.external.*` equivalents (`butter`, `fir ## Running the tests ```matlab -pf2_base.tests.runAllTests() % full suite +pf2_base.tests.runAllTests() % report the full headless lane pf2_base.tests.runQuickTests() % fast subset for iterating ``` +To gate a run so it exits non-zero on any failed **or** incomplete test (what CI +should use), run the lane through `runCI` from `-batch`: + +```bash +matlab -batch "pf2_base.tests.runCI('full')" # main headless gate +matlab -batch "pf2_base.tests.runCI('ui')" # display-dependent tests +matlab -batch "pf2_base.tests.runCI('clean')" # global-isolation, fresh process +``` + +`pf2_base.tests.buildSuite` is the single source of truth for which tests exist; +`runAllTests` and `runCI` both build from it, so adding a test class anywhere +under `+pf2_base/+tests` makes it part of the gated run automatically. + Add or update tests in `+pf2_base/+tests` for any behavior you change. New features should ship with coverage; bug fixes should add a regression test. diff --git a/README.md b/README.md index c631202e..9e9eab48 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![MATLAB](https://img.shields.io/badge/MATLAB-R2025b-blue.svg) ![License](https://img.shields.io/badge/license-GPLv3-blue.svg) -![Version](https://img.shields.io/badge/version-1.0.0-green.svg) +![Version](https://img.shields.io/badge/version-1.0.1-green.svg) A modular MATLAB toolbox for functional Near-Infrared Spectroscopy (fNIRS) data analysis — covering the full workflow from raw device import through signal @@ -120,9 +120,9 @@ mydata = pf2.import.sampleData(); % sample recording with marker processed = processFNIRS2(mydata); % uses the default methods % To pick specific methods, browse the registered names and select one: -pf2.methods.raw.list(); % e.g. 'x2_lpf_smar', 'x5_TDDR' +pf2.methods.raw.list(); % e.g. 'OD_TDDR', 'OD_SMAR' pf2.methods.oxy.list(); -pf2.methods.raw.setMethod('x5_TDDR'); +pf2.methods.raw.setMethod('OD_TDDR'); processed = processFNIRS2(mydata); % process with the chosen method % Visualize @@ -186,7 +186,7 @@ Configure the raw/oxy method chains through the GUI or programmatically, or build a chain step-by-step with the Pipeline API (value objects — every mutating call returns a new copy): ```matlab -pf2.methods.raw.setMethod('x5_TDDR'); +pf2.methods.raw.setMethod('OD_TDDR'); pf2.methods.oxy.setMethod('takizawa_easy'); % Build a raw-stage pipeline from scratch @@ -202,14 +202,13 @@ Examples: `examples/scripts/example_pipeline_basics.m`, `examples/scripts/example_pipeline_custom_function.m`. ### Reproducible & parallel processing (Context) -A `ProcessingContext` bypasses global state, so settings stay isolated — ideal -for testing, `parfor`, and reproducibility: +A `pf2.ProcessingContext` bypasses global state, so settings stay isolated — +ideal for testing, `parfor`, and reproducibility: ```matlab +base = pf2.ProcessingContext('RawMethod', 'OD_TDDR', 'OxyMethod', 'takizawa_easy'); parfor i = 1:numSubjects - ctx = pf2_base.ProcessingContext.fromGlobals(); + ctx = base.copy(); % independent context per worker ctx.subjectAge = ages(i); - ctx.setRawMethod('x5_TDDR'); - ctx.setOxyMethod('takizawa_easy'); results{i} = processFNIRS2(data{i}, 'Context', ctx); end ``` @@ -312,7 +311,7 @@ processFNIRS2.m Main processing entry point pf2.m Convenience wrapper (self-heals the path) exploreFNIRS.m Group-level analysis GUI +pf2/ User-facing API (import, export, data, methods, settings, probe, qc) -+pf2_base/ Internal infrastructure, algorithms, and tests ++pf2_base/ Advanced APIs plus internal infrastructure, algorithms, and tests +exploreFNIRS/ Group analysis (core, connectivity, coupling, hyperscanning, stats, graph, plot, export) functions/ Signal-processing algorithm implementations (TDDR, SMAR, wavelet, ...) devices/ Device configuration files (.cfg) @@ -353,7 +352,7 @@ licenses are documented in [THIRD_PARTY_LICENSES.md](THIRD_PARTY_LICENSES.md). If you use processFNIRS2 in your research, please cite the software (machine-readable metadata is in [CITATION.cff](CITATION.cff)): -> Curtin, A., & Ayaz, H. (2026). *processFNIRS2* (version 1.0.0) +> Curtin, A., & Ayaz, H. (2026). *processFNIRS2* (version 1.0.1) > [Computer software]. https://github.com/AdrianCurtin/processFNIRS2 A companion publication and archival DOI will be added here when available. diff --git a/base_functions/getTakizawaRejection.m b/base_functions/getTakizawaRejection.m index 48cf3de7..8c5bb43f 100644 --- a/base_functions/getTakizawaRejection.m +++ b/base_functions/getTakizawaRejection.m @@ -151,7 +151,7 @@ fsHz=L2; %frequency at 1 hz point1hz=find(freqvec>0.1,1); %should be approximately L/(10*fs) - onehz=find(freqvec>1); % Should be approximately L/fs + onehz=find(freqvec>1,1); % first bin above 1 Hz (scalar); Should be approximately L/fs if(isempty(onehz)) onehz=length(freqvec); diff --git a/docs/API_REFERENCE.md b/docs/API_REFERENCE.md index b0ae9008..404adc78 100644 --- a/docs/API_REFERENCE.md +++ b/docs/API_REFERENCE.md @@ -2,7 +2,7 @@ ## Contents - [+pf2 — User-Facing Interface](#pf2-user-facing-interface) — import, export, data, methods, probe, QC, settings -- [+pf2_base — Internal Infrastructure](#pf2_base-internal-infrastructure) — pipelines, core processing, signal/wavelet +- [+pf2_base — Advanced and Internal Infrastructure](#pf2_base-advanced-and-internal-infrastructure) — pipelines, core processing, signal/wavelet - [+exploreFNIRS — Analysis & Visualization](#explorefnirs-analysis-and-visualization-gui) — Experiment class, stats, connectivity, hyperscanning, graph, report - [Signal Processing Functions (`/functions/`)](#signal-processing-functions-functions) - [Supported Devices (`/devices/`)](#supported-devices-devices) @@ -106,8 +106,9 @@ Each subpackage contains: - `removeFunction.m` - Remove a function from a method chain - `exportMethod.m` - Export method for sharing -**+pf2/+methods (shared):** -- `validateFunction.m` - Validate function compatibility with processing stages +> Method-authoring internals — function validation, the default-method seed +> factories, and the add/edit-function GUI helpers — live under +> `+pf2_base/+methods` (see below), not in the primary `pf2.methods` API. ### +pf2/+process - Processing Pipeline | Function | Purpose | @@ -211,9 +212,13 @@ Auto-attached as `data.device` by all import functions and `processFNIRS2`. --- -## +pf2_base (Internal Infrastructure) +## +pf2_base (Advanced and Internal Infrastructure) -Core processing and utility functions used internally. +`pf2_base` contains both supported expert/low-level interfaces and +implementation machinery. Advanced users may deliberately use documented +pipeline and scientific operations; callbacks, codecs, compatibility plumbing, +and incidental helpers are not thereby public contracts. The advanced API is +an explicit allowlist, not every callable symbol in the package. ### Pipeline Classes | Class | Purpose | @@ -228,6 +233,15 @@ Core processing and utility functions used internally. **Pipeline** is a value class — all mutating methods return a new copy. `toMethod()` converts to the legacy method struct consumed by `processStageRaw2OD` and `processStageFilterHb`. `fromMethod(name, stage)` reconstructs a Pipeline from an existing named method. +### +pf2_base/+methods — Method Extension & Seeds +Advanced method-authoring tools and the shipped default-method seed factories. +| Function | Purpose | +|----------|---------| +| `validateFunction.m` | Validate a processing-function configuration (function existence, argument consistency, output-variable name) before adding it to a method chain | +| `+seeds/list.m` | Enumerate the seed factories for a stage (`raw`/`oxy`) | +| `+seeds/+raw/*`, `+seeds/+oxy/*` | Zero-argument factories that build the shipped default methods (e.g. `OD_TDDR`, `LPF`); used by `pf2_initialize` and `pf2.methods.resetDefaults` | +| `+functions/add.m` / `+functions/edit.m` | Launch the GUI to define / edit processing-function metadata | + ### Core Initialization | Function | Purpose | |----------|---------| diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 9622d788..360122b9 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -196,7 +196,13 @@ flowchart TB | `+export` | `asNIR`, `asSNIRF`, `asBIDS`, `asTensor`, `export`. | | `+settings`, `+GUI` | Processing settings and GUI glue. `Device.m` (top level) is the device value class. | -### `+pf2_base/` — internal infrastructure & algorithms +### `+pf2_base/` — advanced and internal infrastructure & algorithms + +This package serves two roles: documented low-level interfaces for advanced +users, and implementation machinery used by the primary `pf2` workflow. Those +roles are distinguished by the API catalog and documentation tier; package +membership alone is not a support promise. + Top-level: `ProcessingContext`, the pipeline classes (`Pipeline`, `RawPipeline`, `OxyPipeline`, `PipelineFunction`), `pf2_initialize`, `loadDeviceCfg`, `normalizeMarkers`/`normalizeAux`, `hierarchicalAverage`. diff --git a/docs/CLI_UX_GUIDE.md b/docs/CLI_UX_GUIDE.md index 4e25d659..cce7472a 100644 --- a/docs/CLI_UX_GUIDE.md +++ b/docs/CLI_UX_GUIDE.md @@ -42,7 +42,8 @@ pf2.data.plot(processed); ```matlab % Load → Configure → Process → Visualize → Export data = pf2.import.importNIR('subject01.nir'); -pf2.methods.raw.setMethod('x2_lpf_smar'); % Motion correction + filtering +pf2.methods.raw.setMethod('OD_TDDR'); % Motion correction (raw stage) +pf2.methods.oxy.setMethod('BPF'); % Band-pass filter (oxy stage) processed = processFNIRS2(data); pf2.data.plot.oxy(processed); % Time series plot pf2.export.asSNIRF(processed, 'subject01_processed.snirf'); @@ -193,7 +194,7 @@ data = pf2.import.importNIR('myfile.nir'); % No guessing |------------------|------------------|----------| | `pf2.data.plot(data)` | `pf2.data.plot.oxy(data)` | Parent auto-detects: has HbO? → Oxy plot, else → Raw plot | | `pf2.methods()` | `pf2.methods.raw()` | Parent lists ALL methods; child lists only raw methods | -| `pf2.methods.raw.setMethod()` | `pf2.methods.raw.setMethod('x2_lpf_smar')` | No-arg prompts interactively; with-arg sets directly | +| `pf2.methods.raw.setMethod()` | `pf2.methods.raw.setMethod('OD_TDDR')` | No-arg prompts interactively; with-arg sets directly | ### Tab-Completion is Your Friend @@ -455,7 +456,7 @@ pf2.methods.raw.list() % Same as raw() pf2.methods.oxy.list() % Same as oxy() % Set active method -pf2.methods.raw.setMethod('x2_lpf_smar') % By name +pf2.methods.raw.setMethod('OD_TDDR') % By name pf2.methods.raw.setMethod(3) % By index pf2.methods.raw.setMethod() % Interactive @@ -464,7 +465,7 @@ pf2.methods.oxy.setMethod(2) pf2.methods.oxy.setMethod() % Method information -pf2.methods.raw.describeMethod('x2_lpf_smar') +pf2.methods.raw.describeMethod('OD_TDDR') pf2.methods.oxy.describeMethod('takizawa_easy') pf2.methods.describeCurrentMethods() @@ -481,17 +482,21 @@ pf2.methods.oxy.importMethods(filepath) | Raw Method | Description | When to Use | |------------|-------------|-------------| -| `x1_lpf` | Low-pass filter only | Clean data, minimal motion | -| `x2_lpf_smar` | LPF + SMAR motion correction | Moderate motion artifacts | -| `x5_TDDR` | TDDR motion correction | Spike artifacts | -| `x3_bpf` | Band-pass filter (0.008-0.1 Hz) | Isolate hemodynamic response | +| `None` | Optical-density conversion only | Already clean, minimal motion | +| `OD_TDDR` | TDDR motion correction | Spike / motion artifacts | +| `OD_SMAR` | SMAR sliding-window motion artifact rejection | Coefficient-of-variation spikes | + +> Band-pass filtering is applied at the oxy stage via the `BPF` method. For +> spline/wavelet raw chains, build a `pf2_base.RawPipeline` (see +> PROCESSING_PIPELINE.md); the underlying functions ship with the toolbox. | Oxy Method | Description | When to Use | |------------|-------------|-------------| | `None` | No post-processing | Already clean data | -| `takizawa_easy` | Lenient artifact rejection | Most data | -| `takizawa_hard` | Strict artifact rejection | Noisy data | -| `car` | Common Average Reference | Reduce global noise | +| `LPF` | Low-pass filter (0.1 Hz) | Smooth high-frequency noise | +| `BPF` | Band-pass filter (0.01–0.1 Hz) | Isolate hemodynamic response | +| `takizawa_easy` | Lenient channel rejection | Most data | +| `takizawa_hard` | Strict channel rejection | Noisy data | ### 5.9 Settings (`pf2.settings.*`) @@ -525,15 +530,18 @@ rawProcessed = pf2.process.processRaw(data) % Stage 1 only oxyProcessed = pf2.process.processOxy(data) % Stage 3 only ``` -### 5.11 GUI Configuration (`pf2.gui.*`) +### 5.11 GUI Configuration ```matlab -pf2.gui() % Launch main GUI -pf2.gui.configureRawMethods() % Raw method editor -pf2.gui.configureOxyMethods() % Oxy method editor -pf2.gui.functions() % Function library -pf2.gui.functions.add() % Add function -pf2.gui.functions.edit() % Edit function +pf2.gui() % Launch the main processing GUI + +% Method editors (also under 5.8 Method Management) +pf2.methods.raw.configureMethods() % Raw (Stage 1) method editor +pf2.methods.oxy.configureMethods() % Oxy (Stage 3) method editor + +% Advanced: define/edit a processing function's metadata (pf2_base extension API) +pf2_base.methods.functions.add() % Define a new processing function +pf2_base.methods.functions.edit('pf2_lpf') % Edit an existing function's definition ``` --- @@ -769,7 +777,6 @@ pf2.methods.oxy.editFunction pf2.methods.oxy.removeFunction pf2.methods.oxy.exportMethod pf2.methods.oxy.importMethod -pf2.methods.validateFunction pf2.methods.describeCurrentMethods pf2.settings.selectDevice @@ -784,12 +791,6 @@ pf2.settings.dpf.setFixedDPF pf2.process.process pf2.process.processRaw pf2.process.processOxy - -pf2.gui.configureRawMethods -pf2.gui.configureOxyMethods -pf2.gui.functions -pf2.gui.functions.add -pf2.gui.functions.edit ``` ### Layer 2 (exploreFNIRS) @@ -882,7 +883,7 @@ For users transitioning from the GUI to command-line: **In the GUI, you would:** 1. Click File → Open and select a .nir file -2. In the Method dropdown, select "x2_lpf_smar" +2. In the Method dropdown, select "OD_TDDR" 3. Set baseline to 10 seconds 4. Click "Process" 5. View the time series plot @@ -892,7 +893,7 @@ For users transitioning from the GUI to command-line: ```matlab % Steps 1-2: Import and configure data = pf2.import.importNIR('/path/to/file.nir'); -pf2.methods.raw.setMethod('x2_lpf_smar'); +pf2.methods.raw.setMethod('OD_TDDR'); % Step 3: Set baseline pf2.settings.baseline.setBaselineLength(10); diff --git a/docs/PROCESSING_PIPELINE.md b/docs/PROCESSING_PIPELINE.md index 15bebfde..6c8cd9e9 100644 --- a/docs/PROCESSING_PIPELINE.md +++ b/docs/PROCESSING_PIPELINE.md @@ -101,65 +101,44 @@ Output fNIRS struct with all stages ## Available Processing Methods ### Raw Processing Methods -Methods for Stage 1 processing (Raw → Optical Density): +Methods for Stage 1 processing (Raw → Optical Density). Each `OD_` method first +converts raw intensity to optical density, then applies the named motion +correction. Run `pf2.methods.raw.list()` for the authoritative live set. + +| Method | Pipeline | +|--------|----------| +| `None` | Optical-density conversion only (no motion/filter step) | +| `OD_TDDR` | Log→OD, then TDDR motion correction | +| `OD_SMAR` | Log→OD, then SMAR sliding-window motion artifact rejection | + +`OD_TDDR` and `OD_SMAR` are the shipped raw seeds (restored by +`pf2.methods.resetDefaults`). Band-pass filtering is applied at the oxy stage +via the `BPF` method (below). To build other raw chains — e.g. spline or +wavelet motion correction, or a raw-stage band-pass — use the Pipeline API; +the underlying functions (`pf2_MotionCorrectSpline`, `pf2_MotionCorrectSplineSG`, +`pf2_MotionCorrectWavelet`, `pf2_sSMART`, `pf2_bpf_butter`) ship with the toolbox: -| Method | Description | -|--------|-------------| -| `None` | No processing | -| `x1_lpf` | Low-pass filter only | -| `x1_lpf_mask` | Low-pass filter with masking | -| `x2_lpf_smar` | Low-pass filter + SMAR | -| `x2_lpf_smar_permissive` | LPF + SMAR (permissive settings) | -| `x2_lpf_smar_short` | LPF + SMAR (short window) | -| `x3_bpf` | Band-pass filter | -| `x3_bpf_008_1` | BPF (0.008-1 Hz) | -| `x3_bpf_008_1_mask` | BPF (0.008-1 Hz) with masking | -| `x4_bpf_smar` | Band-pass filter + SMAR | -| `x5_TDDR` | Temporal Derivative Distribution Repair | -| `x5_TDDR_mask` | TDDR with masking | -| `x5_TDDR_mask_smar` | TDDR + masking + SMAR | -| `x6_TDDR_lpf` | TDDR + low-pass filter | -| `x6_TDDR_lpf_mask` | TDDR + LPF + masking | -| `x6_lpf_MARA` | LPF + Movement Artifact Reduction | -| `x6_lpf_SMAR_SR` | LPF + SMAR (spline reconstruction) | -| `x6_lpf_TDDR` | LPF + TDDR | -| `x6_medfilt_TDDR` | Median filter + TDDR | -| `x7_kbWF` | Kalman-Butterworth Wiener filter | -| `x7_kbWF_lpf` | kbWF + low-pass filter | -| `x8_lpf_mask_subAmb` | LPF + masking + ambient subtraction | -| `x8_mask_subAmb_tddr` | Masking + ambient subtraction + TDDR | -| `x8_mask_subAmb_wave` | Masking + ambient subtraction + wavelet | -| `x8_mask_subAmb_wave_bpf` | Masking + ambient + wavelet + BPF | -| `x8_mask_subAmb_wave_detrend` | Masking + ambient + wavelet + detrend | -| `x8_mask_subAmb_wave_lpf` | Masking + ambient + wavelet + LPF | -| `x9_bpf_mask_subAmb` | BPF + masking + ambient subtraction | -| `x9_lpf_mask_subAmb_detrend` | LPF + masking + ambient + detrend | +```matlab +raw = pf2_base.RawPipeline('OD_Spline_BPF'); +raw = raw.add('pf2_Intensity2OD'); +raw = raw.add('pf2_MotionCorrectSpline'); +raw = raw.add('pf2_bpf_butter'); +raw.save('raw'); % register as a named raw method +``` ### Oxy Processing Methods -Methods for Stage 3 processing (Hemoglobin post-processing): +Methods for Stage 3 processing (hemoglobin post-processing). Run +`pf2.methods.oxy.list()` for the authoritative live set. -| Method | Description | -|--------|-------------| +| Method | Pipeline | +|--------|----------| | `None` | No processing | -| `bpf_butter` | Butterworth band-pass filter | -| `bpf_fir` | FIR band-pass filter | -| `car` | Common Average Reference | -| `hpf` | High-pass filter | -| `lpf_car` | Low-pass filter + CAR | -| `medfilt` | Median filter | -| `medfilt_car` | Median filter + CAR | -| `takizawa_easy` | Takizawa rejection (easy threshold) | -| `takizawa_easy_car` | Takizawa easy + CAR | -| `takizawa_easy_car_pca` | Takizawa easy + CAR + PCA | -| `takizawa_easy_lpf` | Takizawa easy + low-pass filter | -| `takizawa_easy_lpf_detrend` | Takizawa easy + LPF + detrend | -| `takizawa_easy_pca` | Takizawa easy + PCA | -| `takizawa_hard` | Takizawa rejection (hard threshold) | -| `takizawa_hard_car` | Takizawa hard + CAR | -| `takizawa_hard_car_pca` | Takizawa hard + CAR + PCA | -| `takizawa_hard_lpf` | Takizawa hard + low-pass filter | -| `takizawa_hard_lpf_detrend` | Takizawa hard + LPF + detrend | -| `takizawa_hard_pca` | Takizawa hard + PCA | +| `LPF` | Low-pass filter (0.1 Hz) | +| `HPF` | High-pass filter (0.01 Hz, drift removal) | +| `BPF` | Butterworth band-pass filter (0.01–0.1 Hz) | +| `LPF_ROI` | Low-pass filter + ROI averaging | +| `takizawa_easy` | Takizawa automatic channel rejection (lenient criteria) | +| `takizawa_hard` | Takizawa automatic channel rejection (strict criteria) | --- @@ -202,7 +181,7 @@ raw = raw.add('pf2_lpf', 'freq_cut', 0.2); m = raw.toMethod(); % Reconstruct from existing named method -raw = pf2_base.RawPipeline.fromMethod('x5_TDDR'); +raw = pf2_base.RawPipeline.fromMethod('OD_TDDR'); ``` ### Eager Conversion at Unpack Time @@ -259,12 +238,12 @@ Based on real research usage: **For motion-prone data (head movement):** ```matlab -% Raw stage: Motion correction + filtering -pf2.methods.raw.setMethod('x2_lpf_smar'); % LPF + SMAR -% or -pf2.methods.raw.setMethod('x5_TDDR'); % TDDR alone +% Raw stage: motion correction +pf2.methods.raw.setMethod('OD_TDDR'); % TDDR motion correction % or -pf2.methods.raw.setMethod('x6_lpf_TDDR'); % LPF + TDDR +pf2.methods.raw.setMethod('OD_SMAR'); % SMAR sliding-window rejection +% Add band-pass filtering at the oxy stage: +pf2.methods.oxy.setMethod('BPF'); % 0.01-0.1 Hz Butterworth % Spline interpolation — good for isolated, large artifacts % Can combine with wavelet for hybrid correction (spline first, then wavelet) @@ -277,9 +256,10 @@ pf2_MotionCorrectSplineSG(dod, fs); **For cleaner data (minimal motion):** ```matlab -% Raw stage: Filtering only -pf2.methods.raw.setMethod('x1_lpf'); % Low-pass only -pf2.methods.raw.setMethod('x3_bpf'); % Band-pass (0.008-0.1 Hz) +% Raw stage: light correction (or 'None' for OD conversion only) +pf2.methods.raw.setMethod('OD_TDDR'); % TDDR motion correction +% Do band-pass / low-pass filtering at the oxy stage instead: +pf2.methods.oxy.setMethod('BPF'); % Band-pass (0.01-0.1 Hz) ``` **For oxy stage (post-hemoglobin):** @@ -288,10 +268,10 @@ pf2.methods.raw.setMethod('x3_bpf'); % Band-pass (0.008-0.1 Hz) pf2.methods.oxy.setMethod('takizawa_easy'); % Lenient thresholds % Aggressive artifact rejection -pf2.methods.oxy.setMethod('takizawa_hard_lpf_detrend'); % Strict + detrend +pf2.methods.oxy.setMethod('takizawa_hard'); % Strict thresholds -% Spatial filtering -pf2.methods.oxy.setMethod('medfilt_car'); % Median + CAR +% Band-pass filtering +pf2.methods.oxy.setMethod('BPF'); % 0.01-0.1 Hz Butterworth ``` ## Common Processing Parameters @@ -324,7 +304,7 @@ When using `fitGLM` with drift regressors and AR-IRLS, explicit bandpass filteri Recommended GLM pipeline: ```matlab -pf2.methods.raw.setMethod('x5_TDDR_mask'); % Motion correction on OD +pf2.methods.raw.setMethod('OD_TDDR'); % Motion correction on OD pf2.methods.oxy.setMethod('None'); % No bandpass — GLM handles it % Design matrix includes drift regressors @@ -346,7 +326,7 @@ connResults = ex.connectivity('Method', 'wcoherence'); ### When Bandpass Filtering IS Appropriate -Use bandpass filtering (`bpf_butter`, 0.008–0.1 Hz) for: +Use bandpass filtering (`BPF`, 0.01–0.1 Hz) for: - **Block averaging** without GLM (epoch-based approach) - **Trial-by-trial amplitude extraction** (peak/mean in a time window) @@ -354,7 +334,7 @@ Use bandpass filtering (`bpf_butter`, 0.008–0.1 Hz) for: - **Visualization** of hemodynamic response shape ```matlab -pf2.methods.oxy.setMethod('bpf_butter'); % 0.008-0.1 Hz Butterworth +pf2.methods.oxy.setMethod('BPF'); % 0.01-0.1 Hz Butterworth ``` Short block-extracted segments (e.g., 30s) naturally limit low-frequency drift contribution, so bandpass is less critical for block-wise correlation analyses. @@ -488,7 +468,7 @@ path safe for `parfor` and byte-for-byte reproducible. ```matlab % Configure in one call — usable immediately, no fromGlobals() bootstrap ctx = pf2.ProcessingContext('DPFmode', 'Calc', 'SubjectAge', 30, ... - 'RawMethod', 'x5_TDDR'); + 'RawMethod', 'OD_TDDR'); result = ctx.process(data); % globals left untouched % equivalently: result = processFNIRS2(data, 'Context', ctx); diff --git a/docs/USAGE_EXAMPLES.md b/docs/USAGE_EXAMPLES.md index a64dd66c..7faa952d 100644 --- a/docs/USAGE_EXAMPLES.md +++ b/docs/USAGE_EXAMPLES.md @@ -54,7 +54,7 @@ processFNIRS2(); ## Workflow 3: Programmatic Method Selection ```matlab % Set methods (browse names with pf2.methods.raw.list() / pf2.methods.oxy.list()) -pf2.methods.raw.setMethod('x5_TDDR'); +pf2.methods.raw.setMethod('OD_TDDR'); pf2.methods.oxy.setMethod('takizawa_easy'); % Configure parameters @@ -316,17 +316,12 @@ segments = pf2.data.extractBlocks(data, blocks); % pf2.ProcessingContext is usable straight from construction (no fromGlobals % bootstrap) and accepts processFNIRS2-style Name-Value settings. ctx = pf2.ProcessingContext('DPFmode', 'Calc', 'SubjectAge', 30, ... - 'RawMethod', 'x5_TDDR', 'OxyMethod', 'takizawa_easy'); + 'RawMethod', 'OD_TDDR', 'OxyMethod', 'takizawa_easy'); % Process without modifying global state (either form works) result = ctx.process(data); % context as receiver % result = processFNIRS2(data, 'Context', ctx); % equivalent keyword form -% Save for reproducibility (-struct needs a variable, not an inline call) -settings = ctx.toStruct(); -save('analysis_settings.mat', '-struct', 'settings'); -% Rebuild a usable context later: ctx = pf2.ProcessingContext.fromRecipe(load('analysis_settings.mat')); - % Parallel processing with different ages. Configure ONCE, then take an % independent copy() per worker -- a plain ctx = base would alias one handle, % and fromGlobals() on a worker sees empty globals. diff --git a/examples/scripts/example_connectivity.m b/examples/scripts/example_connectivity.m index 575b9b77..df1b4bb8 100644 --- a/examples/scripts/example_connectivity.m +++ b/examples/scripts/example_connectivity.m @@ -28,7 +28,7 @@ fprintf('=== Step 0: Generate data, define ROIs, process ===\n'); [subjects, blockDefs] = pf2.import.sampleData.experiment('blocks'); -[rawMethod, oxyMethod] = pf2.import.sampleData.addDemoPipelines(); +[rawMethod, oxyMethod] = pf2_base.examples.addDemoPipelines(); % ROI definitions: 3 prefrontal regions nCh = size(subjects{1}.HbO, 2); diff --git a/examples/scripts/example_experiment_cli.m b/examples/scripts/example_experiment_cli.m index 56a969da..6312b15e 100644 --- a/examples/scripts/example_experiment_cli.m +++ b/examples/scripts/example_experiment_cli.m @@ -86,7 +86,7 @@ % Set processing methods - data is reprocessed on first aggregate() call, % then cached. Changing the method triggers reprocessing on next aggregate(). % Use pf2.methods.raw.list() / pf2.methods.oxy.list() to see available names. -[rawM, oxyM] = pf2.import.sampleData.addDemoPipelines(); +[rawM, oxyM] = pf2_base.examples.addDemoPipelines(); ex.settings.rawMethod = rawM; ex.settings.oxyMethod = oxyM; diff --git a/examples/scripts/example_glm_advanced.m b/examples/scripts/example_glm_advanced.m index 327e5a88..95090573 100644 --- a/examples/scripts/example_glm_advanced.m +++ b/examples/scripts/example_glm_advanced.m @@ -36,7 +36,7 @@ fprintf(' Recording duration: %.0fs\n', max(subjects{1}.time) - min(subjects{1}.time)); % Register demo pipelines and process all subjects -[rawMethod, oxyMethod] = pf2.import.sampleData.addDemoPipelines(); +[rawMethod, oxyMethod] = pf2_base.examples.addDemoPipelines(); nSubjects = length(subjects); for s = 1:nSubjects diff --git a/examples/scripts/example_glm_analysis.m b/examples/scripts/example_glm_analysis.m index b1f6845e..5d2ca27c 100644 --- a/examples/scripts/example_glm_analysis.m +++ b/examples/scripts/example_glm_analysis.m @@ -48,7 +48,7 @@ fprintf(' Recording duration: %.0fs\n', max(subjects{1}.time) - min(subjects{1}.time)); % Register demo pipelines -[rawMethod, oxyMethod] = pf2.import.sampleData.addDemoPipelines(); +[rawMethod, oxyMethod] = pf2_base.examples.addDemoPipelines(); %% Step 2: Create and configure GLMExperiment % diff --git a/examples/scripts/example_glm_connectivity.m b/examples/scripts/example_glm_connectivity.m index 18953f63..a266e07c 100644 --- a/examples/scripts/example_glm_connectivity.m +++ b/examples/scripts/example_glm_connectivity.m @@ -35,7 +35,7 @@ fprintf('=== Step 0: Generate data, define ROIs, process ===\n'); [subjects, blockDefs] = pf2.import.sampleData.experiment('blocks'); -[rawMethod, oxyMethod] = pf2.import.sampleData.addDemoPipelines(); +[rawMethod, oxyMethod] = pf2_base.examples.addDemoPipelines(); % ROI definitions: 3 prefrontal regions nCh = size(subjects{1}.HbO, 2); diff --git a/examples/scripts/example_hbica.m b/examples/scripts/example_hbica.m index a6342843..da235c80 100644 --- a/examples/scripts/example_hbica.m +++ b/examples/scripts/example_hbica.m @@ -29,7 +29,7 @@ fprintf('=== Step 1: Prepare paired data ===\n'); [subjects, blockDefs] = pf2.import.sampleData.experiment('blocks'); -[rawMethod, oxyMethod] = pf2.import.sampleData.addDemoPipelines(); +[rawMethod, oxyMethod] = pf2_base.examples.addDemoPipelines(); % Assign dyad metadata (2 dyads, 4 subjects) dyadIDs = {'D1', 'D1', 'D2', 'D2'}; diff --git a/examples/scripts/example_hyperscanning.m b/examples/scripts/example_hyperscanning.m index 44ddee3d..d9931cf7 100644 --- a/examples/scripts/example_hyperscanning.m +++ b/examples/scripts/example_hyperscanning.m @@ -28,7 +28,7 @@ fprintf('=== Step 0: Generate paired data, define ROIs, process ===\n'); [subjects, blockDefs] = pf2.import.sampleData.experiment('blocks'); -[rawMethod, oxyMethod] = pf2.import.sampleData.addDemoPipelines(); +[rawMethod, oxyMethod] = pf2_base.examples.addDemoPipelines(); % Assign dyad metadata dyadIDs = {'D1', 'D1', 'D2', 'D2'}; diff --git a/examples/scripts/tutorial_end_to_end.m b/examples/scripts/tutorial_end_to_end.m index 86a4c1df..f54ee00f 100644 --- a/examples/scripts/tutorial_end_to_end.m +++ b/examples/scripts/tutorial_end_to_end.m @@ -64,7 +64,7 @@ % defaultSubjectAge - age in years (for DPF calculation) % blLength - baseline length in seconds % blStartTime - baseline start (seconds from recording start) -% Raw_Method - name of raw-stage processing method (e.g. 'x5_TDDR') +% Raw_Method - name of raw-stage processing method (e.g. 'OD_TDDR') % Oxy_Method - name of oxy-stage processing method (e.g. 'lpf_car') fprintf('\n=== Part 2: Process ===\n'); diff --git a/functionSignatures.json b/functionSignatures.json index 7d61d33f..e8cb5a24 100644 --- a/functionSignatures.json +++ b/functionSignatures.json @@ -9,7 +9,7 @@ {"name":"rawMethod", "kind":"ordered", "type":["char"], "purpose":"Named raw-stage method"}, {"name":"oxyMethod", "kind":"ordered", "type":["char"], "purpose":"Named oxy-stage method"}, {"name":"ShowGUI", "kind":"namevalue", "type":["logical","scalar"], "purpose":"Force the GUI even when an output is captured"}, - {"name":"Context", "kind":"namevalue", "type":["pf2_base.ProcessingContext"], "purpose":"Isolated processing context (bypasses PF2/setF globals)"}, + {"name":"Context", "kind":"namevalue", "type":["pf2.ProcessingContext"], "purpose":"Isolated processing context (bypasses PF2/setF globals)"}, {"name":"SkipRaw", "kind":"namevalue", "type":["logical","scalar"], "purpose":"Skip the raw stage (oxy-only input)"}, {"name":"SkipOD", "kind":"namevalue", "type":["logical","scalar"], "purpose":"Return optical-density output from the raw stage"} ] diff --git a/functions/pf2_sSMART.m b/functions/pf2_sSMART.m index 8d6272a6..7100813b 100644 --- a/functions/pf2_sSMART.m +++ b/functions/pf2_sSMART.m @@ -6,9 +6,19 @@ % Optionally corrects DC baseline shifts caused by optode displacement % during artifacts before interpolating. % -% Reference: -% Ayaz, H. et al. (2010). Sliding-window motion artifact rejection for -% Functional Near-Infrared Spectroscopy. Conf Proc IEEE Eng Med Biol Soc. +% References: +% sSMART method: +% Curtin, A., & Ayaz, H. (2019). sSMART: statistical Sliding Motion +% Artifact Reconstruction Technique for functional Near-Infrared +% Spectroscopy. AHFE 2019 International Conference on Neuroergonomics +% and Cognitive Engineering, July 24-28, 2019, Washington, D.C., USA. +% Advances in Neuroergonomics and Cognitive Engineering, Springer. +% SMAR sliding-window detection stage (adapted; see pf2_SMAR2): +% Ayaz, H., Izzetoglu, M., Shewokis, P. A., & Onaral, B. (2010). +% Sliding-window motion artifact rejection for Functional Near-Infrared +% Spectroscopy. 2010 Annual International Conference of the IEEE +% Engineering in Medicine and Biology, 6567-6570. +% DOI: 10.1109/IEMBS.2010.5627113 % % Syntax: % x_recon = pf2_sSMART(x, fs) @@ -87,12 +97,25 @@ % [corrected, mask, idx] = pf2_sSMART(odData, 10, [], 4, 2, 10, 5, 'spline', true); % % Notes: -% - The SMAR2 detection stage is based on Ayaz et al. (2010). The -% interpolation-based reconstruction and DC-shift correction stages are -% original to processFNIRS2 and are not part of the published algorithm. +% - The sSMART method (statistical CV thresholding plus interpolated +% reconstruction) is described in Curtin & Ayaz (2019). The +% sliding-window detection it builds on derives from the SMAR algorithm +% of Ayaz et al. (2010) (see pf2_SMAR2). % - Interpolation fills gaps smoothly but cannot recover true underlying % neural signal. Short gaps (< few seconds) are typically acceptable; % long gaps should be treated with caution in downstream analysis. +% - DC-offset adjustment (ShiftCorrect, off by default): motion can +% permanently displace an optode, leaving a step-like shift in the +% baseline across the artifact. When enabled, each gap's pre/post +% baseline difference (mean over a 1 s window on each side) is removed +% from every sample after the gap, so the post-artifact signal realigns +% to the pre-artifact baseline. The correction is CUMULATIVE across gaps +% and applied INDEPENDENTLY per channel, so offsets accumulate over a +% recording and channels can drift relative to one another. It targets +% step-like displacement shifts only; if a gap straddles a genuine slow +% hemodynamic change, that real signal is removed along with the shift. +% Leave it off unless clear baseline jumps from optode displacement are +% visible, and inspect the reconstructed output before trusting it. % % See also: pf2_SMAR2, pf2_SMAR, pf2_MotionCorrectTDDR, interp1 diff --git a/mcp/README.md b/mcp/README.md new file mode 100644 index 00000000..192b4033 --- /dev/null +++ b/mcp/README.md @@ -0,0 +1,169 @@ +# Driving processFNIRS2 from an MCP client + +This folder lets an AI coding agent (Claude Code, Claude Desktop, VS Code / +Copilot, Codex) drive processFNIRS2 through the **official MATLAB MCP Server** +from MathWorks. The MCP server is generic — it starts MATLAB and runs code — so +the value added here is a curated **tool card** (`TOOL_CARD.md`) that teaches the +agent the correct pf2 idioms, plus ready-to-paste client configuration. + +> This is the "MATLAB as the MCP server" path. Publishing individual pf2 +> functions as typed MCP tools (via MATLAB Production Server) is a second, +> larger effort not covered here. + +--- + +## 1. Install the MATLAB MCP Server + +Prerequisites: MATLAB R2021a or later on the system `PATH` (this repo targets +R2025b). + +**macOS (Apple Silicon):** + +```bash +curl -L -o ~/bin/matlab-mcp-server \ + https://github.com/matlab/matlab-mcp-server/releases/latest/download/matlab-mcp-server-macos-arm64 +chmod +x ~/bin/matlab-mcp-server +``` + +macOS (Intel) uses the `...-macos-x64` asset. + +**Windows (x64):** download `matlab-mcp-server-windows-x64.exe` from the +[releases page](https://github.com/matlab/matlab-mcp-server/releases/latest) and +save it somewhere stable, e.g. `%USERPROFILE%\bin\matlab-mcp-server-windows-x64.exe`. +No `chmod` step is needed. In PowerShell: + +```powershell +mkdir $env:USERPROFILE\bin -Force +Invoke-WebRequest -Uri "https://github.com/matlab/matlab-mcp-server/releases/latest/download/matlab-mcp-server-windows-x64.exe" ` + -OutFile "$env:USERPROFILE\bin\matlab-mcp-server-windows-x64.exe" +``` + +**Linux (x64):** download `matlab-mcp-server-linux-x64` from the releases page and +`chmod +x` it. + +Alternatively, on any platform with Go installed, build from source: +`go install github.com/matlab/matlab-mcp-server/cmd/matlab-mcp-server@latest`. + +> **Path conventions used below.** Examples show macOS/Linux paths +> (`~/bin/matlab-mcp-server`, `/Applications/MATLAB_R2025b.app`). On Windows, +> substitute the `.exe` binary path (e.g. +> `C:\Users\YOU\bin\matlab-mcp-server-windows-x64.exe`) and the MATLAB install +> root (e.g. `C:\Program Files\MATLAB\R2025b`). In JSON files, Windows paths +> must escape backslashes (`C:\\Users\\YOU\\...`). + +## 2. Register it with your client + +**Claude Code** (run once, from anywhere): + +```bash +# macOS / Linux +claude mcp add --transport stdio matlab -- \ + ~/bin/matlab-mcp-server \ + --matlab-root /Applications/MATLAB_R2025b.app \ + --matlab-display-mode=nodesktop +``` + +```powershell +# Windows (PowerShell) — one line +claude mcp add --transport stdio matlab -- "C:\Users\YOU\bin\matlab-mcp-server-windows-x64.exe" --matlab-root "C:\Program Files\MATLAB\R2025b" --matlab-display-mode=nodesktop +``` + +**VS Code / Copilot** — copy [`mcp.example.json`](mcp.example.json) to +`.vscode/mcp.json` and fix the binary + MATLAB paths (the file carries both a +macOS/Linux and a Windows entry — keep the one for your OS). + +**Claude Desktop** — install the `matlab-mcp-server.mcpb` bundle from the +releases page (Settings → Extensions → Install Extension), then configure the +MATLAB root there. + +**Codex** (run once): + +```bash +# macOS / Linux +codex mcp add matlab -- \ + ~/bin/matlab-mcp-server \ + --matlab-root /Applications/MATLAB_R2025b.app \ + --matlab-display-mode=nodesktop +``` + +```powershell +# Windows (PowerShell) — one line +codex mcp add matlab -- "C:\Users\YOU\bin\matlab-mcp-server-windows-x64.exe" --matlab-root "C:\Program Files\MATLAB\R2025b" --matlab-display-mode=nodesktop +``` + +On Windows, also add `env_vars = ["WINDIR"]` to the server's entry in +`config.toml` (required for MATLAB to launch). + +**LM Studio** (v0.3.17+) — MCP servers are defined in an `mcp.json` that uses +Cursor's `mcpServers` notation. In the app, open the **Program** tab in the +right sidebar → **Install → Edit mcp.json**, then add: + +```json +{ + "mcpServers": { + "matlab": { + "command": "/Users/YOU/bin/matlab-mcp-server", + "args": [ + "--matlab-root", "/Applications/MATLAB_R2025b.app", + "--matlab-display-mode=nodesktop", + "--disable-telemetry=true" + ] + } + } +} +``` + +On Windows, use the `.exe` and escaped paths: + +```json +{ + "mcpServers": { + "matlab": { + "command": "C:\\Users\\YOU\\bin\\matlab-mcp-server-windows-x64.exe", + "args": [ + "--matlab-root", "C:\\Program Files\\MATLAB\\R2025b", + "--matlab-display-mode=nodesktop", + "--disable-telemetry=true" + ] + } + } +} +``` + +Saving the file reloads the server automatically. Note the key is `mcpServers` +here, versus `servers` in the VS Code format above. + +Useful server flags: `--initialize-matlab-on-startup=true` (warm start), +`--matlab-session-mode=existing` (attach to a MATLAB you already have open), +`--disable-telemetry=true`. + +## 3. Point the agent at pf2 + +The MATLAB MCP Server exposes these tools: + +| Tool | Purpose | +|------|---------| +| `detect_matlab_toolboxes` | Report MATLAB + toolbox versions | +| `check_matlab_code` | Static analysis (lint) of a `.m` file | +| `evaluate_matlab_code` | Run a MATLAB code string | +| `run_matlab_file` | Run a `.m` script | +| `run_matlab_test_file` | Run a `matlab.unittest` file | + +Before a session, make sure the agent has read [`TOOL_CARD.md`](TOOL_CARD.md) — +it carries the canonical import → process → blocks → export recipes, the +headless/GUI rules, and the "gotchas" that keep generated code correct. For +full API detail, the agent can use MATLAB `help` (e.g. `help processFNIRS2`) +and the `docs/` reference. + +## Notes / gotchas + +- **Add pf2 to the path first.** The agent's first `evaluate_matlab_code` call + in a fresh session should `cd` to this repo (or `addpath(genpath(...))`) so + `pf2.*` resolves. +- **Suppress the GUI.** Any call that captures an output (`out = processFNIRS2(data)`) + runs headless; a bare `processFNIRS2(data)` opens the GUI and will block the + MCP session. See the tool card. +- **3D renders** must use the `'savePath'` option, not `figure('Visible','off')` + + `saveas` — the latter is unreliable headless. +- **Long jobs**: `evaluate_matlab_code` is synchronous. For batch processing of + many subjects, have the agent write a `.m` script and use `run_matlab_file`. diff --git a/mcp/TOOL_CARD.md b/mcp/TOOL_CARD.md new file mode 100644 index 00000000..0fcd66ac --- /dev/null +++ b/mcp/TOOL_CARD.md @@ -0,0 +1,104 @@ +# processFNIRS2 — MCP Session Tool Card + +Condensed operating instructions for an AI agent driving processFNIRS2 through +the MATLAB MCP Server. Read this before generating pf2 code. It is a +session-oriented subset of the toolbox's processing conventions; for full API +detail use MATLAB `help` (e.g. `help processFNIRS2`) and the `docs/` reference. + +## Session setup (first call) + +Point MATLAB at the repo. Use the path for your OS (forward slashes work in +MATLAB on Windows too, but the drive prefix differs): + +```matlab +% macOS / Linux +cd /Users/adriancurtin/Documents/GitHub/processFNIRS2 % or addpath(genpath(pwd)) + +% Windows (either form works) +cd 'C:\Users\YOU\Documents\GitHub\processFNIRS2' +% cd C:/Users/YOU/Documents/GitHub/processFNIRS2 +``` + +`pf2.*`, `exploreFNIRS.*`, and `processFNIRS2` must resolve before anything else. +Prefer `fullfile(...)` when building paths in generated scripts so they stay +OS-portable, and write outputs to `tempdir` rather than a hard-coded `/tmp`. + +## GUI / headless rule (critical for MCP) + +`processFNIRS2` opens a **blocking GUI** unless you capture an output. In an MCP +session always capture output: + +```matlab +processed = processFNIRS2(data); % headless — correct for MCP +% processFNIRS2(data); % opens GUI — WILL HANG the session +``` + +Never pass `'ShowGUI', false`; capturing an output already suppresses it. + +## Canonical single-subject recipe + +```matlab +data = pf2.import.sampleData(); % or pf2.import.importSNIRF(path) +proc = processFNIRS2(data); % -> HbO/HbR/HbTotal/HbDiff/CBSI +blocks = pf2.data.defineBlocks(proc, 50, 15, 'Embed', false); +seg = pf2.data.extractBlocks(proc, blocks, 'PreTime', 5, 'PostTime', 15, 'SetT0', true); +ga = pf2.data.blockAverage(seg); % ga.HbO.Mean(:,ch), ga.time +``` + +Always set `PreTime`/`PostTime` explicitly on `extractBlocks`. + +## Batch / group + +```matlab +allData = pf2.import.importDirectory('data/', '*.snirf', 'Dir1', 'Group', 'Dir2', 'SubjectID'); +allData = processFNIRS2(allData); % processes each element +ex = exploreFNIRS.core.Experiment(allData); +ex.groupby({'Group'}); ex.aggregate(); ex.plotTemporal('Biomarkers', {'HbO'}); +``` + +## Headless QC (unattended runs) + +Importers default to "all channels good" when the channel-check GUI is +suppressed, so run the programmatic QC pipeline explicitly: + +```matlab +report = pf2.qc.pipeline.assess(data); +data = pf2.qc.pipeline.apply(data, report); +report = pf2.qc.snapshot(data, 'SaveDir', 'qc_out'); % dashboard + PSD + SCI PNGs +``` + +## Plots / renders (save, don't display) + +```matlab +pf2.data.plot.oxy(proc, 5, 'savePath', 'ch5.png'); % 2D: savePath ok +pf2.probe.plot.topo(proc, 'HbO', 'View', '3d', 'savePath', 'topo.png'); % 3D: MUST use savePath +``` + +Do **not** use `figure('Visible','off')` + `saveas`/`exportgraphics` for 3D +renders (`interpolateValues3D`, `topo 'View','3d'`, `project.*`) — unreliable +headless. Use the built-in `'savePath'`. + +## Export (incl. foundation-model tensor) + +```matlab +pf2.export.asSNIRF(proc, 'out.snirf'); +pf2.export.asBIDS(allData, 'bids_out/', 'Task', 'rest'); +pf2.export.asTensor(proc, 'rec.h5', 'Features', {'HbO','HbR'}, 'QC', true); +``` + +## Running tests via MCP + +Use the server's `run_matlab_test_file` tool on files in `+pf2_base/+tests/`, or: + +```matlab +pf2_base.tests.runQuickTests() +``` + +## Gotchas + +- `data.markers` is a **table** — read `data.markers.Code`, not by column index. +- `-batch`/`evaluate_matlab_code` code must be valid; prefer writing a `.m` file + and `run_matlab_file` for anything multi-statement or long-running. +- Reproducible/parallel runs: pass a public `pf2.ProcessingContext` as + `'Context'` (e.g. `processFNIRS2(data, 'Context', pf2.ProcessingContext())`) + to avoid touching the `PF2`/`setF` globals; use `ctx.copy()` per `parfor` worker. diff --git a/prefs/pf2_functions_default.cfg b/prefs/pf2_functions_default.cfg index 733232d7..57f1fabd 100644 --- a/prefs/pf2_functions_default.cfg +++ b/prefs/pf2_functions_default.cfg @@ -301,15 +301,37 @@ Nf_description = 'Filter length (number of taps)' [pf2_sSMART] Name = 'statistical Sliding Motion Artifact Reconstruction Technique : sSMART (beta)' -Description = 'Uses adaptive coefficient of variation thresholding and sgolay filtering to reconstruct original signal' +Description = 'Detects motion artifacts with adaptive coefficient-of-variation thresholding (SMAR2), then reconstructs the flagged regions by interpolating from surrounding clean data. Optionally corrects DC baseline shifts from optode displacement before interpolating.' validStages = [1,2] requiresOD = 1 -Arguments = {'x','fs','fChannelNumbers','tauArtifact','tauClean','minSeg','SGdegree'} +Role = 'motion' +Arguments = {'x','fs','fChannelNumbers','tauArtifact','tauClean','minSeg','ArtifactTime','InterpMethod','ShiftCorrect'} Output = {'x'} -tauArtifact = 3 -tauClean = 0 -minSeg = 5 -SGdegree = 2 +tauArtifact = 3 +tauClean = 1 +minSeg = 5 +ArtifactTime = 10 +InterpMethod = 'pchip' +ShiftCorrect = 0 +tauArtifact_type = 'double' +tauArtifact_range = [0,Inf] +tauArtifact_description = 'Artifact detection threshold multiplier (lower = more aggressive rejection)' +tauClean_type = 'double' +tauClean_range = [0,Inf] +tauClean_description = 'Clean-boundary threshold multiplier for artifact-region expansion (should be < tauArtifact)' +minSeg_type = 'int' +minSeg_range = [1,Inf] +minSeg_unit = 'samples' +minSeg_description = 'Minimum clean segment length; shorter clean gaps between artifacts are merged' +ArtifactTime_type = 'double' +ArtifactTime_range = [0,Inf] +ArtifactTime_unit = 's' +ArtifactTime_description = 'Expected artifact duration; sets the CV sliding-window length N = round(ArtifactTime*fs)' +InterpMethod_type = 'enum' +InterpMethod_choices = {'pchip','spline','linear','makima'} +InterpMethod_description = 'Gap-fill interpolation: pchip (shape-preserving default), spline (smoother, can overshoot), linear, makima' +ShiftCorrect_type = 'logical' +ShiftCorrect_description = 'Correct DC baseline shifts across artifact gaps before interpolating. For each gap the mean level 1 s before and after is measured and the offset removed so post-artifact data realigns to the pre-artifact baseline. Correction is cumulative across gaps and applied per channel; off by default. Use with caution: a gap straddling a genuine slow hemodynamic change can have that real signal removed.' [pf2_subtractAmbient] Name = 'Subtract Ambient' diff --git a/tests/golden/README.md b/tests/golden/README.md new file mode 100644 index 00000000..2eaa624c --- /dev/null +++ b/tests/golden/README.md @@ -0,0 +1,288 @@ +# Golden Files for Regression Testing + +Golden files store known-good outputs from processFNIRS2 processing pipelines. Tests compare current outputs against these reference files to detect unintended changes in behavior. + +## Directory Structure + +``` +tests/golden/ +├── README.md # This file +├── candidates/ # New outputs awaiting promotion to golden +├── processFNIRS2/ # Golden outputs from main processing pipeline +│ ├── fNIR2000_default.mat +│ └── ... +└── functions/ # Golden outputs from individual functions + ├── pf2_SMAR.mat + └── ... +``` + +## Golden File Contents + +Each `.mat` file contains: + +| Field | Description | +|-------|-------------| +| `output` | The function/pipeline output to verify | +| `inputHash` | SHA-256 hash of input data for validation | +| `version` | processFNIRS2 version that generated the file | +| `timestamp` | Generation timestamp (ISO 8601) | +| `params` | Parameters used (method names, settings) | +| `matlabVersion` | MATLAB version used for generation | + +## Generating Golden Files + +### Generate from sample data + +```matlab +% Load sample data +data = pf2.import.sampleData(); + +% Process with specific method configuration +pf2.methods.raw.setMethod('x2_lpf_smar'); +pf2.methods.oxy.setMethod('takizawa_easy'); +processed = processFNIRS2(data); + +% Create golden file +golden = struct(); +golden.output = processed; +golden.inputHash = computeHash(data); +golden.version = pf2_base.pf2version(); +golden.timestamp = datetime('now', 'Format', 'yyyy-MM-dd''T''HH:mm:ss'); +golden.params = struct('rawMethod', 'x2_lpf_smar', 'oxyMethod', 'takizawa_easy'); +golden.matlabVersion = version(); + +% Save to candidates directory first +save('tests/golden/candidates/processFNIRS2_sampleNIR_x2_lpf_smar.mat', '-struct', 'golden'); +``` + +### Generate for a specific function + +```matlab +% Test pf2_SMAR in isolation +data = pf2.import.sampleData(); +[corrected, mask] = pf2_SMAR(data.raw, data.fs); + +golden = struct(); +golden.output = struct('corrected', corrected, 'mask', mask); +golden.inputHash = computeHash(data.raw); +golden.version = pf2_base.pf2version(); +golden.timestamp = datetime('now', 'Format', 'yyyy-MM-dd''T''HH:mm:ss'); +golden.params = struct('fs', data.fs); +golden.matlabVersion = version(); + +save('tests/golden/candidates/pf2_SMAR_sampleNIR.mat', '-struct', 'golden'); +``` + +## Verifying Against Golden Files + +### Basic verification + +```matlab +function result = verifyGolden(goldenPath, actualOutput, tolerance) + % VERIFYGOLDEN Compare output against golden file + % + % result = verifyGolden(goldenPath, actualOutput) + % result = verifyGolden(goldenPath, actualOutput, tolerance) + % + % tolerance defaults to 1e-10 for floating point comparisons + + if nargin < 3 + tolerance = 1e-10; + end + + golden = load(goldenPath); + result = struct('passed', true, 'failures', {{}}); + + % Compare numeric fields + fields = fieldnames(golden.output); + for i = 1:length(fields) + fname = fields{i}; + if ~isfield(actualOutput, fname) + result.passed = false; + result.failures{end+1} = sprintf('Missing field: %s', fname); + continue; + end + + expected = golden.output.(fname); + actual = actualOutput.(fname); + + if isnumeric(expected) && isnumeric(actual) + maxDiff = max(abs(expected(:) - actual(:)), [], 'omitnan'); + if maxDiff > tolerance + result.passed = false; + result.failures{end+1} = sprintf('%s: max diff %.2e exceeds tolerance %.2e', ... + fname, maxDiff, tolerance); + end + elseif ~isequal(expected, actual) + result.passed = false; + result.failures{end+1} = sprintf('%s: values do not match', fname); + end + end +end +``` + +### Integration with MATLAB unit tests + +```matlab +classdef ProcessingGoldenTest < matlab.unittest.TestCase + + properties (TestParameter) + goldenFile = getGoldenFiles('tests/golden/processFNIRS2'); + end + + methods (Test) + function testAgainstGolden(testCase, goldenFile) + golden = load(goldenFile); + + % Reproduce the processing + data = pf2.import.sampleData(); + testCase.verifyEqual(computeHash(data), golden.inputHash, ... + 'Input data has changed - golden file may need regeneration'); + + % Apply same parameters + pf2.methods.raw.setMethod(golden.params.rawMethod); + pf2.methods.oxy.setMethod(golden.params.oxyMethod); + actual = processFNIRS2(data); + + % Compare outputs + result = verifyGolden(goldenFile, actual); + testCase.verifyTrue(result.passed, strjoin(result.failures, '\n')); + end + end +end + +function files = getGoldenFiles(directory) + listing = dir(fullfile(directory, '*.mat')); + files = fullfile({listing.folder}, {listing.name}); +end +``` + +## Promoting Candidates to Golden + +After verifying a candidate file is correct: + +```matlab +function promoteCandidate(candidateName, targetDir) + % PROMOTECANDIDATE Move verified candidate to golden directory + % + % promoteCandidate('processFNIRS2_sampleNIR_x2_lpf_smar.mat', 'processFNIRS2') + + candidatePath = fullfile('tests', 'golden', 'candidates', candidateName); + targetPath = fullfile('tests', 'golden', targetDir, candidateName); + + % Verify candidate exists + if ~isfile(candidatePath) + error('Candidate file not found: %s', candidatePath); + end + + % Create target directory if needed + if ~isfolder(fullfile('tests', 'golden', targetDir)) + mkdir(fullfile('tests', 'golden', targetDir)); + end + + % Move file + movefile(candidatePath, targetPath); + fprintf('Promoted: %s -> %s\n', candidatePath, targetPath); +end +``` + +## Workflow for Intentional Changes + +When algorithm changes are intentional and golden files need updating: + +1. **Document the change**: Add a comment in the commit explaining why outputs changed + +2. **Generate new candidates**: + ```matlab + % Regenerate affected golden files + generateGoldenFile('processFNIRS2', 'sampleNIR', 'x2_lpf_smar'); + ``` + +3. **Review the differences**: + ```matlab + compareGolden('tests/golden/processFNIRS2/file.mat', 'tests/golden/candidates/file.mat'); + ``` + +4. **Promote after review**: + ```matlab + promoteCandidate('file.mat', 'processFNIRS2'); + ``` + +5. **Commit both code and golden files** together so the history shows the relationship + +## Utility Functions + +### Compute input hash + +```matlab +function hash = computeHash(data) + % COMPUTEHASH Generate SHA-256 hash of input data + % + % Used to verify input data hasn't changed when comparing against golden files + + % Serialize struct to bytes + bytes = getByteStreamFromArray(data); + + % Compute SHA-256 + md = java.security.MessageDigest.getInstance('SHA-256'); + md.update(bytes); + hashBytes = md.digest(); + + % Convert to hex string + hash = sprintf('%02x', typecast(hashBytes, 'uint8')); +end +``` + +### Compare two golden files + +```matlab +function compareGolden(goldenPath, candidatePath) + % COMPAREGOLDEN Show differences between golden and candidate files + + golden = load(goldenPath); + candidate = load(candidatePath); + + fprintf('Golden: %s (v%s, %s)\n', goldenPath, golden.version, golden.timestamp); + fprintf('Candidate: %s (v%s, %s)\n', candidatePath, candidate.version, candidate.timestamp); + fprintf('\n'); + + fields = fieldnames(golden.output); + for i = 1:length(fields) + fname = fields{i}; + g = golden.output.(fname); + c = candidate.output.(fname); + + if isnumeric(g) && isnumeric(c) + maxDiff = max(abs(g(:) - c(:)), [], 'omitnan'); + meanDiff = mean(abs(g(:) - c(:)), 'omitnan'); + fprintf('%s: max=%.2e, mean=%.2e\n', fname, maxDiff, meanDiff); + else + if isequal(g, c) + fprintf('%s: identical\n', fname); + else + fprintf('%s: DIFFERENT\n', fname); + end + end + end +end +``` + +## Best Practices + +1. **Use deterministic inputs**: Always use `pf2.import.sampleData()` or other fixed test data +2. **Pin method configurations**: Explicitly set methods rather than relying on defaults +3. **Include version info**: Always store the processFNIRS2 version in golden files +4. **Review before promoting**: Never blindly promote candidates to golden +5. **One golden per configuration**: Separate files for different method combinations +6. **Keep candidates clean**: Delete rejected candidates promptly + +## Naming Convention + +``` +{pipeline}_{dataset}_{rawMethod}_{oxyMethod}.mat +``` + +Examples: +- `processFNIRS2_sampleNIR_x2_lpf_smar_takizawa_easy.mat` +- `pf2_SMAR_sampleNIR.mat` +- `bvoxy_synthetic_DPFcalc.mat` diff --git a/tests/golden/functions/pf2_TDDR_fNIR2000.mat b/tests/golden/functions/pf2_TDDR_fNIR2000.mat index 8dd0c3b5..d9ce83e2 100644 Binary files a/tests/golden/functions/pf2_TDDR_fNIR2000.mat and b/tests/golden/functions/pf2_TDDR_fNIR2000.mat differ diff --git a/tests/golden/processFNIRS2/fNIR2000_TDDR_None.mat b/tests/golden/processFNIRS2/fNIR2000_TDDR_None.mat index 9bc94b9d..b8dbcbf9 100644 Binary files a/tests/golden/processFNIRS2/fNIR2000_TDDR_None.mat and b/tests/golden/processFNIRS2/fNIR2000_TDDR_None.mat differ diff --git a/tests/golden/processFNIRS2/fNIR2000_default.mat b/tests/golden/processFNIRS2/fNIR2000_default.mat index 3ce7c1d4..fdf46137 100644 Binary files a/tests/golden/processFNIRS2/fNIR2000_default.mat and b/tests/golden/processFNIRS2/fNIR2000_default.mat differ