Bugfix: Force the part of speech tag for "in" to be NN if the token follows a number. In these cases, the token is the abbreviation for inches rather than a preposition (which it would otherwise be tagged as). Add tests for the dimensional phrase structural feature.

Author: strangetom

ingredient_parser/en/_structure_features.py

@@ -329,7 +329,9 @@ def detect_dimensional_phrases(
 
         text_pos = []
         for t in tokenized_sentence:
-            if t.text.lower() in LENGTH_UNITS:
+            if t.text.lower() in LENGTH_UNITS and t.pos_tag != "IN":
+                # We need to check the POS tag so we don't confuse "in" (preposition)
+                # with "in" (abbreviation of inch).
                 pos = "LEN"
             elif t.text.lower() in DIMENSIONS:
                 pos = "DIM"

ingredient_parser/en/preprocess.py

@@ -581,6 +581,11 @@ def _calculate_tokens(self, sentence: str) -> list[Token]:
             elif text.lower() == "/":
                 # Force "/" to have SYM tag.
                 pos = "SYM"
+            elif text == "in" and i > 0 and tokens[i - 1].feat_text == "!num":
+                # If the text is "in" and the previous token is a number then this is
+                # mostly likely to be the abbreviation of inches rather than the
+                # preposition.
+                pos = "NN"
 
             features = TokenFeatures(
                 stem=stem(feat_text),

tests/preprocess/test_sentence_structure_features.py

@@ -195,3 +195,42 @@ def test_example_phrase_detection_feature(self):
                 assert token_features.get("example_phrase", False)
             else:
                 assert not token_features.get("example_phrase", False)
+
+
+class Test_dimensional_phrase_features:
+    def test_dimensional_phrase_detection(self):
+        """
+        Test dimensional phrase comprising number-unit-dimension is detected.
+        """
+        p = PreProcessor("1 2 in thick piece of steak", custom_units={})
+        assert p.sentence_structure.dimensional_phrases == [[1, 2, 3]]
+
+    def test_dimensional_phrase_no_dimension(self):
+        """
+        Test dimensional phrase comprising number-unit is detected.
+        """
+        p = PreProcessor("2in/5cm piece of ginger", custom_units={})
+        assert p.sentence_structure.dimensional_phrases == [[0, 1, 2, 3, 4]]
+
+    def test_dimensional_phrase_with_parenthesis(self):
+        """
+        Test dimensional phrase comprising two pair of number-unit, with the second pair
+        in parentheses, followed by a dimension is detected.
+        """
+        p = PreProcessor("1 2 in (5 cm) long piece of steak", custom_units={})
+        assert p.sentence_structure.dimensional_phrases == [[1, 2, 3, 4, 5, 6, 7]]
+
+    def test_dimensional_phrase_with_slash(self):
+        """
+        Test dimensional phrase comprising two pair of number-unit, with the second pair
+        after a forward slash, followed by a dimension is detected.
+        """
+        p = PreProcessor("1 2 in / 5 cm wide piece of steak", custom_units={})
+        assert p.sentence_structure.dimensional_phrases == [[1, 2, 3, 4, 5, 6]]
+
+    def test_dimensional_phrase_with_preposition(self):
+        """
+        Test dimensional phrase comprising number-unit-"in"-dimension is detected.
+        """
+        p = PreProcessor("1 potato, 3 inches in diameter", custom_units={})
+        assert p.sentence_structure.dimensional_phrases == [[3, 4, 5, 6]]