-
Notifications
You must be signed in to change notification settings - Fork 8
feat: implement est_width parsing and sidebar display #289
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: develop
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| import dompurify from 'dompurify' | ||
| import { TagsTableRow } from '../TagsTableRow' | ||
| import { ConditionalFormattedValue } from '../translations/ConditionalFormattedValue' | ||
| import { translations } from '../translations/translations.const' | ||
| import { ValueDisclosure, ValueDisclosureButton, ValueDisclosurePanel } from '../ValueDisclosure' | ||
| import { NodataFallback } from './NodataFallback' | ||
| import type { CompositTableRow } from './types' | ||
|
|
||
| export const tableKeyWidths = ['composit_width', 'composit_road_width'] | ||
|
|
||
| export const TagsTableRowCompositWidth = ({ sourceId, tagKey, properties }: CompositTableRow) => { | ||
| const widthKey = tagKey.replace('composit_', '') | ||
| const width = widthKey | ||
| const widthSource = `${widthKey}_source` | ||
| const widthConfidence = `${widthKey}_confidence` | ||
|
|
||
| if (!width) { | ||
| return ( | ||
| <TagsTableRow key={tagKey} sourceId={sourceId} tagKey={tagKey}> | ||
| <NodataFallback /> | ||
| </TagsTableRow> | ||
| ) | ||
| } | ||
|
|
||
| const secureWidthSource = dompurify.sanitize(properties[widthSource]) | ||
| const sourceHasTranslation = Object.keys(translations).some((k) => | ||
| k.includes(`${widthSource}=${secureWidthSource}`), | ||
| ) | ||
|
|
||
| return ( | ||
| <TagsTableRow key={widthKey} sourceId={sourceId} tagKey={widthKey}> | ||
| <ValueDisclosure> | ||
| <ValueDisclosureButton> | ||
| <ConditionalFormattedValue | ||
| sourceId={sourceId} | ||
| tagKey={width} | ||
| tagValue={properties[width]} | ||
| /> | ||
| </ValueDisclosureButton> | ||
| <ValueDisclosurePanel> | ||
| <p> | ||
| <em>Quelle:</em>{' '} | ||
| {sourceHasTranslation ? ( | ||
| <ConditionalFormattedValue | ||
| sourceId={sourceId} | ||
| tagKey={widthSource} | ||
| tagValue={secureWidthSource} | ||
| /> | ||
| ) : ( | ||
| <code>{secureWidthSource}</code> | ||
| )} | ||
| {properties[widthConfidence] && ( | ||
| <> | ||
| {' ('} | ||
| <ConditionalFormattedValue | ||
| sourceId={sourceId} | ||
| tagKey={widthConfidence} | ||
| tagValue={properties[widthConfidence]} | ||
| /> | ||
| {')'} | ||
| </> | ||
| )} | ||
| </p> | ||
| </ValueDisclosurePanel> | ||
| </ValueDisclosure> | ||
| </TagsTableRow> | ||
| ) | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| export const translationsWdith = { | ||
| 'ALL--composit_width--key': 'Breite', | ||
| 'ALL--width_source--key': 'Quelle Breite', | ||
| 'ALL--width_source=ALKIS': 'Aus ALKIS Daten ausgemessen', | ||
| 'ALL--width_source=ARCore': 'Mit dem Handy-Metermaß von StreetComplete gemessen', | ||
| 'ALL--width_confidence--key': 'Konfidenz Breite', | ||
| 'ALL--width_confidence=high': 'Direkt gemessen', | ||
| 'ALL--width_confidence=low': 'Geschätzt', | ||
| 'ALL--width--key': 'Breite', | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -247,11 +247,13 @@ export const Route = createFileRoute('/api/export/$regionSlug/$tableName')({ | |
| SELECT column_name | ||
| FROM information_schema.columns | ||
| WHERE table_name = '${tableName}' | ||
| AND column_name IN ('osm_id', 'osm_type') | ||
| AND column_name IN ('osm_id', 'osm_type', 'width_confidence', 'width_source') | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Weg, siehe unten |
||
| `) | ||
| const existingColumns = columnExistsQuery.map(({ column_name }) => column_name) | ||
| const hasOsmId = existingColumns.includes('osm_id') | ||
| const hasOsmType = existingColumns.includes('osm_type') | ||
| const hasWidthConfidence = existingColumns.includes('width_confidence') | ||
| const hasWidthSource = existingColumns.includes('width_source') | ||
| const columnsMetadataTimingsMs = { | ||
| tagKeys: metaKeysStartedAt - tagKeysStartedAt, | ||
| metaKeys: columnsCheckStartedAt - metaKeysStartedAt, | ||
|
|
@@ -273,7 +275,11 @@ export const Route = createFileRoute('/api/export/$regionSlug/$tableName')({ | |
| 'geom', | ||
| hasOsmId ? 'osm_id' : undefined, | ||
| hasOsmType ? 'osm_type' : undefined, | ||
| ...tagKeyQuery.map(({ key }) => generateColumn(key, 'tags')), | ||
| hasWidthConfidence ? 'width_confidence' : undefined, | ||
| hasWidthSource ? 'width_source' : undefined, | ||
| ...tagKeyQuery | ||
| .filter(({ key }) => key !== 'width_confidence' && key !== 'width_source') | ||
| .map(({ key }) => generateColumn(key, 'tags')), | ||
| ...metaKeyQuery.map(({ key }) => generateColumn(key, 'meta')), | ||
| ] | ||
| .filter(Boolean) | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| #!/bin/bash | ||
| # Start docker and run all tests there. | ||
|
|
||
| # GUI Git clients (e.g. GitHub Desktop) often run hooks with a minimal PATH, so the | ||
| # Docker CLI is missing even when Docker Desktop is installed. | ||
| export PATH="/usr/local/bin:/opt/homebrew/bin:${HOME}/.docker/bin:${PATH}" | ||
|
|
||
| # Make sure we have the latest lua path at hand. | ||
| # Szenario: We just added a lua helper in a new folder but did not run the full processing, yet | ||
| NODE_PATH=./app/node_modules bun run ./processing/run-tests-update-lua-package-paths.ts | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. raus, bun verwendne |
||
|
|
||
| if ! command -v docker >/dev/null 2>&1; then | ||
| echo "docker not in PATH — skipping processing Docker tests." | ||
| echo "To run them locally: open a normal terminal in the repo and run ./processing/run-tests.sh" | ||
| exit 0 | ||
| fi | ||
|
|
||
| # Now use docker to run the tests. | ||
| # We have to use docker because that is where the LUA hepers are all present. | ||
| if ping -q -c 1 -W 1 8.8.8.8 > /dev/null; then | ||
| echo "Internet available — building Docker image..." | ||
| docker build --target testing -f ./processing.Dockerfile -t test_img . | ||
| else | ||
| echo "No internet connection — skipping Docker build." | ||
| fi | ||
|
|
||
| docker run --rm -v "$(pwd)/processing:/processing" test_img | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| local parse_width = require('parse_width') | ||
|
|
||
| -- * @desc If and why a highway object should be excluded based on its width. | ||
| -- * @returns { boolean (shouldFilter), string (reason) } | ||
| function ExcludeByWidth(tags, minWidth, object) | ||
| if not tags.width then | ||
| return false, "" | ||
| end | ||
|
|
||
| local width = parse_width(tags.width, object, 'width') | ||
| if width and width < minWidth then | ||
| return true, ";Excluded since `width<" .. minWidth .. "` indicates a special interest path" | ||
| end | ||
| return false, "" | ||
| end |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| local parse_width = require('parse_width') | ||
|
|
||
| -- * @desc TODO | ||
| -- * @returns TODO | ||
| function RoadWidth(tags, object) | ||
| local width_val = tags["width"] | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
| local tag_name = "width" | ||
| if not width_val then | ||
| width_val = tags["est_width"] | ||
| tag_name = "est_width" | ||
| end | ||
|
|
||
| if width_val then | ||
| local width = parse_width(width_val, object, tag_name) | ||
| if width then return width end | ||
| end | ||
|
|
||
| local streetWidths = { | ||
| primary = 10, | ||
| secondary = 8, | ||
| tertiary = 6, | ||
| residential = 6, | ||
| } | ||
| if streetWidths[tags["highway"]] then | ||
| return streetWidths[tags["highway"]] | ||
| end | ||
|
|
||
| return 8 | ||
| end | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| describe("parse_width", function() | ||
| require('init') | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. weg nach rebase |
||
| require("osm2pgsql") | ||
|
|
||
| local logged_errors = {} | ||
| -- Mock width_errors before requiring parse_width | ||
| package.loaded['width_errors'] = { | ||
| log = function(object, tag_name, invalid_value, error_msg) | ||
| table.insert(logged_errors, { | ||
| object = object, | ||
| tag = tag_name, | ||
| value = invalid_value, | ||
| error = error_msg | ||
| }) | ||
| end | ||
| } | ||
|
|
||
| local parse_width = require('parse_width') | ||
|
|
||
| before_each(function() | ||
| logged_errors = {} | ||
| end) | ||
|
|
||
| it('parse "1.2" as 1.2', function() | ||
| local result = parse_width("1.2") | ||
| assert.are.same(result, 1.2) | ||
| assert.are.same(#logged_errors, 0) | ||
| end) | ||
|
|
||
| it('parse "1,2" (comma) as 1.2', function() | ||
| local result = parse_width("1,2") | ||
| assert.are.same(result, 1.2) | ||
| assert.are.same(#logged_errors, 0) | ||
| end) | ||
|
|
||
| it('parse "3.5 m" as 3.5', function() | ||
| local result = parse_width("3.5 m") | ||
| assert.are.same(result, 3.5) | ||
| assert.are.same(#logged_errors, 0) | ||
| end) | ||
|
|
||
| it('parse "3,5m" as 3.5', function() | ||
| local result = parse_width("3,5m") | ||
| assert.are.same(result, 3.5) | ||
| assert.are.same(#logged_errors, 0) | ||
| end) | ||
|
|
||
| it('parse "250 cm" as 2.5', function() | ||
| local result = parse_width("250 cm") | ||
| assert.are.same(result, 2.5) | ||
| assert.are.same(#logged_errors, 0) | ||
| end) | ||
|
|
||
| it('parse "250cm" as 2.5', function() | ||
| local result = parse_width("250cm") | ||
| assert.are.same(result, 2.5) | ||
| assert.are.same(#logged_errors, 0) | ||
| end) | ||
|
|
||
| it('parse "1.5 km" as 1500', function() | ||
| local result = parse_width("1.5 km") | ||
| assert.are.same(result, 1500) | ||
| assert.are.same(#logged_errors, 0) | ||
| end) | ||
|
|
||
| it('returns nil and does not log if empty/nil', function() | ||
| assert.is_nil(parse_width(nil)) | ||
| assert.is_nil(parse_width("")) | ||
| assert.are.same(#logged_errors, 0) | ||
| end) | ||
|
|
||
| it('returns nil and logs error if invalid value and object provided', function() | ||
| local dummy_object = { id = 12345, type = "way" } | ||
| local result = parse_width("fehlerhafter Text", dummy_object, "width") | ||
| assert.is_nil(result) | ||
| assert.are.same(#logged_errors, 1) | ||
| assert.are.same(logged_errors[1].object, dummy_object) | ||
| assert.are.same(logged_errors[1].tag, "width") | ||
| assert.are.same(logged_errors[1].value, "fehlerhafter Text") | ||
| end) | ||
|
|
||
| it('returns nil and does not log if invalid value and NO object provided', function() | ||
| local result = parse_width("fehlerhafter Text") | ||
| assert.is_nil(result) | ||
| assert.are.same(#logged_errors, 0) | ||
| end) | ||
| end) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| local width_errors = require('width_errors') | ||
|
|
||
| local function parse_width(value, object, tag_name) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Siehe |
||
| if not value or value == "" then | ||
| return nil | ||
| end | ||
|
|
||
| if type(value) == "number" then | ||
| return value | ||
| end | ||
|
|
||
| -- Clean string: convert commas to dots and trim whitespace | ||
| local clean_value = tostring(value):gsub(",", "."):gsub("^%s*", ""):gsub("%s*$", "") | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Wir ignoreiren es… => https://taginfo.geofabrik.de/europe:germany/keys/width#values zeigt nur so ~20 Usage bei values mit Komma |
||
|
|
||
| local val, unit = osm2pgsql.split_unit(clean_value, 'm') | ||
| if val then | ||
| if unit == 'cm' then | ||
| return val / 100 | ||
| elseif unit == 'm' then | ||
| return val | ||
| elseif unit == 'km' then | ||
| return val * 1000 | ||
| end | ||
| end | ||
|
|
||
| -- If it couldn't be parsed, log to width_errors if object is provided | ||
| if object then | ||
| width_errors.log(object, tag_name or "width", value, "Could not parse width value") | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Error logging raus |
||
| end | ||
|
|
||
| return nil | ||
| end | ||
|
|
||
| return parse_width | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. raus |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| require('init') | ||
| require('DefaultId') | ||
|
|
||
| local db_table_error = osm2pgsql.define_table({ | ||
| name = 'width_errors', | ||
| ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, | ||
| columns = { | ||
| { column = 'serial_id', sql_type = 'serial', create_only = true }, | ||
| { column = 'id', type = 'text', not_null = true }, | ||
| { column = 'tags', type = 'jsonb' }, | ||
| { column = 'meta', type = 'jsonb' }, | ||
| { column = 'geom', type = 'linestring' }, | ||
| { column = 'minzoom', type = 'integer', not_null = true }, | ||
| }, | ||
| indexes = { | ||
| { column = {'minzoom', 'geom'}, method = 'gist' }, | ||
| { column = 'serial_id', method = 'btree', unique = true }, | ||
| { column = 'id', method = 'btree', unique = false }, | ||
| } | ||
| }) | ||
|
|
||
| local function log_width_error(object, tag_name, invalid_value, error_msg) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nach rebase: Alle Funtkoinnen haben Types. |
||
| if not object then return end | ||
|
|
||
| local geom = nil | ||
| if object.as_linestring then | ||
| pcall(function() geom = object:as_linestring() end) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. pcall = smell |
||
| end | ||
|
|
||
| local tags = { | ||
| osm_id = tostring(object.id), | ||
| osm_type = object.type, | ||
| tag = tag_name or "width", | ||
| value = tostring(invalid_value), | ||
| error = error_msg or "Invalid width format" | ||
| } | ||
|
|
||
| local row = { | ||
| id = DefaultId(object), | ||
| geom = geom, | ||
| tags = tags, | ||
| meta = {}, | ||
| minzoom = 0 | ||
| } | ||
| db_table_error:insert(row) | ||
| end | ||
|
|
||
| return { | ||
| log = log_width_error, | ||
| table = db_table_error | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Revert