Ticket #8687: patch_8687_5.diff

src/org/openstreetmap/josm/data/validation/tests/TagChecker.java

@@ -761 +761 @@
 
     protected static class CheckerData {
         private String description;
-        private List<CheckerElement> data = new ArrayList<CheckerElement>();
+        protected List<CheckerElement> data = new ArrayList<CheckerElement>();
         private OsmPrimitiveType type;
         private int code;
         protected Severity severity;
@@ -769 +769 @@
         protected static final int TAG_CHECK_WARN   = 1260;
         protected static final int TAG_CHECK_INFO   = 1270;
 
-        private static class CheckerElement {
+        protected static class CheckerElement {
             public Object tag;
             public Object value;
             public boolean noMatch;

data/tagchecker.cfg

@@ -179 +179 @@
 * : E : wikipedia == * && wikipedia != /[a-zA-Z_-]{2,12}:.*/ # no wikipedia-language given, use ''wikipedia''=''language:page title''
 # Valid languages are extracted from <http://de.wikipedia.org/w/api.php?action=sitematrix&format=xml>, which may change, so this is a warning only.
 * : W : wikipedia == /[a-zA-Z_-]{2,12}:.*/ && wikipedia != /(aa|ab|ace|af|ak|als|am|an|ang|ar|arc|arz|as|ast|av|ay|az|ba|bar|bat-smg|bcl|be|be-x-old|bg|bh|bi|bjn|bm|bn|bo|bpy|br|bs|bug|bxr|ca|cbk-zam|cdo|ce|ceb|ch|cho|chr|chy|ckb|co|cr|crh|cs|csb|cu|cv|cy|cz|da|de|diq|dk|dsb|dv|dz|ee|el|eml|en|eo|epo|es|et|eu|ext|fa|ff|fi|fiu-vro|fj|fo|fr|frp|frr|fur|fy|ga|gag|gan|gd|gl|glk|gn|got|gu|gv|ha|hak|haw|he|hi|hif|ho|hr|hsb|ht|hu|hy|hz|ia|id|ie|ig|ii|ik|ilo|io|is|it|iu|ja|jbo|jp|jv|ka|kaa|kab|kbd|kg|ki|kj|kk|kl|km|kn|ko|koi|kr|krc|ks|ksh|ku|kv|kw|ky|la|lad|lb|lbe|lez|lg|li|lij|lmo|ln|lo|lt|ltg|lv|map-bms|mdf|mg|mh|mhr|mi|minnan|mk|ml|mn|mo|mr|mrj|ms|mt|mus|mwl|my|myv|mzn|na|nah|nan|nap|nb|nds|nds-nl|ne|new|ng|nl|nn|no|nov|nrm|nso|nv|ny|oc|om|or|os|pa|pag|pam|pap|pcd|pdc|pfl|pi|pih|pl|pms|pnb|pnt|ps|pt|qu|rm|rmy|rn|ro|roa-rup|roa-tara|ru|rue|rw|sa|sah|sc|scn|sco|sd|se|sg|sh|si|simple|sk|sl|sm|sn|so|sq|sr|srn|ss|st|stq|su|sv|sw|szl|ta|te|tet|tg|th|ti|tk|tl|tn|to|tpi|tr|ts|tt|tum|tw|ty|udm|ug|uk|ur|uz|ve|vec|vep|vi|vls|vo|wa|war|wo|wuu|xal|xh|xmf|yi|yo|za|zea|zh|zh-cfr|zh-classical|zh-min-nan|zh-yue|zu):.*/  # unkown language prefix in wikipedia tag
+
+# measurement values and units warnings (ticket #8687)
+way : W : layer == * && layer != /^0$|^-?[1-5]$/  # layer should be between -5 and 5
+*   : W : level == * && level != /^((([0-9]|-[1-9])|[1-9][0-9]*)(\.5)?)(;(([0-9]|-[1-9])|[1-9][0-9]*)(\.5)?)*$|^-0\.5$/  # level should be numbers with optional .5 increments
+*   : W : height == * && height != /^(([0-9]+\.?[0-9]*( m)?)|([1-9][0-9]*\'((10|11|[0-9])((\.[0-9]+)?)\")?))$/  # meters is default; period is separator; if units, put space then unit
+
+*   : W : maxheight == * && maxheight != /^(([1-9][0-9]*(\.[0-9]+)?( m)?)|([0-9]+\'([0-9]|10|11)(\.[0-9]*)?\"))$/  # meters is default; period is separator; if units, put space then unit
+way : W : width == * && width != /^(([0-9]+\.?[0-9]*( [a-z]+)?)|([0-9]+\'[0-9]+\.?[0-9]*\"))$/  # meters is default; period is separator; if units, put space then unit
+*   : W : maxwidth == * && maxwidth != /^(([0-9]+\.?[0-9]*( m)?)|([0-9]+\'[0-9]+\.?[0-9]*\"))$/  # meters is default; period is separator; if units, put space then unit
+way : W : maxspeed == * && maxspeed != /^(signals|none|unposted|unknown|variable|walk|[1-9][0-9]*( [a-z]+)?|[A-Z][A-Z]:(urban|rural|living_street|motorway))$/  # unusual maxspeed format
+way : W : voltage == * && voltage == /(.*[A-Za-z].*)|.*,.*|.*( ).*/  # voltage should be in volts with no units/delimiter/spaces
+# some users are using frequency for other purposes (not electromagnetic) with the values 'perennial' and 'intermittent'; the vast majority are 0, 16.7, 50 and 60
+way : W : frequency == * && frequency != /^(0|[1-9][0-9]*(\.[0-9]+)?)( (kHz|MHz|GHz|THz))?$/  # unusual frequency specification
+way : W : gauge == * && gauge != /^([1-9][0-9]{1,3}(;[1-9][0-9]{1,3})*|broad|standard|narrow)$/  # unusual train track gauge; use mm with no separator
+# the numbers for percentage and degrees include could probably be bracketed a bit more precisely
+way : W : incline == * && include != /^(up|down|-?([0-9]+?(\.[1-9]%)?|100)[%°]?)$/  # unusual incline; use percentages/degrees or up/down

test/data/regexp-data.json

@@ +1 @@
+{
+        "tests": [
+                {
+                        "key": "layer",
+                        "regexp": "^0$|^-?[1-5]$",
+                        "match": [
+                                "0", "1", "2", "3", "4", "5", "-1", "-2", "-3", "-4", "-5"
+                        ],
+                        "reject": [
+                                "+1", "+2", "+3", "+4", "+5", "zero", "yes", "no", "o", "1,2", "'-1", "1;0", "01", "!", "46", "-2000"
+                        ]
+                },
+                {
+                        "key": "level",
+                        "regexp": "^((([0-9]|-[1-9])|[1-9][0-9]*)(\\.5)?)(;(([0-9]|-[1-9])|[1-9][0-9]*)(\\.5)?)*$|^-0\\.5$",
+                        "match": [
+                                "163", "88", "-1", "-2", "0", "1", "10", "0;1", "0.5", "1.5", "-0.5", "-1.5"
+                        ],
+                        "reject": [
+                                "01", "+1", "0,1", "0; 1", "0 ;1", "1.33"
+                        ]
+                },
+                {
+                        "key": "height",
+                        "regexp": "^(([0-9]+\\.?[0-9]*( m)?)|([1-9][0-9]*\\'((10|11|[0-9])((\\.[0-9]+)?)\\\")?))$",
+                        "match": [
+                                "4 m", "0.8", "11'4\"", "10", "2.5", "10 m", "10.2", "10'", "10'0\"", "9'5.5\""
+                        ],
+                        "reject": [
+                                ">20", "10m", "10.2m"
+                        ]
+                },
+                {
+                        "key": "maxheight",
+                        "regexp": "^(([1-9][0-9]*(\\.[0-9]+)?( m)?)|([0-9]+\\'([0-9]|10|11)(\\.[0-9]*)?\\\"))$",
+                        "match": [
+                                "3", "3 m", "3.8", "3.8 m", "6'7\"", "7'0\"", "2.3 m", "7'9\"", "16'3\""
+                        ],
+                        "reject": [
+                                "3.8m", "3.8 meters", "0,6", "7 ft", "7'", "2.3; 7'9\"", "16' 0\""
+                        ]
+                },
+                {
+                        "key": "width",
+                        "regexp": "^(([0-9]+\\.?[0-9]*( [a-z]+)?)|([0-9]+\\'[0-9]+\\.?[0-9]*\\\"))$",
+                        "match": [
+                                "2", "2 m", "0.6 mi", "16'3\"", "12.2", "2.5", "2.00", "0.30479999"
+                        ],
+                        "reject": [
+                                "2m", "12' 6\"", "2km", "0,6", "2,5", "3,00", "8' - 10'", ".2"
+                        ]
+                },
+                {
+                        "key": "maxwidth",
+                        "regexp": "^(([0-9]+\\.?[0-9]*( m)?)|([0-9]+\\'[0-9]+\\.?[0-9]*\\\"))$",
+                        "match": [
+                                "2", "3 m", "2.2", "6'6\"", "10'0\"", "3.6576", "2.50", "33.05"
+                        ],
+                        "reject": [
+                                "3.8m", "2.8 meters", "0,6", "7 ft", "6.5 ft", "7'", "2.3; 7'9\"", "13' 6\"", "7ft6in", "2m"
+                        ]
+                },
+                {
+                        "key": "maxspeed",
+                        "regexp": "^(signals|none|unposted|unknown|variable|walk|[1-9][0-9]*( [a-z]+)?|[A-Z][A-Z]:(urban|rural|living_street|motorway))$",
+                        "match": [
+                                "50", "30", "60", "30 mph", "RO:urban", "100", "none", "10 knots", "signals", "RU:rural", "5", "250", "walk"
+                        ],
+                        "reject": [
+                                "70mph", "ru:urban"
+                        ]
+                },
+                {
+                        "key": "frequency",
+                        "regexp": "^(0|[1-9][0-9]*(\\.[0-9]+)?)( (kHz|MHz|GHz|THz))?$",
+                        "match": [
+                                "0", "16.7", "50", "60", "16.67"
+                        ],
+                        "reject": [
+                                "16.7 ; 50", "16,7", "680kHz", "50 khz", "50 KHZ"
+                        ]
+                },
+                {
+                        "key": "gauge",
+                        "regexp": "^([1-9][0-9]{1,3}(;[1-9][0-9]{1,3})*|broad|standard|narrow)$",
+                        "match": [
+                                "1435", "750", "1000;1435", "broad", "standard", "narrow", "127;184", "89;127;184", "89;127;144;184"
+                        ],
+                        "reject": [
+                                "1435mm"
+                        ]
+                },
+                {
+                        "key": "incline",
+                        "regexp": "^(up|down|-?([0-9]+?(\\.[1-9]%)?|100)[%°]?)$",
+                        "match": [
+                                "up", "down", "10%", "-10%", "12%", "15%", "0", "5%", "10", "15", "-10°", "-6%", "10°", "5°", "-5°"
+                        ],
+                        "reject": [
+                                "10 °", "5 °", "10 %", "-10 %", "5 %", "-5 %", "11.7 %", "2,5%", "3,5", "+33%", "incline=20°", "yes;10%"
+                        ]
+                },
+                {
+                        "key": "voltage",
+                        "regexp": "(.*[A-Za-z].*)|.*,.*|.*( ).*",
+                        "match": [
+                                "15 kV", "15,000", "11000; 400", "11000,400", "11000 ;400"
+                        ],
+                        "reject": [
+                                "15000", "25000", "3000", "1500", "750", "110000", "11000;400", "380000;220000;110000"
+                        ],
+                        "comment": "This test is reversed, it checks for violations rather than what is permitted. No units, no whitespace, no comma for thousands separator."
+                }
+        ]
+}
+

test/unit/org/openstreetmap/josm/data/validation/tests/TagCheckerTest.java

@@ +1 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.validation.tests;
+
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+import org.openstreetmap.josm.Main;
+import org.openstreetmap.josm.data.Preferences;
+
+import com.google.gson.Gson;
+
+public class TagCheckerTest {
+
+    private TagChecker tc;
+    private Gson gson;
+    private RegexpTests testSet;
+
+    private class RegexpTestInstance {
+        public String key;
+        public List<String> match;
+        public List<String> reject;
+        public String regexp;
+    }
+
+    public class RegexpTests {
+        public List<RegexpTestInstance> tests = new ArrayList<RegexpTestInstance>();
+    }
+
+    @Before
+    public void setUp() throws Exception {
+        Main.pref = new Preferences();
+        tc = new TagChecker();
+        tc.initialize();
+        gson = new Gson();
+    }
+
+    @Test
+    public void testRegularExpressions() {
+        Assume.assumeTrue(TagChecker.checkerData.size() > 0);
+        FileReader jsonFileReader = null;
+
+        try {
+            jsonFileReader = new FileReader("test/data/regexp-data.json");
+            testSet = gson.fromJson(jsonFileReader, RegexpTests.class);
+        } catch (FileNotFoundException e) {
+            Assert.fail(e.getMessage());
+        } finally {
+            if (jsonFileReader != null) {
+                try { jsonFileReader.close(); } catch (Throwable t) {}
+            }
+        }
+
+        for (TagChecker.CheckerData cd : TagChecker.checkerData) {
+            for (TagChecker.CheckerData.CheckerElement ce : cd.data) {
+                RegexpTestInstance testInstance = null;
+                if (ce.value instanceof Pattern) {
+                    validateCheckerAgainstRegexpData(ce, testInstance);
+                }
+            }
+        }
+    }
+
+    private void validateCheckerAgainstRegexpData(TagChecker.CheckerData.CheckerElement ce,
+            RegexpTestInstance testInstance) {
+        Pattern patternFromCfg = (Pattern)ce.value;
+        for (RegexpTestInstance rti : testSet.tests) {
+            if (rti.regexp.compareTo( patternFromCfg.pattern()) == 0) {
+                testInstance = rti;
+                System.out.println(rti.key);
+                break;
+            }
+        }
+        if (testInstance == null) {
+            System.out.println("No tests found for regexp='" + patternFromCfg + "'");
+        } else {
+            // Run through the checks
+            for (String acceptString : testInstance.match) {
+                Matcher matcher = patternFromCfg.matcher(acceptString);
+                Assert.assertTrue(testInstance.key + ": " + acceptString + " should match", matcher.matches());
+            }
+            for (String rejectString : testInstance.reject) {
+                Matcher matcher = patternFromCfg.matcher(rejectString);
+                Assert.assertFalse(testInstance.key + ": " + rejectString + " should not match", matcher.matches());
+            }
+        }
+    }
+}