Changeset 14508 in josm for trunk/src/org

Timestamp:

2018-12-04T08:08:46+01:00 (6 years ago)

Author:

GerdP

Message:

see #17055 improve TagChecker

• allow Levenshtein distance of 2 so that residentail is found as typo
• disable Fix for typos where the correct value is not known, e.g. services can be service or services
• don't suggest values if all preset values are short, e.g.

layer=6 should not say "6 looks like one of [1,2,3,4,5]"

File:

• trunk/src/org/openstreetmap/josm/data/validation/tests/TagChecker.java (modified) (10 diffs)

trunk/src/org/openstreetmap/josm/data/validation/tests/TagChecker.java

@@ -14 +14 @@
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
@@ -77 +78 @@
     private static final List<CheckerData> checkerData = new ArrayList<>();
     private static final List<String> ignoreDataStartsWith = new ArrayList<>();
-    private static final List<String> ignoreDataEquals = new ArrayList<>();
+    private static final Set<String> ignoreDataEquals = new HashSet<>();
     private static final List<String> ignoreDataEndsWith = new ArrayList<>();
     private static final List<Tag> ignoreDataTag = new ArrayList<>();
@@ -123 +124 @@
      */
     public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload";
+
+    private static final int MAX_LEVENSHTEIN_DISTANCE = 2;
 
     protected boolean checkKeys;
@@ -142 +145 @@
 
     // CHECKSTYLE.OFF: SingleSpaceSeparator
-    protected static final int EMPTY_VALUES      = 1200;
-    protected static final int INVALID_KEY       = 1201;
-    protected static final int INVALID_VALUE     = 1202;
-    protected static final int FIXME             = 1203;
-    protected static final int INVALID_SPACE     = 1204;
-    protected static final int INVALID_KEY_SPACE = 1205;
-    protected static final int INVALID_HTML      = 1206; /* 1207 was PAINT */
-    protected static final int LONG_VALUE        = 1208;
-    protected static final int LONG_KEY          = 1209;
-    protected static final int LOW_CHAR_VALUE    = 1210;
-    protected static final int LOW_CHAR_KEY      = 1211;
-    protected static final int MISSPELLED_VALUE  = 1212;
-    protected static final int MISSPELLED_KEY    = 1213;
-    protected static final int MULTIPLE_SPACES   = 1214;
+    protected static final int EMPTY_VALUES             = 1200;
+    protected static final int INVALID_KEY              = 1201;
+    protected static final int INVALID_VALUE            = 1202;
+    protected static final int FIXME                    = 1203;
+    protected static final int INVALID_SPACE            = 1204;
+    protected static final int INVALID_KEY_SPACE        = 1205;
+    protected static final int INVALID_HTML             = 1206; /* 1207 was PAINT */
+    protected static final int LONG_VALUE               = 1208;
+    protected static final int LONG_KEY                 = 1209;
+    protected static final int LOW_CHAR_VALUE           = 1210;
+    protected static final int LOW_CHAR_KEY             = 1211;
+    protected static final int MISSPELLED_VALUE         = 1212;
+    protected static final int MISSPELLED_KEY           = 1213;
+    protected static final int MULTIPLE_SPACES          = 1214;
+    protected static final int MISSPELLED_VALUE_NO_FIX  = 1215;
     // CHECKSTYLE.ON: SingleSpaceSeparator
     // 1250 and up is used by tagcheck
@@ -388 +392 @@
      */
     public static boolean isTagIgnored(String key, String value) {
-        boolean tagInPresets = isTagInPresets(key, value);
-        boolean ignore = false;
-
+        if (ignoreDataEquals.contains(key)) {
+            return true;
+        }
         for (String a : ignoreDataStartsWith) {
             if (key.startsWith(a)) {
-                ignore = true;
-            }
-        }
-        for (String a : ignoreDataEquals) {
-            if (key.equals(a)) {
-                ignore = true;
+                return true;
             }
         }
         for (String a : ignoreDataEndsWith) {
             if (key.endsWith(a)) {
-                ignore = true;
-            }
-        }
-
-        if (!tagInPresets) {
+                return true;
+            }
+        }
+
+        if (!isTagInPresets(key, value)) {
             for (Tag a : ignoreDataTag) {
                 if (key.equals(a.getKey()) && value.equals(a.getValue())) {
-                    ignore = true;
+                    return true;
                 }
             }
         }
-        return ignore;
+        return false;
     }
 
@@ -535 +534 @@
                     // try to fix common typos and check again if value is still unknown
                     String fixedValue = harmonizeValue(prop.getValue());
-                    Map<String, String> possibleValues = getPossibleValues(getPresetValues(key));
+                    Set<String> possibleValues = getPresetValues(key);
                     List<String> fixVals = new ArrayList<>();
-                    if (!possibleValues.containsKey(fixedValue)) {
-                        int minDist = 2;
+                    int maxPresetValueLen = 0;
+                    if (!possibleValues.contains(fixedValue)) {
+                        // use Levenshtein distance to find typical typos
+                        int minDist = MAX_LEVENSHTEIN_DISTANCE + 1;
                         String closest = null;
-                        for (String possibleVal : possibleValues.keySet()) {
+                        for (String possibleVal : possibleValues) {
+                            if (possibleVal.isEmpty())
+                                continue;
+                            maxPresetValueLen = Math.max(maxPresetValueLen, possibleVal.length());
                             int dist = Utils.getLevenshteinDistance(possibleVal, fixedValue);
                             if (dist < minDist) {
@@ -551 +555 @@
                             }
                         }
-                        if (minDist <= 1) {
+                        fixedValue = null;
+                        if (minDist <= MAX_LEVENSHTEIN_DISTANCE && maxPresetValueLen > MAX_LEVENSHTEIN_DISTANCE) {
                             if (fixVals.size() < 2) {
                                 fixedValue = closest;
@@ -557 +562 @@
                                 Collections.sort(fixVals);
                                 // misspelled preset value with multiple good alternatives
-                                errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE)
+                                errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE_NO_FIX)
                                         .message(tr("Misspelled property value"),
                                                 marktr("Value ''{0}'' for key ''{1}'' looks like one of {2}."), prop.getValue(), key, fixVals)
@@ -567 +572 @@
                         }
                     }
-                    if (possibleValues.containsKey(fixedValue)) {
-                        final String newValue = possibleValues.get(fixedValue);
+                    if (fixedValue != null && possibleValues.contains(fixedValue)) {
+                        final String newValue = fixedValue;
                         // misspelled preset value
                         errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE)
@@ -603 +608 @@
     }
 
-    private static Map<String, String> getPossibleValues(Set<String> values) {
-        // generate a map with common typos
-        Map<String, String> map = new HashMap<>();
-        if (values != null) {
-            for (String value : values) {
-                map.put(value, value);
-                if (value.contains("_")) {
-                    map.put(value.replace("_", ""), value);
-                }
-            }
-        }
-        return map;
-    }
-
     private static String harmonizeKey(String key) {
         return Utils.strip(key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'), "-_;:,");