source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagChecker.java@ 14469

Last change on this file since 14469 was 14469, checked in by GerdP, 6 years ago

fix #17021 Use MapCSSRuleIndex to speed up MapCSSTagChecker
  • Property svn:eol-style set to native
File size: 44.5 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.data.validation.tests.MapCSSTagChecker.FixCommand.evaluateObject;
5import static org.openstreetmap.josm.tools.I18n.tr;
6
7import java.io.BufferedReader;
8import java.io.IOException;
9import java.io.InputStream;
10import java.io.Reader;
11import java.io.StringReader;
12import java.text.MessageFormat;
13import java.util.ArrayList;
14import java.util.Arrays;
15import java.util.Collection;
16import java.util.Collections;
17import java.util.HashMap;
18import java.util.HashSet;
19import java.util.Iterator;
20import java.util.LinkedHashMap;
21import java.util.LinkedHashSet;
22import java.util.LinkedList;
23import java.util.List;
24import java.util.Locale;
25import java.util.Map;
26import java.util.Objects;
27import java.util.Optional;
28import java.util.Set;
29import java.util.function.Predicate;
30import java.util.regex.Matcher;
31import java.util.regex.Pattern;
32
33import org.openstreetmap.josm.command.ChangePropertyCommand;
34import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
35import org.openstreetmap.josm.command.Command;
36import org.openstreetmap.josm.command.DeleteCommand;
37import org.openstreetmap.josm.command.SequenceCommand;
38import org.openstreetmap.josm.data.osm.DataSet;
39import org.openstreetmap.josm.data.osm.INode;
40import org.openstreetmap.josm.data.osm.IRelation;
41import org.openstreetmap.josm.data.osm.IWay;
42import org.openstreetmap.josm.data.osm.OsmPrimitive;
43import org.openstreetmap.josm.data.osm.OsmUtils;
44import org.openstreetmap.josm.data.osm.Tag;
45import org.openstreetmap.josm.data.preferences.sources.SourceEntry;
46import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
47import org.openstreetmap.josm.data.validation.OsmValidator;
48import org.openstreetmap.josm.data.validation.Severity;
49import org.openstreetmap.josm.data.validation.Test;
50import org.openstreetmap.josm.data.validation.TestError;
51import org.openstreetmap.josm.gui.mappaint.Environment;
52import org.openstreetmap.josm.gui.mappaint.Keyword;
53import org.openstreetmap.josm.gui.mappaint.MultiCascade;
54import org.openstreetmap.josm.gui.mappaint.mapcss.Condition;
55import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ClassCondition;
56import org.openstreetmap.josm.gui.mappaint.mapcss.Expression;
57import org.openstreetmap.josm.gui.mappaint.mapcss.Instruction;
58import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule;
59import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule.Declaration;
60import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource;
61import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource.MapCSSRuleIndex;
62import org.openstreetmap.josm.gui.mappaint.mapcss.Selector;
63import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.AbstractSelector;
64import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector;
65import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.OptimizedGeneralSelector;
66import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.MapCSSParser;
67import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException;
68import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.TokenMgrError;
69import org.openstreetmap.josm.gui.progress.ProgressMonitor;
70import org.openstreetmap.josm.io.CachedFile;
71import org.openstreetmap.josm.io.FileWatcher;
72import org.openstreetmap.josm.io.IllegalDataException;
73import org.openstreetmap.josm.io.UTFInputStreamReader;
74import org.openstreetmap.josm.spi.preferences.Config;
75import org.openstreetmap.josm.tools.CheckParameterUtil;
76import org.openstreetmap.josm.tools.I18n;
77import org.openstreetmap.josm.tools.JosmRuntimeException;
78import org.openstreetmap.josm.tools.Logging;
79import org.openstreetmap.josm.tools.MultiMap;
80import org.openstreetmap.josm.tools.Utils;
81
82/**
83 * MapCSS-based tag checker/fixer.
84 * @since 6506
85 */
86public class MapCSSTagChecker extends Test.TagTest {
87 IndexData indexData;
88
89 /**
90 * Helper class to store indexes of rules.
91 * @author Gerd
92 *
93 */
94 private static class IndexData {
95 final Map<MapCSSRule, TagCheck> ruleToCheckMap = new HashMap<>();
96
97 /**
98 * Rules for nodes
99 */
100 final MapCSSRuleIndex nodeRules = new MapCSSRuleIndex();
101 /**
102 * Rules for ways without tag area=no
103 */
104 final MapCSSRuleIndex wayRules = new MapCSSRuleIndex();
105 /**
106 * Rules for ways with tag area=no
107 */
108 final MapCSSRuleIndex wayNoAreaRules = new MapCSSRuleIndex();
109 /**
110 * Rules for relations that are not multipolygon relations
111 */
112 final MapCSSRuleIndex relationRules = new MapCSSRuleIndex();
113 /**
114 * Rules for multipolygon relations
115 */
116 final MapCSSRuleIndex multipolygonRules = new MapCSSRuleIndex();
117
118 IndexData(MultiMap<String, TagCheck> checks) {
119 buildIndex(checks);
120 }
121
122 private void buildIndex(MultiMap<String, TagCheck> checks) {
123 List<TagCheck> allChecks = new ArrayList<>();
124 for (Set<TagCheck> cs : checks.values()) {
125 allChecks.addAll(cs);
126 }
127
128 ruleToCheckMap.clear();
129 nodeRules.clear();
130 wayRules.clear();
131 wayNoAreaRules.clear();
132 relationRules.clear();
133 multipolygonRules.clear();
134
135 // optimization: filter rules for different primitive types
136 for (TagCheck c : allChecks) {
137 for (Selector s : c.rule.selectors) {
138 // find the rightmost selector, this must be a GeneralSelector
139 Selector selRightmost = s;
140 while (selRightmost instanceof Selector.ChildOrParentSelector) {
141 selRightmost = ((Selector.ChildOrParentSelector) selRightmost).right;
142 }
143 MapCSSRule optRule = new MapCSSRule(s.optimizedBaseCheck(), c.rule.declaration);
144
145 ruleToCheckMap.put(optRule, c);
146 final String base = ((GeneralSelector) selRightmost).getBase();
147 switch (base) {
148 case Selector.BASE_NODE:
149 nodeRules.add(optRule);
150 break;
151 case Selector.BASE_WAY:
152 wayNoAreaRules.add(optRule);
153 wayRules.add(optRule);
154 break;
155 case Selector.BASE_AREA:
156 wayRules.add(optRule);
157 multipolygonRules.add(optRule);
158 break;
159 case Selector.BASE_RELATION:
160 relationRules.add(optRule);
161 multipolygonRules.add(optRule);
162 break;
163 case Selector.BASE_ANY:
164 nodeRules.add(optRule);
165 wayRules.add(optRule);
166 wayNoAreaRules.add(optRule);
167 relationRules.add(optRule);
168 multipolygonRules.add(optRule);
169 break;
170 case Selector.BASE_CANVAS:
171 case Selector.BASE_META:
172 case Selector.BASE_SETTING:
173 break;
174 default:
175 final RuntimeException e = new JosmRuntimeException(MessageFormat.format("Unknown MapCSS base selector {0}", base));
176 Logging.warn(tr("Failed to index validator rules. Error was: {0}", e.getMessage()));
177 Logging.error(e);
178 }
179 }
180 }
181 nodeRules.initIndex();
182 wayRules.initIndex();
183 wayNoAreaRules.initIndex();
184 relationRules.initIndex();
185 multipolygonRules.initIndex();
186 }
187
188 /**
189 * Get the index of rules for the given primitive.
190 * @param p the primitve
191 * @return index of rules for the given primitive
192 */
193 public MapCSSRuleIndex get(OsmPrimitive p) {
194 if (p instanceof INode) {
195 return nodeRules;
196 } else if (p instanceof IWay) {
197 if (OsmUtils.isFalse(p.get("area"))) {
198 return wayNoAreaRules;
199 } else {
200 return wayRules;
201 }
202 } else if (p instanceof IRelation) {
203 if (((IRelation<?>) p).isMultipolygon()) {
204 return multipolygonRules;
205 } else {
206 return relationRules;
207 }
208 } else {
209 throw new IllegalArgumentException("Unsupported type: " + p);
210 }
211 }
212
213 /**
214 * return the TagCheck for which the given indexed rule was created.
215 * @param rule an indexed rule
216 * @return the original TagCheck
217 */
218 public TagCheck getCheck(MapCSSRule rule) {
219 return ruleToCheckMap.get(rule);
220 }
221 }
222
223 /**
224 * A grouped MapCSSRule with multiple selectors for a single declaration.
225 * @see MapCSSRule
226 */
227 public static class GroupedMapCSSRule {
228 /** MapCSS selectors **/
229 public final List<Selector> selectors;
230 /** MapCSS declaration **/
231 public final Declaration declaration;
232
233 /**
234 * Constructs a new {@code GroupedMapCSSRule}.
235 * @param selectors MapCSS selectors
236 * @param declaration MapCSS declaration
237 */
238 public GroupedMapCSSRule(List<Selector> selectors, Declaration declaration) {
239 this.selectors = selectors;
240 this.declaration = declaration;
241 }
242
243 @Override
244 public int hashCode() {
245 return Objects.hash(selectors, declaration);
246 }
247
248 @Override
249 public boolean equals(Object obj) {
250 if (this == obj) return true;
251 if (obj == null || getClass() != obj.getClass()) return false;
252 GroupedMapCSSRule that = (GroupedMapCSSRule) obj;
253 return Objects.equals(selectors, that.selectors) &&
254 Objects.equals(declaration, that.declaration);
255 }
256
257 @Override
258 public String toString() {
259 return "GroupedMapCSSRule [selectors=" + selectors + ", declaration=" + declaration + ']';
260 }
261 }
262
263 /**
264 * The preference key for tag checker source entries.
265 * @since 6670
266 */
267 public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries";
268
269 /**
270 * Constructs a new {@code MapCSSTagChecker}.
271 */
272 public MapCSSTagChecker() {
273 super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values."));
274 }
275
276 /**
277 * Represents a fix to a validation test. The fixing {@link Command} can be obtained by {@link #createCommand(OsmPrimitive, Selector)}.
278 */
279 @FunctionalInterface
280 interface FixCommand {
281 /**
282 * Creates the fixing {@link Command} for the given primitive. The {@code matchingSelector} is used to evaluate placeholders
283 * (cf. {@link MapCSSTagChecker.TagCheck#insertArguments(Selector, String, OsmPrimitive)}).
284 * @param p OSM primitive
285 * @param matchingSelector matching selector
286 * @return fix command
287 */
288 Command createCommand(OsmPrimitive p, Selector matchingSelector);
289
290 /**
291 * Checks that object is either an {@link Expression} or a {@link String}.
292 * @param obj object to check
293 * @throws IllegalArgumentException if object is not an {@code Expression} or a {@code String}
294 */
295 static void checkObject(final Object obj) {
296 CheckParameterUtil.ensureThat(obj instanceof Expression || obj instanceof String,
297 () -> "instance of Exception or String expected, but got " + obj);
298 }
299
300 /**
301 * Evaluates given object as {@link Expression} or {@link String} on the matched {@link OsmPrimitive} and {@code matchingSelector}.
302 * @param obj object to evaluate ({@link Expression} or {@link String})
303 * @param p OSM primitive
304 * @param matchingSelector matching selector
305 * @return result string
306 */
307 static String evaluateObject(final Object obj, final OsmPrimitive p, final Selector matchingSelector) {
308 final String s;
309 if (obj instanceof Expression) {
310 s = (String) ((Expression) obj).evaluate(new Environment(p));
311 } else if (obj instanceof String) {
312 s = (String) obj;
313 } else {
314 return null;
315 }
316 return TagCheck.insertArguments(matchingSelector, s, p);
317 }
318
319 /**
320 * Creates a fixing command which executes a {@link ChangePropertyCommand} on the specified tag.
321 * @param obj object to evaluate ({@link Expression} or {@link String})
322 * @return created fix command
323 */
324 static FixCommand fixAdd(final Object obj) {
325 checkObject(obj);
326 return new FixCommand() {
327 @Override
328 public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
329 final Tag tag = Tag.ofString(evaluateObject(obj, p, matchingSelector));
330 return new ChangePropertyCommand(p, tag.getKey(), tag.getValue());
331 }
332
333 @Override
334 public String toString() {
335 return "fixAdd: " + obj;
336 }
337 };
338 }
339
340 /**
341 * Creates a fixing command which executes a {@link ChangePropertyCommand} to delete the specified key.
342 * @param obj object to evaluate ({@link Expression} or {@link String})
343 * @return created fix command
344 */
345 static FixCommand fixRemove(final Object obj) {
346 checkObject(obj);
347 return new FixCommand() {
348 @Override
349 public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
350 final String key = evaluateObject(obj, p, matchingSelector);
351 return new ChangePropertyCommand(p, key, "");
352 }
353
354 @Override
355 public String toString() {
356 return "fixRemove: " + obj;
357 }
358 };
359 }
360
361 /**
362 * Creates a fixing command which executes a {@link ChangePropertyKeyCommand} on the specified keys.
363 * @param oldKey old key
364 * @param newKey new key
365 * @return created fix command
366 */
367 static FixCommand fixChangeKey(final String oldKey, final String newKey) {
368 return new FixCommand() {
369 @Override
370 public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
371 return new ChangePropertyKeyCommand(p,
372 TagCheck.insertArguments(matchingSelector, oldKey, p),
373 TagCheck.insertArguments(matchingSelector, newKey, p));
374 }
375
376 @Override
377 public String toString() {
378 return "fixChangeKey: " + oldKey + " => " + newKey;
379 }
380 };
381 }
382 }
383
384 final MultiMap<String, TagCheck> checks = new MultiMap<>();
385
386 /**
387 * Result of {@link TagCheck#readMapCSS}
388 * @since 8936
389 */
390 public static class ParseResult {
391 /** Checks successfully parsed */
392 public final List<TagCheck> parseChecks;
393 /** Errors that occurred during parsing */
394 public final Collection<Throwable> parseErrors;
395
396 /**
397 * Constructs a new {@code ParseResult}.
398 * @param parseChecks Checks successfully parsed
399 * @param parseErrors Errors that occurred during parsing
400 */
401 public ParseResult(List<TagCheck> parseChecks, Collection<Throwable> parseErrors) {
402 this.parseChecks = parseChecks;
403 this.parseErrors = parseErrors;
404 }
405 }
406
407 /**
408 * Tag check.
409 */
410 public static class TagCheck implements Predicate<OsmPrimitive> {
411 /** The selector of this {@code TagCheck} */
412 protected final GroupedMapCSSRule rule;
413 /** Commands to apply in order to fix a matching primitive */
414 protected final List<FixCommand> fixCommands = new ArrayList<>();
415 /** Tags (or arbitraty strings) of alternatives to be presented to the user */
416 protected final List<String> alternatives = new ArrayList<>();
417 /** An {@link org.openstreetmap.josm.gui.mappaint.mapcss.Instruction.AssignmentInstruction}-{@link Severity} pair.
418 * Is evaluated on the matching primitive to give the error message. Map is checked to contain exactly one element. */
419 protected final Map<Instruction.AssignmentInstruction, Severity> errors = new HashMap<>();
420 /** Unit tests */
421 protected final Map<String, Boolean> assertions = new HashMap<>();
422 /** MapCSS Classes to set on matching primitives */
423 protected final Set<String> setClassExpressions = new HashSet<>();
424 /** Denotes whether the object should be deleted for fixing it */
425 protected boolean deletion;
426 /** A string used to group similar tests */
427 protected String group;
428
429 TagCheck(GroupedMapCSSRule rule) {
430 this.rule = rule;
431 }
432
433 private static final String POSSIBLE_THROWS = possibleThrows();
434
435 static final String possibleThrows() {
436 StringBuilder sb = new StringBuilder();
437 for (Severity s : Severity.values()) {
438 if (sb.length() > 0) {
439 sb.append('/');
440 }
441 sb.append("throw")
442 .append(s.name().charAt(0))
443 .append(s.name().substring(1).toLowerCase(Locale.ENGLISH));
444 }
445 return sb.toString();
446 }
447
448 static TagCheck ofMapCSSRule(final GroupedMapCSSRule rule) throws IllegalDataException {
449 final TagCheck check = new TagCheck(rule);
450 for (Instruction i : rule.declaration.instructions) {
451 if (i instanceof Instruction.AssignmentInstruction) {
452 final Instruction.AssignmentInstruction ai = (Instruction.AssignmentInstruction) i;
453 if (ai.isSetInstruction) {
454 check.setClassExpressions.add(ai.key);
455 continue;
456 }
457 try {
458 final String val = ai.val instanceof Expression
459 ? Optional.of(((Expression) ai.val).evaluate(new Environment())).map(Object::toString).orElse(null)
460 : ai.val instanceof String
461 ? (String) ai.val
462 : ai.val instanceof Keyword
463 ? ((Keyword) ai.val).val
464 : null;
465 if (ai.key.startsWith("throw")) {
466 try {
467 check.errors.put(ai, Severity.valueOf(ai.key.substring("throw".length()).toUpperCase(Locale.ENGLISH)));
468 } catch (IllegalArgumentException e) {
469 Logging.log(Logging.LEVEL_WARN,
470 "Unsupported "+ai.key+" instruction. Allowed instructions are "+POSSIBLE_THROWS+'.', e);
471 }
472 } else if ("fixAdd".equals(ai.key)) {
473 check.fixCommands.add(FixCommand.fixAdd(ai.val));
474 } else if ("fixRemove".equals(ai.key)) {
475 CheckParameterUtil.ensureThat(!(ai.val instanceof String) || !(val != null && val.contains("=")),
476 "Unexpected '='. Please only specify the key to remove in: " + ai);
477 check.fixCommands.add(FixCommand.fixRemove(ai.val));
478 } else if (val != null && "fixChangeKey".equals(ai.key)) {
479 CheckParameterUtil.ensureThat(val.contains("=>"), "Separate old from new key by '=>'!");
480 final String[] x = val.split("=>", 2);
481 check.fixCommands.add(FixCommand.fixChangeKey(Utils.removeWhiteSpaces(x[0]), Utils.removeWhiteSpaces(x[1])));
482 } else if (val != null && "fixDeleteObject".equals(ai.key)) {
483 CheckParameterUtil.ensureThat("this".equals(val), "fixDeleteObject must be followed by 'this'");
484 check.deletion = true;
485 } else if (val != null && "suggestAlternative".equals(ai.key)) {
486 check.alternatives.add(val);
487 } else if (val != null && "assertMatch".equals(ai.key)) {
488 check.assertions.put(val, Boolean.TRUE);
489 } else if (val != null && "assertNoMatch".equals(ai.key)) {
490 check.assertions.put(val, Boolean.FALSE);
491 } else if (val != null && "group".equals(ai.key)) {
492 check.group = val;
493 } else {
494 throw new IllegalDataException("Cannot add instruction " + ai.key + ": " + ai.val + '!');
495 }
496 } catch (IllegalArgumentException e) {
497 throw new IllegalDataException(e);
498 }
499 }
500 }
501 if (check.errors.isEmpty() && check.setClassExpressions.isEmpty()) {
502 throw new IllegalDataException(
503 "No "+POSSIBLE_THROWS+" given! You should specify a validation error message for " + rule.selectors);
504 } else if (check.errors.size() > 1) {
505 throw new IllegalDataException(
506 "More than one "+POSSIBLE_THROWS+" given! You should specify a single validation error message for "
507 + rule.selectors);
508 }
509 return check;
510 }
511
512 static ParseResult readMapCSS(Reader css) throws ParseException {
513 CheckParameterUtil.ensureParameterNotNull(css, "css");
514
515 final MapCSSStyleSource source = new MapCSSStyleSource("");
516 final MapCSSParser preprocessor = new MapCSSParser(css, MapCSSParser.LexicalState.PREPROCESSOR);
517 final StringReader mapcss = new StringReader(preprocessor.pp_root(source));
518 final MapCSSParser parser = new MapCSSParser(mapcss, MapCSSParser.LexicalState.DEFAULT);
519 parser.sheet(source);
520 // Ignore "meta" rule(s) from external rules of JOSM wiki
521 source.removeMetaRules();
522 // group rules with common declaration block
523 Map<Declaration, List<Selector>> g = new LinkedHashMap<>();
524 for (MapCSSRule rule : source.rules) {
525 if (!g.containsKey(rule.declaration)) {
526 List<Selector> sels = new ArrayList<>();
527 sels.add(rule.selector);
528 g.put(rule.declaration, sels);
529 } else {
530 g.get(rule.declaration).add(rule.selector);
531 }
532 }
533 List<TagCheck> parseChecks = new ArrayList<>();
534 for (Map.Entry<Declaration, List<Selector>> map : g.entrySet()) {
535 try {
536 parseChecks.add(TagCheck.ofMapCSSRule(
537 new GroupedMapCSSRule(map.getValue(), map.getKey())));
538 } catch (IllegalDataException e) {
539 Logging.error("Cannot add MapCss rule: "+e.getMessage());
540 source.logError(e);
541 }
542 }
543 return new ParseResult(parseChecks, source.getErrors());
544 }
545
546 @Override
547 public boolean test(OsmPrimitive primitive) {
548 // Tests whether the primitive contains a deprecated tag which is represented by this MapCSSTagChecker.
549 return whichSelectorMatchesPrimitive(primitive) != null;
550 }
551
552 Selector whichSelectorMatchesPrimitive(OsmPrimitive primitive) {
553 return whichSelectorMatchesEnvironment(new Environment(primitive));
554 }
555
556 Selector whichSelectorMatchesEnvironment(Environment env) {
557 for (Selector i : rule.selectors) {
558 env.clearSelectorMatchingInformation();
559 if (i.matches(env)) {
560 return i;
561 }
562 }
563 return null;
564 }
565
566 /**
567 * Determines the {@code index}-th key/value/tag (depending on {@code type}) of the
568 * {@link org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector}.
569 * @param matchingSelector matching selector
570 * @param index index
571 * @param type selector type ("key", "value" or "tag")
572 * @param p OSM primitive
573 * @return argument value, can be {@code null}
574 */
575 static String determineArgument(OptimizedGeneralSelector matchingSelector, int index, String type, OsmPrimitive p) {
576 try {
577 final Condition c = matchingSelector.getConditions().get(index);
578 final Tag tag = c instanceof Condition.ToTagConvertable
579 ? ((Condition.ToTagConvertable) c).asTag(p)
580 : null;
581 if (tag == null) {
582 return null;
583 } else if ("key".equals(type)) {
584 return tag.getKey();
585 } else if ("value".equals(type)) {
586 return tag.getValue();
587 } else if ("tag".equals(type)) {
588 return tag.toString();
589 }
590 } catch (IndexOutOfBoundsException ignore) {
591 Logging.debug(ignore);
592 }
593 return null;
594 }
595
596 /**
597 * Replaces occurrences of <code>{i.key}</code>, <code>{i.value}</code>, <code>{i.tag}</code> in {@code s} by the corresponding
598 * key/value/tag of the {@code index}-th {@link Condition} of {@code matchingSelector}.
599 * @param matchingSelector matching selector
600 * @param s any string
601 * @param p OSM primitive
602 * @return string with arguments inserted
603 */
604 static String insertArguments(Selector matchingSelector, String s, OsmPrimitive p) {
605 if (s != null && matchingSelector instanceof Selector.ChildOrParentSelector) {
606 return insertArguments(((Selector.ChildOrParentSelector) matchingSelector).right, s, p);
607 } else if (s == null || !(matchingSelector instanceof Selector.OptimizedGeneralSelector)) {
608 return s;
609 }
610 final Matcher m = Pattern.compile("\\{(\\d+)\\.(key|value|tag)\\}").matcher(s);
611 final StringBuffer sb = new StringBuffer();
612 while (m.find()) {
613 final String argument = determineArgument((Selector.OptimizedGeneralSelector) matchingSelector,
614 Integer.parseInt(m.group(1)), m.group(2), p);
615 try {
616 // Perform replacement with null-safe + regex-safe handling
617 m.appendReplacement(sb, String.valueOf(argument).replace("^(", "").replace(")$", ""));
618 } catch (IndexOutOfBoundsException | IllegalArgumentException e) {
619 Logging.log(Logging.LEVEL_ERROR, tr("Unable to replace argument {0} in {1}: {2}", argument, sb, e.getMessage()), e);
620 }
621 }
622 m.appendTail(sb);
623 return sb.toString();
624 }
625
626 /**
627 * Constructs a fix in terms of a {@link org.openstreetmap.josm.command.Command} for the {@link OsmPrimitive}
628 * if the error is fixable, or {@code null} otherwise.
629 *
630 * @param p the primitive to construct the fix for
631 * @return the fix or {@code null}
632 */
633 Command fixPrimitive(OsmPrimitive p) {
634 if (fixCommands.isEmpty() && !deletion) {
635 return null;
636 }
637 try {
638 final Selector matchingSelector = whichSelectorMatchesPrimitive(p);
639 Collection<Command> cmds = new LinkedList<>();
640 for (FixCommand fixCommand : fixCommands) {
641 cmds.add(fixCommand.createCommand(p, matchingSelector));
642 }
643 if (deletion && !p.isDeleted()) {
644 cmds.add(new DeleteCommand(p));
645 }
646 return new SequenceCommand(tr("Fix of {0}", getDescriptionForMatchingSelector(p, matchingSelector)), cmds);
647 } catch (IllegalArgumentException e) {
648 Logging.error(e);
649 return null;
650 }
651 }
652
653 /**
654 * Constructs a (localized) message for this deprecation check.
655 * @param p OSM primitive
656 *
657 * @return a message
658 */
659 String getMessage(OsmPrimitive p) {
660 if (errors.isEmpty()) {
661 // Return something to avoid NPEs
662 return rule.declaration.toString();
663 } else {
664 final Object val = errors.keySet().iterator().next().val;
665 return String.valueOf(
666 val instanceof Expression
667 ? ((Expression) val).evaluate(new Environment(p))
668 : val
669 );
670 }
671 }
672
673 /**
674 * Constructs a (localized) description for this deprecation check.
675 * @param p OSM primitive
676 *
677 * @return a description (possibly with alternative suggestions)
678 * @see #getDescriptionForMatchingSelector
679 */
680 String getDescription(OsmPrimitive p) {
681 if (alternatives.isEmpty()) {
682 return getMessage(p);
683 } else {
684 /* I18N: {0} is the test error message and {1} is an alternative */
685 return tr("{0}, use {1} instead", getMessage(p), Utils.join(tr(" or "), alternatives));
686 }
687 }
688
689 /**
690 * Constructs a (localized) description for this deprecation check
691 * where any placeholders are replaced by values of the matched selector.
692 *
693 * @param matchingSelector matching selector
694 * @param p OSM primitive
695 * @return a description (possibly with alternative suggestions)
696 */
697 String getDescriptionForMatchingSelector(OsmPrimitive p, Selector matchingSelector) {
698 return insertArguments(matchingSelector, getDescription(p), p);
699 }
700
701 Severity getSeverity() {
702 return errors.isEmpty() ? null : errors.values().iterator().next();
703 }
704
705 @Override
706 public String toString() {
707 return getDescription(null);
708 }
709
710 /**
711 * Constructs a {@link TestError} for the given primitive, or returns null if the primitive does not give rise to an error.
712 *
713 * @param p the primitive to construct the error for
714 * @return an instance of {@link TestError}, or returns null if the primitive does not give rise to an error.
715 */
716 TestError getErrorForPrimitive(OsmPrimitive p) {
717 final Environment env = new Environment(p);
718 return getErrorForPrimitive(p, whichSelectorMatchesEnvironment(env), env, null);
719 }
720
721 TestError getErrorForPrimitive(OsmPrimitive p, Selector matchingSelector, Environment env, Test tester) {
722 if (matchingSelector != null && !errors.isEmpty()) {
723 final Command fix = fixPrimitive(p);
724 final String description = getDescriptionForMatchingSelector(p, matchingSelector);
725 final String description1 = group == null ? description : group;
726 final String description2 = group == null ? null : description;
727 final List<OsmPrimitive> primitives;
728 if (env.child instanceof OsmPrimitive) {
729 primitives = Arrays.asList(p, (OsmPrimitive) env.child);
730 } else {
731 primitives = Collections.singletonList(p);
732 }
733 final TestError.Builder error = TestError.builder(tester, getSeverity(), 3000)
734 .messageWithManuallyTranslatedDescription(description1, description2, matchingSelector.toString())
735 .primitives(primitives);
736 if (fix != null) {
737 return error.fix(() -> fix).build();
738 } else {
739 return error.build();
740 }
741 } else {
742 return null;
743 }
744 }
745
746 /**
747 * Returns the set of tagchecks on which this check depends on.
748 * @param schecks the collection of tagcheks to search in
749 * @return the set of tagchecks on which this check depends on
750 * @since 7881
751 */
752 public Set<TagCheck> getTagCheckDependencies(Collection<TagCheck> schecks) {
753 Set<TagCheck> result = new HashSet<>();
754 Set<String> classes = getClassesIds();
755 if (schecks != null && !classes.isEmpty()) {
756 for (TagCheck tc : schecks) {
757 if (this.equals(tc)) {
758 continue;
759 }
760 for (String id : tc.setClassExpressions) {
761 if (classes.contains(id)) {
762 result.add(tc);
763 break;
764 }
765 }
766 }
767 }
768 return result;
769 }
770
771 /**
772 * Returns the list of ids of all MapCSS classes referenced in the rule selectors.
773 * @return the list of ids of all MapCSS classes referenced in the rule selectors
774 * @since 7881
775 */
776 public Set<String> getClassesIds() {
777 Set<String> result = new HashSet<>();
778 for (Selector s : rule.selectors) {
779 if (s instanceof AbstractSelector) {
780 for (Condition c : ((AbstractSelector) s).getConditions()) {
781 if (c instanceof ClassCondition) {
782 result.add(((ClassCondition) c).id);
783 }
784 }
785 }
786 }
787 return result;
788 }
789 }
790
791 static class MapCSSTagCheckerAndRule extends MapCSSTagChecker {
792 public final GroupedMapCSSRule rule;
793
794 MapCSSTagCheckerAndRule(GroupedMapCSSRule rule) {
795 this.rule = rule;
796 }
797
798 @Override
799 public synchronized boolean equals(Object obj) {
800 return super.equals(obj)
801 || (obj instanceof TagCheck && rule.equals(((TagCheck) obj).rule))
802 || (obj instanceof GroupedMapCSSRule && rule.equals(obj));
803 }
804
805 @Override
806 public synchronized int hashCode() {
807 return Objects.hash(super.hashCode(), rule);
808 }
809
810 @Override
811 public String toString() {
812 return "MapCSSTagCheckerAndRule [rule=" + rule + ']';
813 }
814 }
815
816 /**
817 * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}.
818 * @param p The OSM primitive
819 * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned
820 * @return all errors for the given primitive, with or without those of "info" severity
821 */
822 public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) {
823 final List<TestError> res = new ArrayList<>();
824 if (indexData == null)
825 indexData = new IndexData(checks);
826
827 MapCSSRuleIndex matchingRuleIndex = indexData.get(p);
828
829 Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
830 // the declaration indices are sorted, so it suffices to save the last used index
831 Declaration lastDeclUsed = null;
832
833 Iterator<MapCSSRule> candidates = matchingRuleIndex.getRuleCandidates(p);
834 while (candidates.hasNext()) {
835 MapCSSRule r = candidates.next();
836 env.clearSelectorMatchingInformation();
837 if (r.selector.matches(env)) { // as side effect env.parent will be set (if s is a child selector)
838 TagCheck check = indexData.getCheck(r);
839 if (check != null) {
840 boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity;
841 // Do not run "information" level checks if not wanted, unless they also set a MapCSS class
842 if (ignoreError && check.setClassExpressions.isEmpty()) {
843 continue;
844 }
845 if (r.declaration == lastDeclUsed)
846 continue; // don't apply one declaration more than once
847 lastDeclUsed = r.declaration;
848
849 r.declaration.execute(env);
850 if (!ignoreError && !check.errors.isEmpty()) {
851 final TestError error = check.getErrorForPrimitive(p, r.selector, env, new MapCSSTagCheckerAndRule(check.rule));
852 if (error != null) {
853 res.add(error);
854 }
855 }
856
857 }
858 }
859 }
860 return res;
861 }
862
863 private static Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity,
864 Collection<Set<TagCheck>> checksCol) {
865 final List<TestError> r = new ArrayList<>();
866 final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
867 for (Set<TagCheck> schecks : checksCol) {
868 for (TagCheck check : schecks) {
869 boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity;
870 // Do not run "information" level checks if not wanted, unless they also set a MapCSS class
871 if (ignoreError && check.setClassExpressions.isEmpty()) {
872 continue;
873 }
874 final Selector selector = check.whichSelectorMatchesEnvironment(env);
875 if (selector != null) {
876 check.rule.declaration.execute(env);
877 if (!ignoreError && !check.errors.isEmpty()) {
878 final TestError error = check.getErrorForPrimitive(p, selector, env, new MapCSSTagCheckerAndRule(check.rule));
879 if (error != null) {
880 r.add(error);
881 }
882 }
883 }
884 }
885 }
886 return r;
887 }
888
889 /**
890 * Visiting call for primitives.
891 *
892 * @param p The primitive to inspect.
893 */
894 @Override
895 public void check(OsmPrimitive p) {
896 errors.addAll(getErrorsForPrimitive(p, ValidatorPrefHelper.PREF_OTHER.get()));
897 }
898
899 /**
900 * Adds a new MapCSS config file from the given URL.
901 * @param url The unique URL of the MapCSS config file
902 * @return List of tag checks and parsing errors, or null
903 * @throws ParseException if the config file does not match MapCSS syntax
904 * @throws IOException if any I/O error occurs
905 * @since 7275
906 */
907 public synchronized ParseResult addMapCSS(String url) throws ParseException, IOException {
908 CheckParameterUtil.ensureParameterNotNull(url, "url");
909 ParseResult result;
910 try (CachedFile cache = new CachedFile(url);
911 InputStream zip = cache.findZipEntryInputStream("validator.mapcss", "");
912 InputStream s = zip != null ? zip : cache.getInputStream();
913 Reader reader = new BufferedReader(UTFInputStreamReader.create(s))) {
914 if (zip != null)
915 I18n.addTexts(cache.getFile());
916 result = TagCheck.readMapCSS(reader);
917 checks.remove(url);
918 checks.putAll(url, result.parseChecks);
919 indexData = null;
920 // Check assertions, useful for development of local files
921 if (Config.getPref().getBoolean("validator.check_assert_local_rules", false) && Utils.isLocalUrl(url)) {
922 for (String msg : checkAsserts(result.parseChecks)) {
923 Logging.warn(msg);
924 }
925 }
926 }
927 return result;
928 }
929
930 @Override
931 public synchronized void initialize() throws Exception {
932 checks.clear();
933 indexData = null;
934 for (SourceEntry source : new ValidatorPrefHelper().get()) {
935 if (!source.active) {
936 continue;
937 }
938 String i = source.url;
939 try {
940 if (!i.startsWith("resource:")) {
941 Logging.info(tr("Adding {0} to tag checker", i));
942 } else if (Logging.isDebugEnabled()) {
943 Logging.debug(tr("Adding {0} to tag checker", i));
944 }
945 addMapCSS(i);
946 if (Config.getPref().getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) {
947 FileWatcher.getDefaultInstance().registerSource(source);
948 }
949 } catch (IOException | IllegalStateException | IllegalArgumentException ex) {
950 Logging.warn(tr("Failed to add {0} to tag checker", i));
951 Logging.log(Logging.LEVEL_WARN, ex);
952 } catch (ParseException | TokenMgrError ex) {
953 Logging.warn(tr("Failed to add {0} to tag checker", i));
954 Logging.warn(ex);
955 }
956 }
957 }
958
959 /**
960 * Checks that rule assertions are met for the given set of TagChecks.
961 * @param schecks The TagChecks for which assertions have to be checked
962 * @return A set of error messages, empty if all assertions are met
963 * @since 7356
964 */
965 public Set<String> checkAsserts(final Collection<TagCheck> schecks) {
966 Set<String> assertionErrors = new LinkedHashSet<>();
967 final DataSet ds = new DataSet();
968 for (final TagCheck check : schecks) {
969 Logging.debug("Check: {0}", check);
970 for (final Map.Entry<String, Boolean> i : check.assertions.entrySet()) {
971 Logging.debug("- Assertion: {0}", i);
972 final OsmPrimitive p = OsmUtils.createPrimitive(i.getKey());
973 // Build minimal ordered list of checks to run to test the assertion
974 List<Set<TagCheck>> checksToRun = new ArrayList<>();
975 Set<TagCheck> checkDependencies = check.getTagCheckDependencies(schecks);
976 if (!checkDependencies.isEmpty()) {
977 checksToRun.add(checkDependencies);
978 }
979 checksToRun.add(Collections.singleton(check));
980 // Add primitive to dataset to avoid DataIntegrityProblemException when evaluating selectors
981 ds.addPrimitive(p);
982 final Collection<TestError> pErrors = getErrorsForPrimitive(p, true, checksToRun);
983 Logging.debug("- Errors: {0}", pErrors);
984 @SuppressWarnings({"EqualsBetweenInconvertibleTypes", "EqualsIncompatibleType"})
985 final boolean isError = pErrors.stream().anyMatch(e -> e.getTester().equals(check.rule));
986 if (isError != i.getValue()) {
987 final String error = MessageFormat.format("Expecting test ''{0}'' (i.e., {1}) to {2} {3} (i.e., {4})",
988 check.getMessage(p), check.rule.selectors, i.getValue() ? "match" : "not match", i.getKey(), p.getKeys());
989 assertionErrors.add(error);
990 }
991 ds.removePrimitive(p);
992 }
993 }
994 return assertionErrors;
995 }
996
997 @Override
998 public synchronized int hashCode() {
999 return Objects.hash(super.hashCode(), checks);
1000 }
1001
1002 @Override
1003 public synchronized boolean equals(Object obj) {
1004 if (this == obj) return true;
1005 if (obj == null || getClass() != obj.getClass()) return false;
1006 if (!super.equals(obj)) return false;
1007 MapCSSTagChecker that = (MapCSSTagChecker) obj;
1008 return Objects.equals(checks, that.checks);
1009 }
1010
1011 /**
1012 * Reload tagchecker rule.
1013 * @param rule tagchecker rule to reload
1014 * @since 12825
1015 */
1016 public static void reloadRule(SourceEntry rule) {
1017 MapCSSTagChecker tagChecker = OsmValidator.getTest(MapCSSTagChecker.class);
1018 if (tagChecker != null) {
1019 try {
1020 tagChecker.addMapCSS(rule.url);
1021 } catch (IOException | ParseException | TokenMgrError e) {
1022 Logging.warn(e);
1023 }
1024 }
1025 }
1026
1027 @Override
1028 public void startTest(ProgressMonitor progressMonitor) {
1029 super.startTest(progressMonitor);
1030 if (indexData == null) {
1031 indexData = new IndexData(checks);
1032 }
1033 }
1034
1035 @Override
1036 public void endTest() {
1037 super.endTest();
1038 // no need to keep the index, it is quickly build and doubles the memory needs
1039 indexData = null;
1040 }
1041
1042}
Note: See TracBrowser for help on using the repository browser.