source: josm/trunk/scripts/SyncEditorLayerIndex.java@ 15702

Last change on this file since 15702 was 15699, checked in by Klumbumbus, 5 years ago

see #18172 - fix typo in message, include actual categories in message
  • Property svn:eol-style set to native
File size: 64.1 KB
Line 
1// License: GPL. For details, see LICENSE file.
2import static java.nio.charset.StandardCharsets.UTF_8;
3import static org.apache.commons.lang3.StringUtils.isBlank;
4import static org.apache.commons.lang3.StringUtils.isNotBlank;
5
6import java.io.BufferedReader;
7import java.io.IOException;
8import java.io.InputStreamReader;
9import java.io.OutputStream;
10import java.io.OutputStreamWriter;
11import java.lang.reflect.Field;
12import java.net.MalformedURLException;
13import java.net.URL;
14import java.nio.file.Files;
15import java.nio.file.Paths;
16import java.text.DecimalFormat;
17import java.text.ParseException;
18import java.text.SimpleDateFormat;
19import java.util.ArrayList;
20import java.util.Arrays;
21import java.util.Calendar;
22import java.util.Collection;
23import java.util.Collections;
24import java.util.Date;
25import java.util.HashMap;
26import java.util.LinkedList;
27import java.util.List;
28import java.util.Locale;
29import java.util.Map;
30import java.util.Map.Entry;
31import java.util.Objects;
32import java.util.Set;
33import java.util.function.BiConsumer;
34import java.util.function.Function;
35import java.util.regex.Matcher;
36import java.util.regex.Pattern;
37import java.util.stream.Collectors;
38
39import javax.json.Json;
40import javax.json.JsonArray;
41import javax.json.JsonNumber;
42import javax.json.JsonObject;
43import javax.json.JsonReader;
44import javax.json.JsonString;
45import javax.json.JsonValue;
46
47import org.openstreetmap.gui.jmapviewer.Coordinate;
48import org.openstreetmap.josm.data.Preferences;
49import org.openstreetmap.josm.data.imagery.ImageryInfo;
50import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
51import org.openstreetmap.josm.data.imagery.Shape;
52import org.openstreetmap.josm.data.preferences.JosmBaseDirectories;
53import org.openstreetmap.josm.data.preferences.JosmUrls;
54import org.openstreetmap.josm.data.projection.Projections;
55import org.openstreetmap.josm.data.validation.routines.DomainValidator;
56import org.openstreetmap.josm.io.imagery.ImageryReader;
57import org.openstreetmap.josm.spi.preferences.Config;
58import org.openstreetmap.josm.tools.ImageProvider;
59import org.openstreetmap.josm.tools.JosmRuntimeException;
60import org.openstreetmap.josm.tools.Logging;
61import org.openstreetmap.josm.tools.OptionParser;
62import org.openstreetmap.josm.tools.OptionParser.OptionCount;
63import org.openstreetmap.josm.tools.ReflectionUtils;
64import org.xml.sax.SAXException;
65
66/**
67 * Compare and analyse the differences of the editor layer index and the JOSM imagery list.
68 * The goal is to keep both lists in sync.
69 *
70 * The editor layer index project (https://github.com/osmlab/editor-layer-index)
71 * provides also a version in the JOSM format, but the GEOJSON is the original source
72 * format, so we read that.
73 *
74 * How to run:
75 * -----------
76 *
77 * Main JOSM binary needs to be in classpath, e.g.
78 *
79 * $ java -cp ../dist/josm-custom.jar SyncEditorLayerIndex
80 *
81 * Add option "-h" to show the available command line flags.
82 */
83@SuppressWarnings("unchecked")
84public class SyncEditorLayerIndex {
85
86 private static final int MAXLEN = 140;
87
88 private List<ImageryInfo> josmEntries;
89 private JsonArray eliEntries;
90
91 private final Map<String, JsonObject> eliUrls = new HashMap<>();
92 private final Map<String, ImageryInfo> josmUrls = new HashMap<>();
93 private final Map<String, ImageryInfo> josmMirrors = new HashMap<>();
94 private static final Map<String, String> oldproj = new HashMap<>();
95 private static final List<String> ignoreproj = new LinkedList<>();
96
97 private static String eliInputFile = "imagery_eli.geojson";
98 private static String josmInputFile = "imagery_josm.imagery.xml";
99 private static String ignoreInputFile = "imagery_josm.ignores.txt";
100 private static OutputStream outputFile;
101 private static OutputStreamWriter outputStream;
102 private static String optionOutput;
103 private static boolean optionShorten;
104 private static boolean optionNoSkip;
105 private static boolean optionXhtmlBody;
106 private static boolean optionXhtml;
107 private static String optionEliXml;
108 private static String optionJosmXml;
109 private static String optionEncoding;
110 private static boolean optionNoEli;
111 private Map<String, String> skip = new HashMap<>();
112
113 /**
114 * Main method.
115 * @param args program arguments
116 * @throws IOException if any I/O error occurs
117 * @throws ReflectiveOperationException if any reflective operation error occurs
118 * @throws SAXException if any SAX error occurs
119 */
120 public static void main(String[] args) throws IOException, SAXException, ReflectiveOperationException {
121 Locale.setDefault(Locale.ROOT);
122 parseCommandLineArguments(args);
123 Config.setUrlsProvider(JosmUrls.getInstance());
124 Preferences pref = new Preferences(JosmBaseDirectories.getInstance());
125 Config.setPreferencesInstance(pref);
126 pref.init(false);
127 SyncEditorLayerIndex script = new SyncEditorLayerIndex();
128 script.setupProj();
129 script.loadSkip();
130 script.start();
131 script.loadJosmEntries();
132 if (optionJosmXml != null) {
133 try (OutputStreamWriter stream = new OutputStreamWriter(Files.newOutputStream(Paths.get(optionJosmXml)), UTF_8)) {
134 script.printentries(script.josmEntries, stream);
135 }
136 }
137 script.loadELIEntries();
138 if (optionEliXml != null) {
139 try (OutputStreamWriter stream = new OutputStreamWriter(Files.newOutputStream(Paths.get(optionEliXml)), UTF_8)) {
140 script.printentries(script.eliEntries, stream);
141 }
142 }
143 script.checkInOneButNotTheOther();
144 script.checkCommonEntries();
145 script.end();
146 if (outputStream != null) {
147 outputStream.close();
148 }
149 if (outputFile != null) {
150 outputFile.close();
151 }
152 }
153
154 /**
155 * Displays help on the console
156 */
157 private static void showHelp() {
158 System.out.println(getHelp());
159 System.exit(0);
160 }
161
162 static String getHelp() {
163 return "usage: java -cp build SyncEditorLayerIndex\n" +
164 "-c,--encoding <encoding> output encoding (defaults to UTF-8 or cp850 on Windows)\n" +
165 "-e,--eli_input <eli_input> Input file for the editor layer index (geojson). " +
166 "Default is imagery_eli.geojson (current directory).\n" +
167 "-h,--help show this help\n" +
168 "-i,--ignore_input <ignore_input> Input file for the ignore list. Default is imagery_josm.ignores.txt (current directory).\n" +
169 "-j,--josm_input <josm_input> Input file for the JOSM imagery list (xml). " +
170 "Default is imagery_josm.imagery.xml (current directory).\n" +
171 "-m,--noeli don't show output for ELI problems\n" +
172 "-n,--noskip don't skip known entries\n" +
173 "-o,--output <output> Output file, - prints to stdout (default: -)\n" +
174 "-p,--elixml <elixml> ELI entries for use in JOSM as XML file (incomplete)\n" +
175 "-q,--josmxml <josmxml> JOSM entries reoutput as XML file (incomplete)\n" +
176 "-s,--shorten shorten the output, so it is easier to read in a console window\n" +
177 "-x,--xhtmlbody create XHTML body for display in a web page\n" +
178 "-X,--xhtml create XHTML for display in a web page\n";
179 }
180
181 /**
182 * Parse command line arguments.
183 * @param args program arguments
184 * @throws IOException in case of I/O error
185 */
186 static void parseCommandLineArguments(String[] args) throws IOException {
187 new OptionParser("JOSM/ELI synchronization script")
188 .addFlagParameter("help", SyncEditorLayerIndex::showHelp)
189 .addShortAlias("help", "h")
190 .addArgumentParameter("output", OptionCount.OPTIONAL, x -> optionOutput = x)
191 .addShortAlias("output", "o")
192 .addArgumentParameter("eli_input", OptionCount.OPTIONAL, x -> eliInputFile = x)
193 .addShortAlias("eli_input", "e")
194 .addArgumentParameter("josm_input", OptionCount.OPTIONAL, x -> josmInputFile = x)
195 .addShortAlias("josm_input", "j")
196 .addArgumentParameter("ignore_input", OptionCount.OPTIONAL, x -> ignoreInputFile = x)
197 .addShortAlias("ignore_input", "i")
198 .addFlagParameter("shorten", () -> optionShorten = true)
199 .addShortAlias("shorten", "s")
200 .addFlagParameter("noskip", () -> optionNoSkip = true)
201 .addShortAlias("noskip", "n")
202 .addFlagParameter("xhtmlbody", () -> optionXhtmlBody = true)
203 .addShortAlias("xhtmlbody", "x")
204 .addFlagParameter("xhtml", () -> optionXhtml = true)
205 .addShortAlias("xhtml", "X")
206 .addArgumentParameter("elixml", OptionCount.OPTIONAL, x -> optionEliXml = x)
207 .addShortAlias("elixml", "p")
208 .addArgumentParameter("josmxml", OptionCount.OPTIONAL, x -> optionJosmXml = x)
209 .addShortAlias("josmxml", "q")
210 .addFlagParameter("noeli", () -> optionNoEli = true)
211 .addShortAlias("noeli", "m")
212 .addArgumentParameter("encoding", OptionCount.OPTIONAL, x -> optionEncoding = x)
213 .addShortAlias("encoding", "c")
214 .parseOptionsOrExit(Arrays.asList(args));
215
216 if (optionOutput != null && !"-".equals(optionOutput)) {
217 outputFile = Files.newOutputStream(Paths.get(optionOutput));
218 outputStream = new OutputStreamWriter(outputFile, optionEncoding != null ? optionEncoding : "UTF-8");
219 } else if (optionEncoding != null) {
220 outputStream = new OutputStreamWriter(System.out, optionEncoding);
221 }
222 }
223
224 void setupProj() {
225 oldproj.put("EPSG:3359", "EPSG:3404");
226 oldproj.put("EPSG:3785", "EPSG:3857");
227 oldproj.put("EPSG:31297", "EPGS:31287");
228 oldproj.put("EPSG:31464", "EPSG:31468");
229 oldproj.put("EPSG:54004", "EPSG:3857");
230 oldproj.put("EPSG:102100", "EPSG:3857");
231 oldproj.put("EPSG:102113", "EPSG:3857");
232 oldproj.put("EPSG:900913", "EPGS:3857");
233 ignoreproj.add("EPSG:4267");
234 ignoreproj.add("EPSG:5221");
235 ignoreproj.add("EPSG:5514");
236 ignoreproj.add("EPSG:32019");
237 ignoreproj.add("EPSG:102066");
238 ignoreproj.add("EPSG:102067");
239 ignoreproj.add("EPSG:102685");
240 ignoreproj.add("EPSG:102711");
241 }
242
243 void loadSkip() throws IOException {
244 final Pattern pattern = Pattern.compile("^\\|\\| *(ELI|Ignore) *\\|\\| *\\{\\{\\{(.+)\\}\\}\\} *\\|\\|");
245 try (BufferedReader fr = new BufferedReader(new InputStreamReader(Files.newInputStream(Paths.get(ignoreInputFile)), UTF_8))) {
246 String line;
247
248 while ((line = fr.readLine()) != null) {
249 Matcher res = pattern.matcher(line);
250 if (res.matches()) {
251 if ("Ignore".equals(res.group(1))) {
252 skip.put(res.group(2), "green");
253 } else {
254 skip.put(res.group(2), "darkgoldenrod");
255 }
256 }
257 }
258 }
259 }
260
261 void myprintlnfinal(String s) {
262 if (outputStream != null) {
263 try {
264 outputStream.write(s + System.getProperty("line.separator"));
265 } catch (IOException e) {
266 throw new JosmRuntimeException(e);
267 }
268 } else {
269 System.out.println(s);
270 }
271 }
272
273 void myprintln(String s) {
274 if (skip.containsKey(s)) {
275 String color = skip.get(s);
276 skip.remove(s);
277 if (optionXhtmlBody || optionXhtml) {
278 s = "<pre style=\"margin:3px;color:"+color+"\">"
279 + s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
280 }
281 if (!optionNoSkip) {
282 return;
283 }
284 } else if (optionXhtmlBody || optionXhtml) {
285 String color =
286 s.startsWith("***") ? "black" :
287 ((s.startsWith("+ ") || s.startsWith("+++ ELI")) ? "blue" :
288 (s.startsWith("#") ? "indigo" :
289 (s.startsWith("!") ? "orange" : "red")));
290 s = "<pre style=\"margin:3px;color:"+color+"\">"+s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
291 }
292 if ((s.startsWith("+ ") || s.startsWith("+++ ELI") || s.startsWith("#")) && optionNoEli) {
293 return;
294 }
295 myprintlnfinal(s);
296 }
297
298 void start() {
299 if (optionXhtml) {
300 myprintlnfinal(
301 "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n");
302 myprintlnfinal(
303 "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>"+
304 "<title>JOSM - ELI differences</title></head><body>\n");
305 }
306 }
307
308 void end() {
309 for (String s : skip.keySet()) {
310 myprintln("+++ Obsolete skip entry: " + s);
311 }
312 if (optionXhtml) {
313 myprintlnfinal("</body></html>\n");
314 }
315 }
316
317 void loadELIEntries() throws IOException {
318 try (JsonReader jr = Json.createReader(new InputStreamReader(Files.newInputStream(Paths.get(eliInputFile)), UTF_8))) {
319 eliEntries = jr.readObject().getJsonArray("features");
320 }
321
322 for (JsonValue e : eliEntries) {
323 String url = getUrlStripped(e);
324 if (url.contains("{z}")) {
325 myprintln("+++ ELI-URL uses {z} instead of {zoom}: "+url);
326 url = url.replace("{z}", "{zoom}");
327 }
328 if (eliUrls.containsKey(url)) {
329 myprintln("+++ ELI-URL is not unique: "+url);
330 } else {
331 eliUrls.put(url, e.asJsonObject());
332 }
333 JsonArray s = e.asJsonObject().get("properties").asJsonObject().getJsonArray("available_projections");
334 if (s != null) {
335 String urlLc = url.toLowerCase(Locale.ENGLISH);
336 List<String> old = new LinkedList<>();
337 for (JsonValue p : s) {
338 String proj = ((JsonString) p).getString();
339 if (oldproj.containsKey(proj) || ("CRS:84".equals(proj) && !urlLc.contains("version=1.3"))) {
340 old.add(proj);
341 }
342 }
343 if (!old.isEmpty()) {
344 myprintln("+ ELI Projections "+String.join(", ", old)+" not useful: "+getDescription(e));
345 }
346 }
347 }
348 myprintln("*** Loaded "+eliEntries.size()+" entries (ELI). ***");
349 }
350
351 String cdata(String s) {
352 return cdata(s, false);
353 }
354
355 String cdata(String s, boolean escape) {
356 if (escape) {
357 return s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
358 } else if (s.matches("[<>&]"))
359 return "<![CDATA["+s+"]]>";
360 return s;
361 }
362
363 String maininfo(Object entry, String offset) {
364 String t = getType(entry);
365 String res = offset + "<type>"+t+"</type>\n";
366 res += offset + "<url>"+cdata(getUrl(entry))+"</url>\n";
367 if (getMinZoom(entry) != null)
368 res += offset + "<min-zoom>"+getMinZoom(entry)+"</min-zoom>\n";
369 if (getMaxZoom(entry) != null)
370 res += offset + "<max-zoom>"+getMaxZoom(entry)+"</max-zoom>\n";
371 if ("wms".equals(t)) {
372 List<String> p = getProjections(entry);
373 if (p != null) {
374 res += offset + "<projections>\n";
375 for (String c : p) {
376 res += offset + " <code>"+c+"</code>\n";
377 }
378 res += offset + "</projections>\n";
379 }
380 }
381 return res;
382 }
383
384 void printentries(List<?> entries, OutputStreamWriter stream) throws IOException {
385 DecimalFormat df = new DecimalFormat("#.#######");
386 df.setRoundingMode(java.math.RoundingMode.CEILING);
387 stream.write("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n");
388 stream.write("<imagery xmlns=\"http://josm.openstreetmap.de/maps-1.0\">\n");
389 for (Object e : entries) {
390 stream.write(" <entry"
391 + ("eli-best".equals(getQuality(e)) ? " eli-best=\"true\"" : "")
392 + (getOverlay(e) ? " overlay=\"true\"" : "")
393 + ">\n");
394 String t;
395 if (isNotBlank(t = getName(e)))
396 stream.write(" <name>"+cdata(t, true)+"</name>\n");
397 if (isNotBlank(t = getId(e)))
398 stream.write(" <id>"+t+"</id>\n");
399 if (isNotBlank(t = getCategory(e)))
400 stream.write(" <category>"+t+"</category>\n");
401 if (isNotBlank(t = getDate(e)))
402 stream.write(" <date>"+t+"</date>\n");
403 if (isNotBlank(t = getCountryCode(e)))
404 stream.write(" <country-code>"+t+"</country-code>\n");
405 if ((getDefault(e)))
406 stream.write(" <default>true</default>\n");
407 stream.write(maininfo(e, " "));
408 if (isNotBlank(t = getAttributionText(e)))
409 stream.write(" <attribution-text mandatory=\"true\">"+cdata(t, true)+"</attribution-text>\n");
410 if (isNotBlank(t = getAttributionUrl(e)))
411 stream.write(" <attribution-url>"+cdata(t)+"</attribution-url>\n");
412 if (isNotBlank(t = getLogoImage(e)))
413 stream.write(" <logo-image>"+cdata(t, true)+"</logo-image>\n");
414 if (isNotBlank(t = getLogoUrl(e)))
415 stream.write(" <logo-url>"+cdata(t)+"</logo-url>\n");
416 if (isNotBlank(t = getTermsOfUseText(e)))
417 stream.write(" <terms-of-use-text>"+cdata(t, true)+"</terms-of-use-text>\n");
418 if (isNotBlank(t = getTermsOfUseUrl(e)))
419 stream.write(" <terms-of-use-url>"+cdata(t)+"</terms-of-use-url>\n");
420 if (isNotBlank(t = getPermissionReferenceUrl(e)))
421 stream.write(" <permission-ref>"+cdata(t)+"</permission-ref>\n");
422 if ((getValidGeoreference(e)))
423 stream.write(" <valid-georeference>true</valid-georeference>\n");
424 if (isNotBlank(t = getIcon(e)))
425 stream.write(" <icon>"+cdata(t)+"</icon>\n");
426 for (Entry<String, String> d : getDescriptions(e).entrySet()) {
427 stream.write(" <description lang=\""+d.getKey()+"\">"+d.getValue()+"</description>\n");
428 }
429 for (ImageryInfo m : getMirrors(e)) {
430 stream.write(" <mirror>\n"+maininfo(m, " ")+" </mirror>\n");
431 }
432 double minlat = 1000;
433 double minlon = 1000;
434 double maxlat = -1000;
435 double maxlon = -1000;
436 String shapes = "";
437 String sep = "\n ";
438 try {
439 for (Shape s: getShapes(e)) {
440 shapes += " <shape>";
441 int i = 0;
442 for (Coordinate p: s.getPoints()) {
443 double lat = p.getLat();
444 double lon = p.getLon();
445 if (lat > maxlat) maxlat = lat;
446 if (lon > maxlon) maxlon = lon;
447 if (lat < minlat) minlat = lat;
448 if (lon < minlon) minlon = lon;
449 if ((i++ % 3) == 0) {
450 shapes += sep + " ";
451 }
452 shapes += "<point lat='"+df.format(lat)+"' lon='"+df.format(lon)+"'/>";
453 }
454 shapes += sep + "</shape>\n";
455 }
456 } catch (IllegalArgumentException ignored) {
457 Logging.trace(ignored);
458 }
459 if (!shapes.isEmpty()) {
460 stream.write(" <bounds min-lat='"+df.format(minlat)
461 +"' min-lon='"+df.format(minlon)
462 +"' max-lat='"+df.format(maxlat)
463 +"' max-lon='"+df.format(maxlon)+"'>\n");
464 stream.write(shapes + " </bounds>\n");
465 }
466 stream.write(" </entry>\n");
467 }
468 stream.write("</imagery>\n");
469 stream.close();
470 }
471
472 void loadJosmEntries() throws IOException, SAXException, ReflectiveOperationException {
473 try (ImageryReader reader = new ImageryReader(josmInputFile)) {
474 josmEntries = reader.parse();
475 }
476
477 for (ImageryInfo e : josmEntries) {
478 if (isBlank(getUrl(e))) {
479 myprintln("+++ JOSM-Entry without URL: " + getDescription(e));
480 continue;
481 }
482 if (isBlank(getName(e))) {
483 myprintln("+++ JOSM-Entry without Name: " + getDescription(e));
484 continue;
485 }
486 String url = getUrlStripped(e);
487 if (url.contains("{z}")) {
488 myprintln("+++ JOSM-URL uses {z} instead of {zoom}: "+url);
489 url = url.replace("{z}", "{zoom}");
490 }
491 if (josmUrls.containsKey(url)) {
492 myprintln("+++ JOSM-URL is not unique: "+url);
493 } else {
494 josmUrls.put(url, e);
495 }
496 for (ImageryInfo m : e.getMirrors()) {
497 url = getUrlStripped(m);
498 Field origNameField = ImageryInfo.class.getDeclaredField("origName");
499 ReflectionUtils.setObjectsAccessible(origNameField);
500 origNameField.set(m, m.getOriginalName().replaceAll(" mirror server( \\d+)?", ""));
501 if (josmUrls.containsKey(url)) {
502 myprintln("+++ JOSM-Mirror-URL is not unique: "+url);
503 } else {
504 josmUrls.put(url, m);
505 josmMirrors.put(url, m);
506 }
507 }
508 }
509 myprintln("*** Loaded "+josmEntries.size()+" entries (JOSM). ***");
510 }
511
512 void checkInOneButNotTheOther() {
513 List<String> le = new LinkedList<>(eliUrls.keySet());
514 List<String> lj = new LinkedList<>(josmUrls.keySet());
515
516 List<String> ke = new LinkedList<>(le);
517 for (String url : ke) {
518 if (lj.contains(url)) {
519 le.remove(url);
520 lj.remove(url);
521 }
522 }
523
524 if (!le.isEmpty() && !lj.isEmpty()) {
525 ke = new LinkedList<>(le);
526 for (String urle : ke) {
527 JsonObject e = eliUrls.get(urle);
528 String ide = getId(e);
529 String urlhttps = urle.replace("http:", "https:");
530 if (lj.contains(urlhttps)) {
531 myprintln("+ Missing https: "+getDescription(e));
532 eliUrls.put(urlhttps, eliUrls.get(urle));
533 eliUrls.remove(urle);
534 le.remove(urle);
535 lj.remove(urlhttps);
536 } else if (isNotBlank(ide)) {
537 List<String> kj = new LinkedList<>(lj);
538 for (String urlj : kj) {
539 ImageryInfo j = josmUrls.get(urlj);
540 String idj = getId(j);
541
542 if (ide.equals(idj) && Objects.equals(getType(j), getType(e))) {
543 myprintln("* URL for id "+idj+" differs ("+urle+"): "+getDescription(j));
544 le.remove(urle);
545 lj.remove(urlj);
546 // replace key for this entry with JOSM URL
547 eliUrls.remove(urle);
548 eliUrls.put(urlj, e);
549 break;
550 }
551 }
552 }
553 }
554 }
555
556 myprintln("*** URLs found in ELI but not in JOSM ("+le.size()+"): ***");
557 Collections.sort(le);
558 if (!le.isEmpty()) {
559 for (String l : le) {
560 myprintln("- " + getDescription(eliUrls.get(l)));
561 }
562 }
563 myprintln("*** URLs found in JOSM but not in ELI ("+lj.size()+"): ***");
564 Collections.sort(lj);
565 if (!lj.isEmpty()) {
566 for (String l : lj) {
567 myprintln("+ " + getDescription(josmUrls.get(l)));
568 }
569 }
570 }
571
572 void checkCommonEntries() {
573 doSameUrlButDifferentName();
574 doSameUrlButDifferentId();
575 doSameUrlButDifferentType();
576 doSameUrlButDifferentZoomBounds();
577 doSameUrlButDifferentCountryCode();
578 doSameUrlButDifferentQuality();
579 doSameUrlButDifferentDates();
580 doSameUrlButDifferentInformation();
581 doMismatchingShapes();
582 doMismatchingIcons();
583 doMismatchingCategories();
584 doMiscellaneousChecks();
585 }
586
587 void doSameUrlButDifferentName() {
588 myprintln("*** Same URL, but different name: ***");
589 for (String url : eliUrls.keySet()) {
590 JsonObject e = eliUrls.get(url);
591 if (!josmUrls.containsKey(url)) continue;
592 ImageryInfo j = josmUrls.get(url);
593 String ename = getName(e).replace("'", "\u2019");
594 String jname = getName(j).replace("'", "\u2019");
595 if (!ename.equals(jname)) {
596 myprintln("* Name differs ('"+getName(e)+"' != '"+getName(j)+"'): "+getUrl(j));
597 }
598 }
599 }
600
601 void doSameUrlButDifferentId() {
602 myprintln("*** Same URL, but different Id: ***");
603 for (String url : eliUrls.keySet()) {
604 JsonObject e = eliUrls.get(url);
605 if (!josmUrls.containsKey(url)) continue;
606 ImageryInfo j = josmUrls.get(url);
607 String ename = getId(e);
608 String jname = getId(j);
609 if (!Objects.equals(ename, jname)) {
610 myprintln("# Id differs ('"+getId(e)+"' != '"+getId(j)+"'): "+getUrl(j));
611 }
612 }
613 }
614
615 void doSameUrlButDifferentType() {
616 myprintln("*** Same URL, but different type: ***");
617 for (String url : eliUrls.keySet()) {
618 JsonObject e = eliUrls.get(url);
619 if (!josmUrls.containsKey(url)) continue;
620 ImageryInfo j = josmUrls.get(url);
621 if (!Objects.equals(getType(e), getType(j))) {
622 myprintln("* Type differs ("+getType(e)+" != "+getType(j)+"): "+getName(j)+" - "+getUrl(j));
623 }
624 }
625 }
626
627 void doSameUrlButDifferentZoomBounds() {
628 myprintln("*** Same URL, but different zoom bounds: ***");
629 for (String url : eliUrls.keySet()) {
630 JsonObject e = eliUrls.get(url);
631 if (!josmUrls.containsKey(url)) continue;
632 ImageryInfo j = josmUrls.get(url);
633
634 Integer eMinZoom = getMinZoom(e);
635 Integer jMinZoom = getMinZoom(j);
636 /* dont warn for entries copied from the base of the mirror */
637 if (eMinZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
638 jMinZoom = null;
639 if (!Objects.equals(eMinZoom, jMinZoom) && !(Objects.equals(eMinZoom, 0) && jMinZoom == null)) {
640 myprintln("* Minzoom differs ("+eMinZoom+" != "+jMinZoom+"): "+getDescription(j));
641 }
642 Integer eMaxZoom = getMaxZoom(e);
643 Integer jMaxZoom = getMaxZoom(j);
644 /* dont warn for entries copied from the base of the mirror */
645 if (eMaxZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
646 jMaxZoom = null;
647 if (!Objects.equals(eMaxZoom, jMaxZoom)) {
648 myprintln("* Maxzoom differs ("+eMaxZoom+" != "+jMaxZoom+"): "+getDescription(j));
649 }
650 }
651 }
652
653 void doSameUrlButDifferentCountryCode() {
654 myprintln("*** Same URL, but different country code: ***");
655 for (String url : eliUrls.keySet()) {
656 JsonObject e = eliUrls.get(url);
657 if (!josmUrls.containsKey(url)) continue;
658 ImageryInfo j = josmUrls.get(url);
659 String cce = getCountryCode(e);
660 if ("ZZ".equals(cce)) { /* special ELI country code */
661 cce = null;
662 }
663 if (cce != null && !cce.equals(getCountryCode(j))) {
664 myprintln("* Country code differs ("+getCountryCode(e)+" != "+getCountryCode(j)+"): "+getDescription(j));
665 }
666 }
667 }
668
669 void doSameUrlButDifferentQuality() {
670 myprintln("*** Same URL, but different quality: ***");
671 for (String url : eliUrls.keySet()) {
672 JsonObject e = eliUrls.get(url);
673 if (!josmUrls.containsKey(url)) {
674 String q = getQuality(e);
675 if ("eli-best".equals(q)) {
676 myprintln("- Quality best entry not in JOSM for "+getDescription(e));
677 }
678 continue;
679 }
680 ImageryInfo j = josmUrls.get(url);
681 if (!Objects.equals(getQuality(e), getQuality(j))) {
682 myprintln("* Quality differs ("+getQuality(e)+" != "+getQuality(j)+"): "+getDescription(j));
683 }
684 }
685 }
686
687 void doSameUrlButDifferentDates() {
688 myprintln("*** Same URL, but different dates: ***");
689 Pattern pattern = Pattern.compile("^(.*;)(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?$");
690 for (String url : eliUrls.keySet()) {
691 String ed = getDate(eliUrls.get(url));
692 if (!josmUrls.containsKey(url)) continue;
693 ImageryInfo j = josmUrls.get(url);
694 String jd = getDate(j);
695 // The forms 2015;- or -;2015 or 2015;2015 are handled equal to 2015
696 String ef = ed.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
697 // ELI has a strange and inconsistent used end_date definition, so we try again with subtraction by one
698 String ed2 = ed;
699 Matcher m = pattern.matcher(ed);
700 if (m.matches()) {
701 Calendar cal = Calendar.getInstance();
702 cal.set(Integer.valueOf(m.group(2)),
703 m.group(4) == null ? 0 : Integer.valueOf(m.group(4))-1,
704 m.group(6) == null ? 1 : Integer.valueOf(m.group(6)));
705 cal.add(Calendar.DAY_OF_MONTH, -1);
706 ed2 = m.group(1) + cal.get(Calendar.YEAR);
707 if (m.group(4) != null)
708 ed2 += "-" + String.format("%02d", cal.get(Calendar.MONTH)+1);
709 if (m.group(6) != null)
710 ed2 += "-" + String.format("%02d", cal.get(Calendar.DAY_OF_MONTH));
711 }
712 String ef2 = ed2.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
713 if (!ed.equals(jd) && !ef.equals(jd) && !ed2.equals(jd) && !ef2.equals(jd)) {
714 String t = "'"+ed+"'";
715 if (!ed.equals(ef)) {
716 t += " or '"+ef+"'";
717 }
718 if (jd.isEmpty()) {
719 myprintln("- Missing JOSM date ("+t+"): "+getDescription(j));
720 } else if (!ed.isEmpty()) {
721 myprintln("* Date differs ('"+t+"' != '"+jd+"'): "+getDescription(j));
722 } else if (!optionNoEli) {
723 myprintln("+ Missing ELI date ('"+jd+"'): "+getDescription(j));
724 }
725 }
726 }
727 }
728
729 void doSameUrlButDifferentInformation() {
730 myprintln("*** Same URL, but different information: ***");
731 for (String url : eliUrls.keySet()) {
732 if (!josmUrls.containsKey(url)) continue;
733 JsonObject e = eliUrls.get(url);
734 ImageryInfo j = josmUrls.get(url);
735
736 compareDescriptions(e, j);
737 comparePermissionReferenceUrls(e, j);
738 compareAttributionUrls(e, j);
739 compareAttributionTexts(e, j);
740 compareProjections(e, j);
741 compareDefaults(e, j);
742 compareOverlays(e, j);
743 compareNoTileHeaders(e, j);
744 }
745 }
746
747 void compareDescriptions(JsonObject e, ImageryInfo j) {
748 String et = getDescriptions(e).getOrDefault("en", "");
749 String jt = getDescriptions(j).getOrDefault("en", "");
750 if (!et.equals(jt)) {
751 if (jt.isEmpty()) {
752 myprintln("- Missing JOSM description ("+et+"): "+getDescription(j));
753 } else if (!et.isEmpty()) {
754 myprintln("* Description differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
755 } else if (!optionNoEli) {
756 myprintln("+ Missing ELI description ('"+jt+"'): "+getDescription(j));
757 }
758 }
759 }
760
761 void comparePermissionReferenceUrls(JsonObject e, ImageryInfo j) {
762 String et = getPermissionReferenceUrl(e);
763 String jt = getPermissionReferenceUrl(j);
764 String jt2 = getTermsOfUseUrl(j);
765 if (isBlank(jt)) jt = jt2;
766 if (!Objects.equals(et, jt)) {
767 if (isBlank(jt)) {
768 myprintln("- Missing JOSM license URL ("+et+"): "+getDescription(j));
769 } else if (isNotBlank(et)) {
770 String ethttps = et.replace("http:", "https:");
771 if (isBlank(jt2) || !(jt2.equals(ethttps) || jt2.equals(et+"/") || jt2.equals(ethttps+"/"))) {
772 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
773 myprintln("+ License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
774 } else {
775 String ja = getAttributionUrl(j);
776 if (ja != null && (ja.equals(et) || ja.equals(ethttps) || ja.equals(et+"/") || ja.equals(ethttps+"/"))) {
777 myprintln("+ ELI License URL in JOSM Attribution: "+getDescription(j));
778 } else {
779 myprintln("* License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
780 }
781 }
782 }
783 } else if (!optionNoEli) {
784 myprintln("+ Missing ELI license URL ('"+jt+"'): "+getDescription(j));
785 }
786 }
787 }
788
789 void compareAttributionUrls(JsonObject e, ImageryInfo j) {
790 String et = getAttributionUrl(e);
791 String jt = getAttributionUrl(j);
792 if (!Objects.equals(et, jt)) {
793 if (isBlank(jt)) {
794 myprintln("- Missing JOSM attribution URL ("+et+"): "+getDescription(j));
795 } else if (isNotBlank(et)) {
796 String ethttps = et.replace("http:", "https:");
797 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
798 myprintln("+ Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
799 } else {
800 myprintln("* Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
801 }
802 } else if (!optionNoEli) {
803 myprintln("+ Missing ELI attribution URL ('"+jt+"'): "+getDescription(j));
804 }
805 }
806 }
807
808 void compareAttributionTexts(JsonObject e, ImageryInfo j) {
809 String et = getAttributionText(e);
810 String jt = getAttributionText(j);
811 if (!Objects.equals(et, jt)) {
812 if (isBlank(jt)) {
813 myprintln("- Missing JOSM attribution text ("+et+"): "+getDescription(j));
814 } else if (isNotBlank(et)) {
815 myprintln("* Attribution text differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
816 } else if (!optionNoEli) {
817 myprintln("+ Missing ELI attribution text ('"+jt+"'): "+getDescription(j));
818 }
819 }
820 }
821
822 void compareProjections(JsonObject e, ImageryInfo j) {
823 String et = getProjections(e).stream().sorted().collect(Collectors.joining(" "));
824 String jt = getProjections(j).stream().sorted().collect(Collectors.joining(" "));
825 if (!Objects.equals(et, jt)) {
826 if (isBlank(jt)) {
827 String t = getType(e);
828 if ("wms_endpoint".equals(t) || "tms".equals(t)) {
829 myprintln("+ ELI projections for type "+t+": "+getDescription(j));
830 } else {
831 myprintln("- Missing JOSM projections ("+et+"): "+getDescription(j));
832 }
833 } else if (isNotBlank(et)) {
834 if ("EPSG:3857 EPSG:4326".equals(et) || "EPSG:3857".equals(et) || "EPSG:4326".equals(et)) {
835 myprintln("+ ELI has minimal projections ('"+et+"' != '"+jt+"'): "+getDescription(j));
836 } else {
837 myprintln("* Projections differ ('"+et+"' != '"+jt+"'): "+getDescription(j));
838 }
839 } else if (!optionNoEli && !"tms".equals(getType(e))) {
840 myprintln("+ Missing ELI projections ('"+jt+"'): "+getDescription(j));
841 }
842 }
843 }
844
845 void compareDefaults(JsonObject e, ImageryInfo j) {
846 boolean ed = getDefault(e);
847 boolean jd = getDefault(j);
848 if (ed != jd) {
849 if (!jd) {
850 myprintln("- Missing JOSM default: "+getDescription(j));
851 } else if (!optionNoEli) {
852 myprintln("+ Missing ELI default: "+getDescription(j));
853 }
854 }
855 }
856
857 void compareOverlays(JsonObject e, ImageryInfo j) {
858 boolean eo = getOverlay(e);
859 boolean jo = getOverlay(j);
860 if (eo != jo) {
861 if (!jo) {
862 myprintln("- Missing JOSM overlay flag: "+getDescription(j));
863 } else if (!optionNoEli) {
864 myprintln("+ Missing ELI overlay flag: "+getDescription(j));
865 }
866 }
867 }
868
869 void compareNoTileHeaders(JsonObject e, ImageryInfo j) {
870 Map<String, Set<String>> eh = getNoTileHeader(e);
871 Map<String, Set<String>> jh = getNoTileHeader(j);
872 if (!Objects.equals(eh, jh)) {
873 if (jh == null || jh.isEmpty()) {
874 myprintln("- Missing JOSM no tile headers ("+eh+"): "+getDescription(j));
875 } else if (eh != null && !eh.isEmpty()) {
876 myprintln("* No tile headers differ ('"+eh+"' != '"+jh+"'): "+getDescription(j));
877 } else if (!optionNoEli) {
878 myprintln("+ Missing ELI no tile headers ('"+jh+"'): "+getDescription(j));
879 }
880 }
881 }
882
883 void doMismatchingShapes() {
884 myprintln("*** Mismatching shapes: ***");
885 for (String url : josmUrls.keySet()) {
886 ImageryInfo j = josmUrls.get(url);
887 int num = 1;
888 for (Shape shape : getShapes(j)) {
889 List<Coordinate> p = shape.getPoints();
890 if (!p.get(0).equals(p.get(p.size()-1))) {
891 myprintln("+++ JOSM shape "+num+" unclosed: "+getDescription(j));
892 }
893 for (int nump = 1; nump < p.size(); ++nump) {
894 if (Objects.equals(p.get(nump-1), p.get(nump))) {
895 myprintln("+++ JOSM shape "+num+" double point at "+(nump-1)+": "+getDescription(j));
896 }
897 }
898 ++num;
899 }
900 }
901 for (String url : eliUrls.keySet()) {
902 JsonObject e = eliUrls.get(url);
903 int num = 1;
904 List<Shape> s = null;
905 try {
906 s = getShapes(e);
907 for (Shape shape : s) {
908 List<Coordinate> p = shape.getPoints();
909 if (!p.get(0).equals(p.get(p.size()-1)) && !optionNoEli) {
910 myprintln("+++ ELI shape "+num+" unclosed: "+getDescription(e));
911 }
912 for (int nump = 1; nump < p.size(); ++nump) {
913 if (Objects.equals(p.get(nump-1), p.get(nump))) {
914 myprintln("+++ ELI shape "+num+" double point at "+(nump-1)+": "+getDescription(e));
915 }
916 }
917 ++num;
918 }
919 } catch (IllegalArgumentException err) {
920 String desc = getDescription(e);
921 myprintln("* Invalid data in ELI geometry for "+desc+": "+err.getMessage());
922 }
923 if (s == null || !josmUrls.containsKey(url)) {
924 continue;
925 }
926 ImageryInfo j = josmUrls.get(url);
927 List<Shape> js = getShapes(j);
928 if (s.isEmpty() && !js.isEmpty()) {
929 if (!optionNoEli) {
930 myprintln("+ No ELI shape: "+getDescription(j));
931 }
932 } else if (js.isEmpty() && !s.isEmpty()) {
933 // don't report boundary like 5 point shapes as difference
934 if (s.size() != 1 || s.get(0).getPoints().size() != 5) {
935 myprintln("- No JOSM shape: "+getDescription(j));
936 }
937 } else if (s.size() != js.size()) {
938 myprintln("* Different number of shapes ("+s.size()+" != "+js.size()+"): "+getDescription(j));
939 } else {
940 boolean[] edone = new boolean[s.size()];
941 boolean[] jdone = new boolean[js.size()];
942 for (int enums = 0; enums < s.size(); ++enums) {
943 List<Coordinate> ep = s.get(enums).getPoints();
944 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
945 List<Coordinate> jp = js.get(jnums).getPoints();
946 if (ep.size() == jp.size() && !jdone[jnums]) {
947 boolean err = false;
948 for (int nump = 0; nump < ep.size() && !err; ++nump) {
949 Coordinate ept = ep.get(nump);
950 Coordinate jpt = jp.get(nump);
951 if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.00001)
952 err = true;
953 }
954 if (!err) {
955 edone[enums] = true;
956 jdone[jnums] = true;
957 break;
958 }
959 }
960 }
961 }
962 for (int enums = 0; enums < s.size(); ++enums) {
963 List<Coordinate> ep = s.get(enums).getPoints();
964 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
965 List<Coordinate> jp = js.get(jnums).getPoints();
966 if (ep.size() == jp.size() && !jdone[jnums]) {
967 boolean err = false;
968 for (int nump = 0; nump < ep.size() && !err; ++nump) {
969 Coordinate ept = ep.get(nump);
970 Coordinate jpt = jp.get(nump);
971 if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.00001) {
972 String numtxt = Integer.toString(enums+1);
973 if (enums != jnums) {
974 numtxt += '/' + Integer.toString(jnums+1);
975 }
976 myprintln("* Different coordinate for point "+(nump+1)+" of shape "+numtxt+": "+getDescription(j));
977 break;
978 }
979 }
980 edone[enums] = true;
981 jdone[jnums] = true;
982 break;
983 }
984 }
985 }
986 for (int enums = 0; enums < s.size(); ++enums) {
987 List<Coordinate> ep = s.get(enums).getPoints();
988 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
989 List<Coordinate> jp = js.get(jnums).getPoints();
990 if (!jdone[jnums]) {
991 String numtxt = Integer.toString(enums+1);
992 if (enums != jnums) {
993 numtxt += '/' + Integer.toString(jnums+1);
994 }
995 myprintln("* Different number of points for shape "+numtxt+" ("+ep.size()+" ! = "+jp.size()+"): "
996 + getDescription(j));
997 edone[enums] = true;
998 jdone[jnums] = true;
999 break;
1000 }
1001 }
1002 }
1003 }
1004 }
1005 }
1006
1007 void doMismatchingIcons() {
1008 myprintln("*** Mismatching icons: ***");
1009 doMismatching(this::compareIcons);
1010 }
1011
1012 void doMismatchingCategories() {
1013 myprintln("*** Mismatching categories: ***");
1014 doMismatching(this::compareCategories);
1015 }
1016
1017 void doMismatching(BiConsumer<ImageryInfo, JsonObject> comparator) {
1018 for (String url : eliUrls.keySet()) {
1019 if (josmUrls.containsKey(url)) {
1020 comparator.accept(josmUrls.get(url), eliUrls.get(url));
1021 }
1022 }
1023 }
1024
1025 void compareIcons(ImageryInfo j, JsonObject e) {
1026 String ij = getIcon(j);
1027 String ie = getIcon(e);
1028 boolean ijok = isNotBlank(ij);
1029 boolean ieok = isNotBlank(ie);
1030 if (ijok && !ieok) {
1031 if (!optionNoEli) {
1032 myprintln("+ No ELI icon: "+getDescription(j));
1033 }
1034 } else if (!ijok && ieok) {
1035 myprintln("- No JOSM icon: "+getDescription(j));
1036 } else if (ijok && ieok && !Objects.equals(ij, ie) && !(
1037 (ie.startsWith("https://osmlab.github.io/editor-layer-index/")
1038 || ie.startsWith("https://raw.githubusercontent.com/osmlab/editor-layer-index/")) &&
1039 ij.startsWith("data:"))) {
1040 String iehttps = ie.replace("http:", "https:");
1041 if (ij.equals(iehttps)) {
1042 myprintln("+ Different icons: "+getDescription(j));
1043 } else {
1044 myprintln("* Different icons: "+getDescription(j));
1045 }
1046 }
1047 }
1048
1049 void compareCategories(ImageryInfo j, JsonObject e) {
1050 String cj = getCategory(j);
1051 String ce = getCategory(e);
1052 boolean cjok = isNotBlank(cj);
1053 boolean ceok = isNotBlank(ce);
1054 if (cjok && !ceok) {
1055 if (!optionNoEli) {
1056 myprintln("+ No ELI category: "+getDescription(j));
1057 }
1058 } else if (!cjok && ceok) {
1059 myprintln("- No JOSM category: "+getDescription(j));
1060 } else if (cjok && ceok && !Objects.equals(cj, ce)) {
1061 myprintln("* Different categories ('"+ce+"' != '"+cj+"'): "+getDescription(j));
1062 }
1063 }
1064
1065 void doMiscellaneousChecks() {
1066 myprintln("*** Miscellaneous checks: ***");
1067 Map<String, ImageryInfo> josmIds = new HashMap<>();
1068 Collection<String> all = Projections.getAllProjectionCodes();
1069 DomainValidator dv = DomainValidator.getInstance();
1070 for (String url : josmUrls.keySet()) {
1071 ImageryInfo j = josmUrls.get(url);
1072 String id = getId(j);
1073 if ("wms".equals(getType(j))) {
1074 String urlLc = url.toLowerCase(Locale.ENGLISH);
1075 if (getProjections(j).isEmpty()) {
1076 myprintln("* WMS without projections: "+getDescription(j));
1077 } else {
1078 List<String> unsupported = new LinkedList<>();
1079 List<String> old = new LinkedList<>();
1080 for (String p : getProjectionsUnstripped(j)) {
1081 if ("CRS:84".equals(p)) {
1082 if (!urlLc.contains("version=1.3")) {
1083 myprintln("* CRS:84 without WMS 1.3: "+getDescription(j));
1084 }
1085 } else if (oldproj.containsKey(p)) {
1086 old.add(p);
1087 } else if (!all.contains(p) && !ignoreproj.contains(p)) {
1088 unsupported.add(p);
1089 }
1090 }
1091 if (!unsupported.isEmpty()) {
1092 myprintln("* Projections "+String.join(", ", unsupported)+" not supported by JOSM: "+getDescription(j));
1093 }
1094 for (String o : old) {
1095 myprintln("* Projection "+o+" is an old unsupported code and has been replaced by "+oldproj.get(o)+": "
1096 + getDescription(j));
1097 }
1098 }
1099 if (urlLc.contains("version=1.3") && !urlLc.contains("crs={proj}")) {
1100 myprintln("* WMS 1.3 with strange CRS specification: "+getDescription(j));
1101 } else if (urlLc.contains("version=1.1") && !urlLc.contains("srs={proj}")) {
1102 myprintln("* WMS 1.1 with strange SRS specification: "+getDescription(j));
1103 }
1104 }
1105 List<String> urls = new LinkedList<>();
1106 if (!"scanex".equals(getType(j))) {
1107 urls.add(url);
1108 }
1109 String jt = getPermissionReferenceUrl(j);
1110 if (isNotBlank(jt) && !"Public Domain".equalsIgnoreCase(jt))
1111 urls.add(jt);
1112 jt = getTermsOfUseUrl(j);
1113 if (isNotBlank(jt))
1114 urls.add(jt);
1115 jt = getAttributionUrl(j);
1116 if (isNotBlank(jt))
1117 urls.add(jt);
1118 jt = getIcon(j);
1119 if (isNotBlank(jt)) {
1120 if (!jt.startsWith("data:image/"))
1121 urls.add(jt);
1122 else {
1123 try {
1124 new ImageProvider(jt).get();
1125 } catch (RuntimeException e) {
1126 myprintln("* Strange Icon: "+getDescription(j));
1127 }
1128 }
1129 }
1130 Pattern patternU = Pattern.compile("^https?://([^/]+?)(:\\d+)?(/.*)?");
1131 for (String u : urls) {
1132 if (!patternU.matcher(u).matches() || u.matches(".*[ \t]+$")) {
1133 myprintln("* Strange URL '"+u+"': "+getDescription(j));
1134 } else {
1135 try {
1136 URL jurl = new URL(u.replaceAll("\\{switch:[^\\}]*\\}", "x"));
1137 String domain = jurl.getHost();
1138 int port = jurl.getPort();
1139 if (!(domain.matches("^\\d+\\.\\d+\\.\\d+\\.\\d+$")) && !dv.isValid(domain))
1140 myprintln("* Strange Domain '"+domain+"': "+getDescription(j));
1141 else if (80 == port || 443 == port) {
1142 myprintln("* Useless port '"+port+"': "+getDescription(j));
1143 }
1144 } catch (MalformedURLException e) {
1145 myprintln("* Malformed URL '"+u+"': "+getDescription(j)+" => "+e.getMessage());
1146 }
1147 }
1148 }
1149
1150 if (josmMirrors.containsKey(url)) {
1151 continue;
1152 }
1153 if (isBlank(id)) {
1154 myprintln("* No JOSM-ID: "+getDescription(j));
1155 } else if (josmIds.containsKey(id)) {
1156 myprintln("* JOSM-ID "+id+" not unique: "+getDescription(j));
1157 } else {
1158 josmIds.put(id, j);
1159 }
1160 String d = getDate(j);
1161 if (isNotBlank(d)) {
1162 Pattern patternD = Pattern.compile("^(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?)(;(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?))?$");
1163 Matcher m = patternD.matcher(d);
1164 if (!m.matches()) {
1165 myprintln("* JOSM-Date '"+d+"' is strange: "+getDescription(j));
1166 } else {
1167 try {
1168 Date first = verifyDate(m.group(2), m.group(4), m.group(6));
1169 Date second = verifyDate(m.group(9), m.group(11), m.group(13));
1170 if (second.compareTo(first) < 0) {
1171 myprintln("* JOSM-Date '"+d+"' is strange (second earlier than first): "+getDescription(j));
1172 }
1173 } catch (Exception e) {
1174 myprintln("* JOSM-Date '"+d+"' is strange ("+e.getMessage()+"): "+getDescription(j));
1175 }
1176 }
1177 }
1178 if (isNotBlank(getAttributionUrl(j)) && isBlank(getAttributionText(j))) {
1179 myprintln("* Attribution link without text: "+getDescription(j));
1180 }
1181 if (isNotBlank(getLogoUrl(j)) && isBlank(getLogoImage(j))) {
1182 myprintln("* Logo link without image: "+getDescription(j));
1183 }
1184 if (isNotBlank(getTermsOfUseText(j)) && isBlank(getTermsOfUseUrl(j))) {
1185 myprintln("* Terms of Use text without link: "+getDescription(j));
1186 }
1187 List<Shape> js = getShapes(j);
1188 if (!js.isEmpty()) {
1189 double minlat = 1000;
1190 double minlon = 1000;
1191 double maxlat = -1000;
1192 double maxlon = -1000;
1193 for (Shape s: js) {
1194 for (Coordinate p: s.getPoints()) {
1195 double lat = p.getLat();
1196 double lon = p.getLon();
1197 if (lat > maxlat) maxlat = lat;
1198 if (lon > maxlon) maxlon = lon;
1199 if (lat < minlat) minlat = lat;
1200 if (lon < minlon) minlon = lon;
1201 }
1202 }
1203 ImageryBounds b = j.getBounds();
1204 if (b.getMinLat() != minlat || b.getMinLon() != minlon || b.getMaxLat() != maxlat || b.getMaxLon() != maxlon) {
1205 myprintln("* Bounds do not match shape (is "+b.getMinLat()+","+b.getMinLon()+","+b.getMaxLat()+","+b.getMaxLon()
1206 + ", calculated <bounds min-lat='"+minlat+"' min-lon='"+minlon+"' max-lat='"+maxlat+"' max-lon='"+maxlon+"'>): "
1207 + getDescription(j));
1208 }
1209 }
1210 List<String> knownCategories = Arrays.asList(
1211 "photo", "elevation", "map", "historicmap", "osmbasedmap", "historicphoto", "qa", "other");
1212 String cat = getCategory(j);
1213 if (isBlank(cat)) {
1214 myprintln("* No category: "+getDescription(j));
1215 } else if (!knownCategories.contains(cat)) {
1216 myprintln("* Strange category "+cat+": "+getDescription(j));
1217 }
1218 }
1219 }
1220
1221 /*
1222 * Utility functions that allow uniform access for both ImageryInfo and JsonObject.
1223 */
1224
1225 static String getUrl(Object e) {
1226 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getUrl();
1227 return ((Map<String, JsonObject>) e).get("properties").getString("url");
1228 }
1229
1230 static String getUrlStripped(Object e) {
1231 return getUrl(e).replaceAll("\\?(apikey|access_token)=.*", "");
1232 }
1233
1234 static String getDate(Object e) {
1235 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getDate() != null ? ((ImageryInfo) e).getDate() : "";
1236 JsonObject p = ((Map<String, JsonObject>) e).get("properties");
1237 String start = p.containsKey("start_date") ? p.getString("start_date") : "";
1238 String end = p.containsKey("end_date") ? p.getString("end_date") : "";
1239 if (!start.isEmpty() && !end.isEmpty())
1240 return start+";"+end;
1241 else if (!start.isEmpty())
1242 return start+";-";
1243 else if (!end.isEmpty())
1244 return "-;"+end;
1245 return "";
1246 }
1247
1248 static Date verifyDate(String year, String month, String day) throws ParseException {
1249 String date;
1250 if (year == null) {
1251 date = "3000-01-01";
1252 } else {
1253 date = year + "-" + (month == null ? "01" : month) + "-" + (day == null ? "01" : day);
1254 }
1255 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");
1256 df.setLenient(false);
1257 return df.parse(date);
1258 }
1259
1260 static String getId(Object e) {
1261 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getId();
1262 return ((Map<String, JsonObject>) e).get("properties").getString("id");
1263 }
1264
1265 static String getName(Object e) {
1266 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getOriginalName();
1267 return ((Map<String, JsonObject>) e).get("properties").getString("name");
1268 }
1269
1270 static List<ImageryInfo> getMirrors(Object e) {
1271 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getMirrors();
1272 return Collections.emptyList();
1273 }
1274
1275 static List<String> getProjections(Object e) {
1276 List<String> r = new ArrayList<>();
1277 List<String> u = getProjectionsUnstripped(e);
1278 if (u != null) {
1279 for (String p : u) {
1280 if (!oldproj.containsKey(p) && !("CRS:84".equals(p) && !(getUrlStripped(e).matches("(?i)version=1\\.3")))) {
1281 r.add(p);
1282 }
1283 }
1284 }
1285 return r;
1286 }
1287
1288 static List<String> getProjectionsUnstripped(Object e) {
1289 List<String> r = null;
1290 if (e instanceof ImageryInfo) {
1291 r = ((ImageryInfo) e).getServerProjections();
1292 } else {
1293 JsonValue s = ((Map<String, JsonObject>) e).get("properties").get("available_projections");
1294 if (s != null) {
1295 r = new ArrayList<>();
1296 for (JsonValue p : s.asJsonArray()) {
1297 r.add(((JsonString) p).getString());
1298 }
1299 }
1300 }
1301 return r != null ? r : Collections.emptyList();
1302 }
1303
1304 static List<Shape> getShapes(Object e) {
1305 if (e instanceof ImageryInfo) {
1306 ImageryBounds bounds = ((ImageryInfo) e).getBounds();
1307 if (bounds != null) {
1308 return bounds.getShapes();
1309 }
1310 return Collections.emptyList();
1311 }
1312 JsonValue ex = ((Map<String, JsonValue>) e).get("geometry");
1313 if (ex != null && !JsonValue.NULL.equals(ex) && !ex.asJsonObject().isNull("coordinates")) {
1314 JsonArray poly = ex.asJsonObject().getJsonArray("coordinates");
1315 List<Shape> l = new ArrayList<>();
1316 for (JsonValue shapes: poly) {
1317 Shape s = new Shape();
1318 for (JsonValue point: shapes.asJsonArray()) {
1319 String lon = point.asJsonArray().getJsonNumber(0).toString();
1320 String lat = point.asJsonArray().getJsonNumber(1).toString();
1321 s.addPoint(lat, lon);
1322 }
1323 l.add(s);
1324 }
1325 return l;
1326 }
1327 return Collections.emptyList();
1328 }
1329
1330 static String getType(Object e) {
1331 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getImageryType().getTypeString();
1332 return ((Map<String, JsonObject>) e).get("properties").getString("type");
1333 }
1334
1335 static Integer getMinZoom(Object e) {
1336 if (e instanceof ImageryInfo) {
1337 int mz = ((ImageryInfo) e).getMinZoom();
1338 return mz == 0 ? null : mz;
1339 } else {
1340 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("min_zoom");
1341 if (num == null) return null;
1342 return num.intValue();
1343 }
1344 }
1345
1346 static Integer getMaxZoom(Object e) {
1347 if (e instanceof ImageryInfo) {
1348 int mz = ((ImageryInfo) e).getMaxZoom();
1349 return mz == 0 ? null : mz;
1350 } else {
1351 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("max_zoom");
1352 if (num == null) return null;
1353 return num.intValue();
1354 }
1355 }
1356
1357 static String getCountryCode(Object e) {
1358 if (e instanceof ImageryInfo) return "".equals(((ImageryInfo) e).getCountryCode()) ? null : ((ImageryInfo) e).getCountryCode();
1359 return ((Map<String, JsonObject>) e).get("properties").getString("country_code", null);
1360 }
1361
1362 static String getQuality(Object e) {
1363 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isBestMarked() ? "eli-best" : null;
1364 return (((Map<String, JsonObject>) e).get("properties").containsKey("best")
1365 && ((Map<String, JsonObject>) e).get("properties").getBoolean("best")) ? "eli-best" : null;
1366 }
1367
1368 static boolean getOverlay(Object e) {
1369 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isOverlay();
1370 return (((Map<String, JsonObject>) e).get("properties").containsKey("overlay")
1371 && ((Map<String, JsonObject>) e).get("properties").getBoolean("overlay"));
1372 }
1373
1374 static String getIcon(Object e) {
1375 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getIcon();
1376 return ((Map<String, JsonObject>) e).get("properties").getString("icon", null);
1377 }
1378
1379 static String getAttributionText(Object e) {
1380 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionText(0, null, null);
1381 try {
1382 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("text", null);
1383 } catch (NullPointerException ex) {
1384 return null;
1385 }
1386 }
1387
1388 static String getAttributionUrl(Object e) {
1389 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionLinkURL();
1390 try {
1391 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("url", null);
1392 } catch (NullPointerException ex) {
1393 return null;
1394 }
1395 }
1396
1397 static String getTermsOfUseText(Object e) {
1398 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseText();
1399 return null;
1400 }
1401
1402 static String getTermsOfUseUrl(Object e) {
1403 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseURL();
1404 return null;
1405 }
1406
1407 static String getCategory(Object e) {
1408 if (e instanceof ImageryInfo) {
1409 return ((ImageryInfo) e).getImageryCategoryOriginalString();
1410 }
1411 return ((Map<String, JsonObject>) e).get("properties").getString("category", null);
1412 }
1413
1414 static String getLogoImage(Object e) {
1415 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageRaw();
1416 return null;
1417 }
1418
1419 static String getLogoUrl(Object e) {
1420 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageURL();
1421 return null;
1422 }
1423
1424 static String getPermissionReferenceUrl(Object e) {
1425 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getPermissionReferenceURL();
1426 return ((Map<String, JsonObject>) e).get("properties").getString("license_url", null);
1427 }
1428
1429 static Map<String, Set<String>> getNoTileHeader(Object e) {
1430 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getNoTileHeaders();
1431 JsonObject nth = ((Map<String, JsonObject>) e).get("properties").getJsonObject("no_tile_header");
1432 return nth == null ? null : nth.keySet().stream().collect(Collectors.toMap(
1433 Function.identity(),
1434 k -> nth.getJsonArray(k).stream().map(x -> ((JsonString) x).getString()).collect(Collectors.toSet())));
1435 }
1436
1437 static Map<String, String> getDescriptions(Object e) {
1438 Map<String, String> res = new HashMap<>();
1439 if (e instanceof ImageryInfo) {
1440 String a = ((ImageryInfo) e).getDescription();
1441 if (a != null) res.put("en", a);
1442 } else {
1443 String a = ((Map<String, JsonObject>) e).get("properties").getString("description", null);
1444 if (a != null) res.put("en", a.replaceAll("''", "'"));
1445 }
1446 return res;
1447 }
1448
1449 static boolean getValidGeoreference(Object e) {
1450 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isGeoreferenceValid();
1451 return false;
1452 }
1453
1454 static boolean getDefault(Object e) {
1455 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isDefaultEntry();
1456 return ((Map<String, JsonObject>) e).get("properties").getBoolean("default", false);
1457 }
1458
1459 String getDescription(Object o) {
1460 String url = getUrl(o);
1461 String cc = getCountryCode(o);
1462 if (cc == null) {
1463 ImageryInfo j = josmUrls.get(url);
1464 if (j != null) cc = getCountryCode(j);
1465 if (cc == null) {
1466 JsonObject e = eliUrls.get(url);
1467 if (e != null) cc = getCountryCode(e);
1468 }
1469 }
1470 if (cc == null) {
1471 cc = "";
1472 } else {
1473 cc = "["+cc+"] ";
1474 }
1475 String d = cc + getName(o) + " - " + getUrl(o);
1476 if (optionShorten) {
1477 if (d.length() > MAXLEN) d = d.substring(0, MAXLEN-1) + "...";
1478 }
1479 return d;
1480 }
1481}
Note: See TracBrowser for help on using the repository browser.