source: josm/trunk/scripts/SyncEditorLayerIndex.java@ 16090

Last change on this file since 16090 was 15879, checked in by Don-vip, 5 years ago

checkstyle
  • Property svn:eol-style set to native
File size: 65.3 KB
Line 
1// License: GPL. For details, see LICENSE file.
2import static java.nio.charset.StandardCharsets.UTF_8;
3import static org.apache.commons.lang3.StringUtils.isBlank;
4import static org.apache.commons.lang3.StringUtils.isNotBlank;
5
6import java.io.BufferedReader;
7import java.io.IOException;
8import java.io.InputStreamReader;
9import java.io.OutputStream;
10import java.io.OutputStreamWriter;
11import java.lang.reflect.Field;
12import java.net.MalformedURLException;
13import java.net.URL;
14import java.nio.file.Files;
15import java.nio.file.Paths;
16import java.text.DecimalFormat;
17import java.text.ParseException;
18import java.text.SimpleDateFormat;
19import java.util.ArrayList;
20import java.util.Arrays;
21import java.util.Calendar;
22import java.util.Collection;
23import java.util.Collections;
24import java.util.Date;
25import java.util.HashMap;
26import java.util.LinkedList;
27import java.util.List;
28import java.util.Locale;
29import java.util.Map;
30import java.util.Map.Entry;
31import java.util.Objects;
32import java.util.Set;
33import java.util.function.BiConsumer;
34import java.util.function.Function;
35import java.util.regex.Matcher;
36import java.util.regex.Pattern;
37import java.util.stream.Collectors;
38
39import javax.json.Json;
40import javax.json.JsonArray;
41import javax.json.JsonNumber;
42import javax.json.JsonObject;
43import javax.json.JsonReader;
44import javax.json.JsonString;
45import javax.json.JsonValue;
46
47import org.openstreetmap.gui.jmapviewer.Coordinate;
48import org.openstreetmap.josm.data.Preferences;
49import org.openstreetmap.josm.data.imagery.ImageryInfo;
50import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
51import org.openstreetmap.josm.data.imagery.Shape;
52import org.openstreetmap.josm.data.preferences.JosmBaseDirectories;
53import org.openstreetmap.josm.data.preferences.JosmUrls;
54import org.openstreetmap.josm.data.projection.Projections;
55import org.openstreetmap.josm.data.validation.routines.DomainValidator;
56import org.openstreetmap.josm.io.imagery.ImageryReader;
57import org.openstreetmap.josm.spi.preferences.Config;
58import org.openstreetmap.josm.tools.ImageProvider;
59import org.openstreetmap.josm.tools.JosmRuntimeException;
60import org.openstreetmap.josm.tools.Logging;
61import org.openstreetmap.josm.tools.OptionParser;
62import org.openstreetmap.josm.tools.OptionParser.OptionCount;
63import org.openstreetmap.josm.tools.ReflectionUtils;
64import org.xml.sax.SAXException;
65
66/**
67 * Compare and analyse the differences of the editor layer index and the JOSM imagery list.
68 * The goal is to keep both lists in sync.
69 *
70 * The editor layer index project (https://github.com/osmlab/editor-layer-index)
71 * provides also a version in the JOSM format, but the GEOJSON is the original source
72 * format, so we read that.
73 *
74 * How to run:
75 * -----------
76 *
77 * Main JOSM binary needs to be in classpath, e.g.
78 *
79 * $ java -cp ../dist/josm-custom.jar SyncEditorLayerIndex
80 *
81 * Add option "-h" to show the available command line flags.
82 */
83@SuppressWarnings("unchecked")
84public class SyncEditorLayerIndex {
85
86 private static final int MAXLEN = 140;
87
88 private List<ImageryInfo> josmEntries;
89 private JsonArray eliEntries;
90
91 private final Map<String, JsonObject> eliUrls = new HashMap<>();
92 private final Map<String, ImageryInfo> josmUrls = new HashMap<>();
93 private final Map<String, ImageryInfo> josmMirrors = new HashMap<>();
94 private static final Map<String, String> oldproj = new HashMap<>();
95 private static final List<String> ignoreproj = new LinkedList<>();
96
97 private static String eliInputFile = "imagery_eli.geojson";
98 private static String josmInputFile = "imagery_josm.imagery.xml";
99 private static String ignoreInputFile = "imagery_josm.ignores.txt";
100 private static OutputStream outputFile;
101 private static OutputStreamWriter outputStream;
102 private static String optionOutput;
103 private static boolean optionShorten;
104 private static boolean optionNoSkip;
105 private static boolean optionXhtmlBody;
106 private static boolean optionXhtml;
107 private static String optionEliXml;
108 private static String optionJosmXml;
109 private static String optionEncoding;
110 private static boolean optionNoEli;
111 private Map<String, String> skip = new HashMap<>();
112 private Map<String, String> skipStart = new HashMap<>();
113
114 /**
115 * Main method.
116 * @param args program arguments
117 * @throws IOException if any I/O error occurs
118 * @throws ReflectiveOperationException if any reflective operation error occurs
119 * @throws SAXException if any SAX error occurs
120 */
121 public static void main(String[] args) throws IOException, SAXException, ReflectiveOperationException {
122 Locale.setDefault(Locale.ROOT);
123 parseCommandLineArguments(args);
124 Config.setUrlsProvider(JosmUrls.getInstance());
125 Preferences pref = new Preferences(JosmBaseDirectories.getInstance());
126 Config.setPreferencesInstance(pref);
127 pref.init(false);
128 SyncEditorLayerIndex script = new SyncEditorLayerIndex();
129 script.setupProj();
130 script.loadSkip();
131 script.start();
132 script.loadJosmEntries();
133 if (optionJosmXml != null) {
134 try (OutputStreamWriter stream = new OutputStreamWriter(Files.newOutputStream(Paths.get(optionJosmXml)), UTF_8)) {
135 script.printentries(script.josmEntries, stream);
136 }
137 }
138 script.loadELIEntries();
139 if (optionEliXml != null) {
140 try (OutputStreamWriter stream = new OutputStreamWriter(Files.newOutputStream(Paths.get(optionEliXml)), UTF_8)) {
141 script.printentries(script.eliEntries, stream);
142 }
143 }
144 script.checkInOneButNotTheOther();
145 script.checkCommonEntries();
146 script.end();
147 if (outputStream != null) {
148 outputStream.close();
149 }
150 if (outputFile != null) {
151 outputFile.close();
152 }
153 }
154
155 /**
156 * Displays help on the console
157 */
158 private static void showHelp() {
159 System.out.println(getHelp());
160 System.exit(0);
161 }
162
163 static String getHelp() {
164 return "usage: java -cp build SyncEditorLayerIndex\n" +
165 "-c,--encoding <encoding> output encoding (defaults to UTF-8 or cp850 on Windows)\n" +
166 "-e,--eli_input <eli_input> Input file for the editor layer index (geojson). " +
167 "Default is imagery_eli.geojson (current directory).\n" +
168 "-h,--help show this help\n" +
169 "-i,--ignore_input <ignore_input> Input file for the ignore list. Default is imagery_josm.ignores.txt (current directory).\n" +
170 "-j,--josm_input <josm_input> Input file for the JOSM imagery list (xml). " +
171 "Default is imagery_josm.imagery.xml (current directory).\n" +
172 "-m,--noeli don't show output for ELI problems\n" +
173 "-n,--noskip don't skip known entries\n" +
174 "-o,--output <output> Output file, - prints to stdout (default: -)\n" +
175 "-p,--elixml <elixml> ELI entries for use in JOSM as XML file (incomplete)\n" +
176 "-q,--josmxml <josmxml> JOSM entries reoutput as XML file (incomplete)\n" +
177 "-s,--shorten shorten the output, so it is easier to read in a console window\n" +
178 "-x,--xhtmlbody create XHTML body for display in a web page\n" +
179 "-X,--xhtml create XHTML for display in a web page\n";
180 }
181
182 /**
183 * Parse command line arguments.
184 * @param args program arguments
185 * @throws IOException in case of I/O error
186 */
187 static void parseCommandLineArguments(String[] args) throws IOException {
188 new OptionParser("JOSM/ELI synchronization script")
189 .addFlagParameter("help", SyncEditorLayerIndex::showHelp)
190 .addShortAlias("help", "h")
191 .addArgumentParameter("output", OptionCount.OPTIONAL, x -> optionOutput = x)
192 .addShortAlias("output", "o")
193 .addArgumentParameter("eli_input", OptionCount.OPTIONAL, x -> eliInputFile = x)
194 .addShortAlias("eli_input", "e")
195 .addArgumentParameter("josm_input", OptionCount.OPTIONAL, x -> josmInputFile = x)
196 .addShortAlias("josm_input", "j")
197 .addArgumentParameter("ignore_input", OptionCount.OPTIONAL, x -> ignoreInputFile = x)
198 .addShortAlias("ignore_input", "i")
199 .addFlagParameter("shorten", () -> optionShorten = true)
200 .addShortAlias("shorten", "s")
201 .addFlagParameter("noskip", () -> optionNoSkip = true)
202 .addShortAlias("noskip", "n")
203 .addFlagParameter("xhtmlbody", () -> optionXhtmlBody = true)
204 .addShortAlias("xhtmlbody", "x")
205 .addFlagParameter("xhtml", () -> optionXhtml = true)
206 .addShortAlias("xhtml", "X")
207 .addArgumentParameter("elixml", OptionCount.OPTIONAL, x -> optionEliXml = x)
208 .addShortAlias("elixml", "p")
209 .addArgumentParameter("josmxml", OptionCount.OPTIONAL, x -> optionJosmXml = x)
210 .addShortAlias("josmxml", "q")
211 .addFlagParameter("noeli", () -> optionNoEli = true)
212 .addShortAlias("noeli", "m")
213 .addArgumentParameter("encoding", OptionCount.OPTIONAL, x -> optionEncoding = x)
214 .addShortAlias("encoding", "c")
215 .parseOptionsOrExit(Arrays.asList(args));
216
217 if (optionOutput != null && !"-".equals(optionOutput)) {
218 outputFile = Files.newOutputStream(Paths.get(optionOutput));
219 outputStream = new OutputStreamWriter(outputFile, optionEncoding != null ? optionEncoding : "UTF-8");
220 } else if (optionEncoding != null) {
221 outputStream = new OutputStreamWriter(System.out, optionEncoding);
222 }
223 }
224
225 void setupProj() {
226 oldproj.put("EPSG:3359", "EPSG:3404");
227 oldproj.put("EPSG:3785", "EPSG:3857");
228 oldproj.put("EPSG:31297", "EPGS:31287");
229 oldproj.put("EPSG:31464", "EPSG:31468");
230 oldproj.put("EPSG:54004", "EPSG:3857");
231 oldproj.put("EPSG:102100", "EPSG:3857");
232 oldproj.put("EPSG:102113", "EPSG:3857");
233 oldproj.put("EPSG:900913", "EPGS:3857");
234 ignoreproj.add("EPSG:4267");
235 ignoreproj.add("EPSG:5221");
236 ignoreproj.add("EPSG:5514");
237 ignoreproj.add("EPSG:32019");
238 ignoreproj.add("EPSG:102066");
239 ignoreproj.add("EPSG:102067");
240 ignoreproj.add("EPSG:102685");
241 ignoreproj.add("EPSG:102711");
242 }
243
244 void loadSkip() throws IOException {
245 final Pattern pattern = Pattern.compile("^\\|\\| *(ELI|Ignore) *\\|\\| *\\{\\{\\{(.+)\\}\\}\\} *\\|\\|");
246 try (BufferedReader fr = new BufferedReader(new InputStreamReader(Files.newInputStream(Paths.get(ignoreInputFile)), UTF_8))) {
247 String line;
248
249 while ((line = fr.readLine()) != null) {
250 Matcher res = pattern.matcher(line);
251 if (res.matches()) {
252 String s = res.group(2);
253 if (s.endsWith("...")) {
254 s = s.substring(0, s.length() - 3);
255 if ("Ignore".equals(res.group(1))) {
256 skipStart.put(s, "green");
257 } else {
258 skipStart.put(s, "darkgoldenrod");
259 }
260 } else {
261 if ("Ignore".equals(res.group(1))) {
262 skip.put(s, "green");
263 } else {
264 skip.put(s, "darkgoldenrod");
265 }
266 }
267 }
268 }
269 }
270 }
271
272 void myprintlnfinal(String s) {
273 if (outputStream != null) {
274 try {
275 outputStream.write(s + System.getProperty("line.separator"));
276 } catch (IOException e) {
277 throw new JosmRuntimeException(e);
278 }
279 } else {
280 System.out.println(s);
281 }
282 }
283
284 String isSkipString(String s) {
285 if (skip.containsKey(s))
286 return skip.get(s);
287 for (Entry<String, String> str : skipStart.entrySet()) {
288 if (s.startsWith(str.getKey())) {
289 skipStart.remove(str.getKey());
290 return str.getValue();
291 }
292 }
293 return null;
294 }
295
296 void myprintln(String s) {
297 String color;
298 if ((color = isSkipString(s)) != null) {
299 skip.remove(s);
300 if (optionXhtmlBody || optionXhtml) {
301 s = "<pre style=\"margin:3px;color:"+color+"\">"
302 + s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
303 }
304 if (!optionNoSkip) {
305 return;
306 }
307 } else if (optionXhtmlBody || optionXhtml) {
308 color =
309 s.startsWith("***") ? "black" :
310 ((s.startsWith("+ ") || s.startsWith("+++ ELI")) ? "blue" :
311 (s.startsWith("#") ? "indigo" :
312 (s.startsWith("!") ? "orange" : "red")));
313 s = "<pre style=\"margin:3px;color:"+color+"\">"+s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
314 }
315 if ((s.startsWith("+ ") || s.startsWith("+++ ELI") || s.startsWith("#")) && optionNoEli) {
316 return;
317 }
318 myprintlnfinal(s);
319 }
320
321 void start() {
322 if (optionXhtml) {
323 myprintlnfinal(
324 "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n");
325 myprintlnfinal(
326 "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>"+
327 "<title>JOSM - ELI differences</title></head><body>\n");
328 }
329 }
330
331 void end() {
332 for (String s : skip.keySet()) {
333 myprintln("+++ Obsolete skip entry: " + s);
334 }
335 for (String s : skipStart.keySet()) {
336 myprintln("+++ Obsolete skip entry: " + s + "...");
337 }
338 if (optionXhtml) {
339 myprintlnfinal("</body></html>\n");
340 }
341 }
342
343 void loadELIEntries() throws IOException {
344 try (JsonReader jr = Json.createReader(new InputStreamReader(Files.newInputStream(Paths.get(eliInputFile)), UTF_8))) {
345 eliEntries = jr.readObject().getJsonArray("features");
346 }
347
348 for (JsonValue e : eliEntries) {
349 String url = getUrlStripped(e);
350 if (url.contains("{z}")) {
351 myprintln("+++ ELI-URL uses {z} instead of {zoom}: "+getDescription(e));
352 url = url.replace("{z}", "{zoom}");
353 }
354 if (eliUrls.containsKey(url)) {
355 myprintln("+++ ELI-URL is not unique: "+url);
356 } else {
357 eliUrls.put(url, e.asJsonObject());
358 }
359 JsonArray s = e.asJsonObject().get("properties").asJsonObject().getJsonArray("available_projections");
360 if (s != null) {
361 String urlLc = url.toLowerCase(Locale.ENGLISH);
362 List<String> old = new LinkedList<>();
363 for (JsonValue p : s) {
364 String proj = ((JsonString) p).getString();
365 if (oldproj.containsKey(proj) || ("CRS:84".equals(proj) && !urlLc.contains("version=1.3"))) {
366 old.add(proj);
367 }
368 }
369 if (!old.isEmpty()) {
370 myprintln("+ ELI Projections "+String.join(", ", old)+" not useful: "+getDescription(e));
371 }
372 }
373 }
374 myprintln("*** Loaded "+eliEntries.size()+" entries (ELI). ***");
375 }
376
377 String cdata(String s) {
378 return cdata(s, false);
379 }
380
381 String cdata(String s, boolean escape) {
382 if (escape) {
383 return s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
384 } else if (s.matches(".*[<>&].*"))
385 return "<![CDATA["+s+"]]>";
386 return s;
387 }
388
389 String maininfo(Object entry, String offset) {
390 String t = getType(entry);
391 String res = offset + "<type>"+t+"</type>\n";
392 res += offset + "<url>"+cdata(getUrl(entry))+"</url>\n";
393 if (getMinZoom(entry) != null)
394 res += offset + "<min-zoom>"+getMinZoom(entry)+"</min-zoom>\n";
395 if (getMaxZoom(entry) != null)
396 res += offset + "<max-zoom>"+getMaxZoom(entry)+"</max-zoom>\n";
397 if ("wms".equals(t)) {
398 List<String> p = getProjections(entry);
399 if (p != null) {
400 res += offset + "<projections>\n";
401 for (String c : p) {
402 res += offset + " <code>"+c+"</code>\n";
403 }
404 res += offset + "</projections>\n";
405 }
406 }
407 return res;
408 }
409
410 void printentries(List<?> entries, OutputStreamWriter stream) throws IOException {
411 DecimalFormat df = new DecimalFormat("#.#######");
412 df.setRoundingMode(java.math.RoundingMode.CEILING);
413 stream.write("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n");
414 stream.write("<imagery xmlns=\"http://josm.openstreetmap.de/maps-1.0\">\n");
415 for (Object e : entries) {
416 stream.write(" <entry"
417 + ("eli-best".equals(getQuality(e)) ? " eli-best=\"true\"" : "")
418 + (getOverlay(e) ? " overlay=\"true\"" : "")
419 + ">\n");
420 String t;
421 if (isNotBlank(t = getName(e)))
422 stream.write(" <name>"+cdata(t, true)+"</name>\n");
423 if (isNotBlank(t = getId(e)))
424 stream.write(" <id>"+t+"</id>\n");
425 if (isNotBlank(t = getCategory(e)))
426 stream.write(" <category>"+t+"</category>\n");
427 if (isNotBlank(t = getDate(e)))
428 stream.write(" <date>"+t+"</date>\n");
429 if (isNotBlank(t = getCountryCode(e)))
430 stream.write(" <country-code>"+t+"</country-code>\n");
431 if ((getDefault(e)))
432 stream.write(" <default>true</default>\n");
433 stream.write(maininfo(e, " "));
434 if (isNotBlank(t = getAttributionText(e)))
435 stream.write(" <attribution-text mandatory=\"true\">"+cdata(t, true)+"</attribution-text>\n");
436 if (isNotBlank(t = getAttributionUrl(e)))
437 stream.write(" <attribution-url>"+cdata(t)+"</attribution-url>\n");
438 if (isNotBlank(t = getLogoImage(e)))
439 stream.write(" <logo-image>"+cdata(t, true)+"</logo-image>\n");
440 if (isNotBlank(t = getLogoUrl(e)))
441 stream.write(" <logo-url>"+cdata(t)+"</logo-url>\n");
442 if (isNotBlank(t = getTermsOfUseText(e)))
443 stream.write(" <terms-of-use-text>"+cdata(t, true)+"</terms-of-use-text>\n");
444 if (isNotBlank(t = getTermsOfUseUrl(e)))
445 stream.write(" <terms-of-use-url>"+cdata(t)+"</terms-of-use-url>\n");
446 if (isNotBlank(t = getPermissionReferenceUrl(e)))
447 stream.write(" <permission-ref>"+cdata(t)+"</permission-ref>\n");
448 if ((getValidGeoreference(e)))
449 stream.write(" <valid-georeference>true</valid-georeference>\n");
450 if (isNotBlank(t = getIcon(e)))
451 stream.write(" <icon>"+cdata(t)+"</icon>\n");
452 for (Entry<String, String> d : getDescriptions(e).entrySet()) {
453 stream.write(" <description lang=\""+d.getKey()+"\">"+d.getValue()+"</description>\n");
454 }
455 for (ImageryInfo m : getMirrors(e)) {
456 stream.write(" <mirror>\n"+maininfo(m, " ")+" </mirror>\n");
457 }
458 double minlat = 1000;
459 double minlon = 1000;
460 double maxlat = -1000;
461 double maxlon = -1000;
462 String shapes = "";
463 String sep = "\n ";
464 try {
465 for (Shape s: getShapes(e)) {
466 shapes += " <shape>";
467 int i = 0;
468 for (Coordinate p: s.getPoints()) {
469 double lat = p.getLat();
470 double lon = p.getLon();
471 if (lat > maxlat) maxlat = lat;
472 if (lon > maxlon) maxlon = lon;
473 if (lat < minlat) minlat = lat;
474 if (lon < minlon) minlon = lon;
475 if ((i++ % 3) == 0) {
476 shapes += sep + " ";
477 }
478 shapes += "<point lat='"+df.format(lat)+"' lon='"+df.format(lon)+"'/>";
479 }
480 shapes += sep + "</shape>\n";
481 }
482 } catch (IllegalArgumentException ignored) {
483 Logging.trace(ignored);
484 }
485 if (!shapes.isEmpty()) {
486 stream.write(" <bounds min-lat='"+df.format(minlat)
487 +"' min-lon='"+df.format(minlon)
488 +"' max-lat='"+df.format(maxlat)
489 +"' max-lon='"+df.format(maxlon)+"'>\n");
490 stream.write(shapes + " </bounds>\n");
491 }
492 stream.write(" </entry>\n");
493 }
494 stream.write("</imagery>\n");
495 stream.close();
496 }
497
498 void loadJosmEntries() throws IOException, SAXException, ReflectiveOperationException {
499 try (ImageryReader reader = new ImageryReader(josmInputFile)) {
500 josmEntries = reader.parse();
501 }
502
503 for (ImageryInfo e : josmEntries) {
504 if (isBlank(getUrl(e))) {
505 myprintln("+++ JOSM-Entry without URL: " + getDescription(e));
506 continue;
507 }
508 if (isBlank(getName(e))) {
509 myprintln("+++ JOSM-Entry without Name: " + getDescription(e));
510 continue;
511 }
512 String url = getUrlStripped(e);
513 if (url.contains("{z}")) {
514 myprintln("+++ JOSM-URL uses {z} instead of {zoom}: "+getDescription(e));
515 url = url.replace("{z}", "{zoom}");
516 }
517 if (josmUrls.containsKey(url)) {
518 myprintln("+++ JOSM-URL is not unique: "+url);
519 } else {
520 josmUrls.put(url, e);
521 }
522 for (ImageryInfo m : e.getMirrors()) {
523 url = getUrlStripped(m);
524 Field origNameField = ImageryInfo.class.getDeclaredField("origName");
525 ReflectionUtils.setObjectsAccessible(origNameField);
526 origNameField.set(m, m.getOriginalName().replaceAll(" mirror server( \\d+)?", ""));
527 if (josmUrls.containsKey(url)) {
528 myprintln("+++ JOSM-Mirror-URL is not unique: "+url);
529 } else {
530 josmUrls.put(url, m);
531 josmMirrors.put(url, m);
532 }
533 }
534 }
535 myprintln("*** Loaded "+josmEntries.size()+" entries (JOSM). ***");
536 }
537
538 void checkInOneButNotTheOther() {
539 List<String> le = new LinkedList<>(eliUrls.keySet());
540 List<String> lj = new LinkedList<>(josmUrls.keySet());
541
542 List<String> ke = new LinkedList<>(le);
543 for (String url : ke) {
544 if (lj.contains(url)) {
545 le.remove(url);
546 lj.remove(url);
547 }
548 }
549
550 if (!le.isEmpty() && !lj.isEmpty()) {
551 ke = new LinkedList<>(le);
552 for (String urle : ke) {
553 JsonObject e = eliUrls.get(urle);
554 String ide = getId(e);
555 String urlhttps = urle.replace("http:", "https:");
556 if (lj.contains(urlhttps)) {
557 myprintln("+ Missing https: "+getDescription(e));
558 eliUrls.put(urlhttps, eliUrls.get(urle));
559 eliUrls.remove(urle);
560 le.remove(urle);
561 lj.remove(urlhttps);
562 } else if (isNotBlank(ide)) {
563 List<String> kj = new LinkedList<>(lj);
564 for (String urlj : kj) {
565 ImageryInfo j = josmUrls.get(urlj);
566 String idj = getId(j);
567
568 if (ide.equals(idj) && Objects.equals(getType(j), getType(e))) {
569 myprintln("* URL for id "+idj+" differs ("+urle+"): "+getDescription(j));
570 le.remove(urle);
571 lj.remove(urlj);
572 // replace key for this entry with JOSM URL
573 eliUrls.remove(urle);
574 eliUrls.put(urlj, e);
575 break;
576 }
577 }
578 }
579 }
580 }
581
582 myprintln("*** URLs found in ELI but not in JOSM ("+le.size()+"): ***");
583 Collections.sort(le);
584 if (!le.isEmpty()) {
585 for (String l : le) {
586 myprintln("- " + getDescription(eliUrls.get(l)));
587 }
588 }
589 myprintln("*** URLs found in JOSM but not in ELI ("+lj.size()+"): ***");
590 Collections.sort(lj);
591 if (!lj.isEmpty()) {
592 for (String l : lj) {
593 myprintln("+ " + getDescription(josmUrls.get(l)));
594 }
595 }
596 }
597
598 void checkCommonEntries() {
599 doSameUrlButDifferentName();
600 doSameUrlButDifferentId();
601 doSameUrlButDifferentType();
602 doSameUrlButDifferentZoomBounds();
603 doSameUrlButDifferentCountryCode();
604 doSameUrlButDifferentQuality();
605 doSameUrlButDifferentDates();
606 doSameUrlButDifferentInformation();
607 doMismatchingShapes();
608 doMismatchingIcons();
609 doMismatchingCategories();
610 doMiscellaneousChecks();
611 }
612
613 void doSameUrlButDifferentName() {
614 myprintln("*** Same URL, but different name: ***");
615 for (String url : eliUrls.keySet()) {
616 JsonObject e = eliUrls.get(url);
617 if (!josmUrls.containsKey(url)) continue;
618 ImageryInfo j = josmUrls.get(url);
619 String ename = getName(e).replace("'", "\u2019");
620 String jname = getName(j).replace("'", "\u2019");
621 if (!ename.equals(jname)) {
622 myprintln("* Name differs ('"+getName(e)+"' != '"+getName(j)+"'): "+getUrl(j));
623 }
624 }
625 }
626
627 void doSameUrlButDifferentId() {
628 myprintln("*** Same URL, but different Id: ***");
629 for (String url : eliUrls.keySet()) {
630 JsonObject e = eliUrls.get(url);
631 if (!josmUrls.containsKey(url)) continue;
632 ImageryInfo j = josmUrls.get(url);
633 String ename = getId(e);
634 String jname = getId(j);
635 if (!Objects.equals(ename, jname)) {
636 myprintln("# Id differs ('"+getId(e)+"' != '"+getId(j)+"'): "+getUrl(j));
637 }
638 }
639 }
640
641 void doSameUrlButDifferentType() {
642 myprintln("*** Same URL, but different type: ***");
643 for (String url : eliUrls.keySet()) {
644 JsonObject e = eliUrls.get(url);
645 if (!josmUrls.containsKey(url)) continue;
646 ImageryInfo j = josmUrls.get(url);
647 if (!Objects.equals(getType(e), getType(j))) {
648 myprintln("* Type differs ("+getType(e)+" != "+getType(j)+"): "+getName(j)+" - "+getUrl(j));
649 }
650 }
651 }
652
653 void doSameUrlButDifferentZoomBounds() {
654 myprintln("*** Same URL, but different zoom bounds: ***");
655 for (String url : eliUrls.keySet()) {
656 JsonObject e = eliUrls.get(url);
657 if (!josmUrls.containsKey(url)) continue;
658 ImageryInfo j = josmUrls.get(url);
659
660 Integer eMinZoom = getMinZoom(e);
661 Integer jMinZoom = getMinZoom(j);
662 /* dont warn for entries copied from the base of the mirror */
663 if (eMinZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
664 jMinZoom = null;
665 if (!Objects.equals(eMinZoom, jMinZoom) && !(Objects.equals(eMinZoom, 0) && jMinZoom == null)) {
666 myprintln("* Minzoom differs ("+eMinZoom+" != "+jMinZoom+"): "+getDescription(j));
667 }
668 Integer eMaxZoom = getMaxZoom(e);
669 Integer jMaxZoom = getMaxZoom(j);
670 /* dont warn for entries copied from the base of the mirror */
671 if (eMaxZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
672 jMaxZoom = null;
673 if (!Objects.equals(eMaxZoom, jMaxZoom)) {
674 myprintln("* Maxzoom differs ("+eMaxZoom+" != "+jMaxZoom+"): "+getDescription(j));
675 }
676 }
677 }
678
679 void doSameUrlButDifferentCountryCode() {
680 myprintln("*** Same URL, but different country code: ***");
681 for (String url : eliUrls.keySet()) {
682 JsonObject e = eliUrls.get(url);
683 if (!josmUrls.containsKey(url)) continue;
684 ImageryInfo j = josmUrls.get(url);
685 String cce = getCountryCode(e);
686 if ("ZZ".equals(cce)) { /* special ELI country code */
687 cce = null;
688 }
689 if (cce != null && !cce.equals(getCountryCode(j))) {
690 myprintln("* Country code differs ("+getCountryCode(e)+" != "+getCountryCode(j)+"): "+getDescription(j));
691 }
692 }
693 }
694
695 void doSameUrlButDifferentQuality() {
696 myprintln("*** Same URL, but different quality: ***");
697 for (String url : eliUrls.keySet()) {
698 JsonObject e = eliUrls.get(url);
699 if (!josmUrls.containsKey(url)) {
700 String q = getQuality(e);
701 if ("eli-best".equals(q)) {
702 myprintln("- Quality best entry not in JOSM for "+getDescription(e));
703 }
704 continue;
705 }
706 ImageryInfo j = josmUrls.get(url);
707 if (!Objects.equals(getQuality(e), getQuality(j))) {
708 myprintln("* Quality differs ("+getQuality(e)+" != "+getQuality(j)+"): "+getDescription(j));
709 }
710 }
711 }
712
713 void doSameUrlButDifferentDates() {
714 myprintln("*** Same URL, but different dates: ***");
715 Pattern pattern = Pattern.compile("^(.*;)(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?$");
716 for (String url : eliUrls.keySet()) {
717 String ed = getDate(eliUrls.get(url));
718 if (!josmUrls.containsKey(url)) continue;
719 ImageryInfo j = josmUrls.get(url);
720 String jd = getDate(j);
721 // The forms 2015;- or -;2015 or 2015;2015 are handled equal to 2015
722 String ef = ed.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
723 // ELI has a strange and inconsistent used end_date definition, so we try again with subtraction by one
724 String ed2 = ed;
725 Matcher m = pattern.matcher(ed);
726 if (m.matches()) {
727 Calendar cal = Calendar.getInstance();
728 cal.set(Integer.valueOf(m.group(2)),
729 m.group(4) == null ? 0 : Integer.valueOf(m.group(4))-1,
730 m.group(6) == null ? 1 : Integer.valueOf(m.group(6)));
731 cal.add(Calendar.DAY_OF_MONTH, -1);
732 ed2 = m.group(1) + cal.get(Calendar.YEAR);
733 if (m.group(4) != null)
734 ed2 += "-" + String.format("%02d", cal.get(Calendar.MONTH)+1);
735 if (m.group(6) != null)
736 ed2 += "-" + String.format("%02d", cal.get(Calendar.DAY_OF_MONTH));
737 }
738 String ef2 = ed2.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
739 if (!ed.equals(jd) && !ef.equals(jd) && !ed2.equals(jd) && !ef2.equals(jd)) {
740 String t = "'"+ed+"'";
741 if (!ed.equals(ef)) {
742 t += " or '"+ef+"'";
743 }
744 if (jd.isEmpty()) {
745 myprintln("- Missing JOSM date ("+t+"): "+getDescription(j));
746 } else if (!ed.isEmpty()) {
747 myprintln("* Date differs ('"+t+"' != '"+jd+"'): "+getDescription(j));
748 } else if (!optionNoEli) {
749 myprintln("+ Missing ELI date ('"+jd+"'): "+getDescription(j));
750 }
751 }
752 }
753 }
754
755 void doSameUrlButDifferentInformation() {
756 myprintln("*** Same URL, but different information: ***");
757 for (String url : eliUrls.keySet()) {
758 if (!josmUrls.containsKey(url)) continue;
759 JsonObject e = eliUrls.get(url);
760 ImageryInfo j = josmUrls.get(url);
761
762 compareDescriptions(e, j);
763 comparePermissionReferenceUrls(e, j);
764 compareAttributionUrls(e, j);
765 compareAttributionTexts(e, j);
766 compareProjections(e, j);
767 compareDefaults(e, j);
768 compareOverlays(e, j);
769 compareNoTileHeaders(e, j);
770 }
771 }
772
773 void compareDescriptions(JsonObject e, ImageryInfo j) {
774 String et = getDescriptions(e).getOrDefault("en", "");
775 String jt = getDescriptions(j).getOrDefault("en", "");
776 if (!et.equals(jt)) {
777 if (jt.isEmpty()) {
778 myprintln("- Missing JOSM description ("+et+"): "+getDescription(j));
779 } else if (!et.isEmpty()) {
780 myprintln("* Description differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
781 } else if (!optionNoEli) {
782 myprintln("+ Missing ELI description ('"+jt+"'): "+getDescription(j));
783 }
784 }
785 }
786
787 void comparePermissionReferenceUrls(JsonObject e, ImageryInfo j) {
788 String et = getPermissionReferenceUrl(e);
789 String jt = getPermissionReferenceUrl(j);
790 String jt2 = getTermsOfUseUrl(j);
791 if (isBlank(jt)) jt = jt2;
792 if (!Objects.equals(et, jt)) {
793 if (isBlank(jt)) {
794 myprintln("- Missing JOSM license URL ("+et+"): "+getDescription(j));
795 } else if (isNotBlank(et)) {
796 String ethttps = et.replace("http:", "https:");
797 if (isBlank(jt2) || !(jt2.equals(ethttps) || jt2.equals(et+"/") || jt2.equals(ethttps+"/"))) {
798 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
799 myprintln("+ License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
800 } else {
801 String ja = getAttributionUrl(j);
802 if (ja != null && (ja.equals(et) || ja.equals(ethttps) || ja.equals(et+"/") || ja.equals(ethttps+"/"))) {
803 myprintln("+ ELI License URL in JOSM Attribution: "+getDescription(j));
804 } else {
805 myprintln("* License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
806 }
807 }
808 }
809 } else if (!optionNoEli) {
810 myprintln("+ Missing ELI license URL ('"+jt+"'): "+getDescription(j));
811 }
812 }
813 }
814
815 void compareAttributionUrls(JsonObject e, ImageryInfo j) {
816 String et = getAttributionUrl(e);
817 String jt = getAttributionUrl(j);
818 if (!Objects.equals(et, jt)) {
819 if (isBlank(jt)) {
820 myprintln("- Missing JOSM attribution URL ("+et+"): "+getDescription(j));
821 } else if (isNotBlank(et)) {
822 String ethttps = et.replace("http:", "https:");
823 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
824 myprintln("+ Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
825 } else {
826 myprintln("* Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
827 }
828 } else if (!optionNoEli) {
829 myprintln("+ Missing ELI attribution URL ('"+jt+"'): "+getDescription(j));
830 }
831 }
832 }
833
834 void compareAttributionTexts(JsonObject e, ImageryInfo j) {
835 String et = getAttributionText(e);
836 String jt = getAttributionText(j);
837 if (!Objects.equals(et, jt)) {
838 if (isBlank(jt)) {
839 myprintln("- Missing JOSM attribution text ("+et+"): "+getDescription(j));
840 } else if (isNotBlank(et)) {
841 myprintln("* Attribution text differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
842 } else if (!optionNoEli) {
843 myprintln("+ Missing ELI attribution text ('"+jt+"'): "+getDescription(j));
844 }
845 }
846 }
847
848 void compareProjections(JsonObject e, ImageryInfo j) {
849 String et = getProjections(e).stream().sorted().collect(Collectors.joining(" "));
850 String jt = getProjections(j).stream().sorted().collect(Collectors.joining(" "));
851 if (!Objects.equals(et, jt)) {
852 if (isBlank(jt)) {
853 String t = getType(e);
854 if ("wms_endpoint".equals(t) || "tms".equals(t)) {
855 myprintln("+ ELI projections for type "+t+": "+getDescription(j));
856 } else {
857 myprintln("- Missing JOSM projections ("+et+"): "+getDescription(j));
858 }
859 } else if (isNotBlank(et)) {
860 if ("EPSG:3857 EPSG:4326".equals(et) || "EPSG:3857".equals(et) || "EPSG:4326".equals(et)) {
861 myprintln("+ ELI has minimal projections ('"+et+"' != '"+jt+"'): "+getDescription(j));
862 } else {
863 myprintln("* Projections differ ('"+et+"' != '"+jt+"'): "+getDescription(j));
864 }
865 } else if (!optionNoEli && !"tms".equals(getType(e))) {
866 myprintln("+ Missing ELI projections ('"+jt+"'): "+getDescription(j));
867 }
868 }
869 }
870
871 void compareDefaults(JsonObject e, ImageryInfo j) {
872 boolean ed = getDefault(e);
873 boolean jd = getDefault(j);
874 if (ed != jd) {
875 if (!jd) {
876 myprintln("- Missing JOSM default: "+getDescription(j));
877 } else if (!optionNoEli) {
878 myprintln("+ Missing ELI default: "+getDescription(j));
879 }
880 }
881 }
882
883 void compareOverlays(JsonObject e, ImageryInfo j) {
884 boolean eo = getOverlay(e);
885 boolean jo = getOverlay(j);
886 if (eo != jo) {
887 if (!jo) {
888 myprintln("- Missing JOSM overlay flag: "+getDescription(j));
889 } else if (!optionNoEli) {
890 myprintln("+ Missing ELI overlay flag: "+getDescription(j));
891 }
892 }
893 }
894
895 void compareNoTileHeaders(JsonObject e, ImageryInfo j) {
896 Map<String, Set<String>> eh = getNoTileHeader(e);
897 Map<String, Set<String>> jh = getNoTileHeader(j);
898 if (!Objects.equals(eh, jh)) {
899 if (jh == null || jh.isEmpty()) {
900 myprintln("- Missing JOSM no tile headers ("+eh+"): "+getDescription(j));
901 } else if (eh != null && !eh.isEmpty()) {
902 myprintln("* No tile headers differ ('"+eh+"' != '"+jh+"'): "+getDescription(j));
903 } else if (!optionNoEli) {
904 myprintln("+ Missing ELI no tile headers ('"+jh+"'): "+getDescription(j));
905 }
906 }
907 }
908
909 void doMismatchingShapes() {
910 myprintln("*** Mismatching shapes: ***");
911 for (String url : josmUrls.keySet()) {
912 ImageryInfo j = josmUrls.get(url);
913 int num = 1;
914 for (Shape shape : getShapes(j)) {
915 List<Coordinate> p = shape.getPoints();
916 if (!p.get(0).equals(p.get(p.size()-1))) {
917 myprintln("+++ JOSM shape "+num+" unclosed: "+getDescription(j));
918 }
919 for (int nump = 1; nump < p.size(); ++nump) {
920 if (Objects.equals(p.get(nump-1), p.get(nump))) {
921 myprintln("+++ JOSM shape "+num+" double point at "+(nump-1)+": "+getDescription(j));
922 }
923 }
924 ++num;
925 }
926 }
927 for (String url : eliUrls.keySet()) {
928 JsonObject e = eliUrls.get(url);
929 int num = 1;
930 List<Shape> s = null;
931 try {
932 s = getShapes(e);
933 for (Shape shape : s) {
934 List<Coordinate> p = shape.getPoints();
935 if (!p.get(0).equals(p.get(p.size()-1)) && !optionNoEli) {
936 myprintln("+++ ELI shape "+num+" unclosed: "+getDescription(e));
937 }
938 for (int nump = 1; nump < p.size(); ++nump) {
939 if (Objects.equals(p.get(nump-1), p.get(nump))) {
940 myprintln("+++ ELI shape "+num+" double point at "+(nump-1)+": "+getDescription(e));
941 }
942 }
943 ++num;
944 }
945 } catch (IllegalArgumentException err) {
946 String desc = getDescription(e);
947 myprintln("* Invalid data in ELI geometry for "+desc+": "+err.getMessage());
948 }
949 if (s == null || !josmUrls.containsKey(url)) {
950 continue;
951 }
952 ImageryInfo j = josmUrls.get(url);
953 List<Shape> js = getShapes(j);
954 if (s.isEmpty() && !js.isEmpty()) {
955 if (!optionNoEli) {
956 myprintln("+ No ELI shape: "+getDescription(j));
957 }
958 } else if (js.isEmpty() && !s.isEmpty()) {
959 // don't report boundary like 5 point shapes as difference
960 if (s.size() != 1 || s.get(0).getPoints().size() != 5) {
961 myprintln("- No JOSM shape: "+getDescription(j));
962 }
963 } else if (s.size() != js.size()) {
964 myprintln("* Different number of shapes ("+s.size()+" != "+js.size()+"): "+getDescription(j));
965 } else {
966 boolean[] edone = new boolean[s.size()];
967 boolean[] jdone = new boolean[js.size()];
968 for (int enums = 0; enums < s.size(); ++enums) {
969 List<Coordinate> ep = s.get(enums).getPoints();
970 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
971 List<Coordinate> jp = js.get(jnums).getPoints();
972 if (ep.size() == jp.size() && !jdone[jnums]) {
973 boolean err = false;
974 for (int nump = 0; nump < ep.size() && !err; ++nump) {
975 Coordinate ept = ep.get(nump);
976 Coordinate jpt = jp.get(nump);
977 if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.00001)
978 err = true;
979 }
980 if (!err) {
981 edone[enums] = true;
982 jdone[jnums] = true;
983 break;
984 }
985 }
986 }
987 }
988 for (int enums = 0; enums < s.size(); ++enums) {
989 List<Coordinate> ep = s.get(enums).getPoints();
990 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
991 List<Coordinate> jp = js.get(jnums).getPoints();
992 if (ep.size() == jp.size() && !jdone[jnums]) {
993 boolean err = false;
994 for (int nump = 0; nump < ep.size() && !err; ++nump) {
995 Coordinate ept = ep.get(nump);
996 Coordinate jpt = jp.get(nump);
997 if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.00001) {
998 String numtxt = Integer.toString(enums+1);
999 if (enums != jnums) {
1000 numtxt += '/' + Integer.toString(jnums+1);
1001 }
1002 myprintln("* Different coordinate for point "+(nump+1)+" of shape "+numtxt+": "+getDescription(j));
1003 break;
1004 }
1005 }
1006 edone[enums] = true;
1007 jdone[jnums] = true;
1008 break;
1009 }
1010 }
1011 }
1012 for (int enums = 0; enums < s.size(); ++enums) {
1013 List<Coordinate> ep = s.get(enums).getPoints();
1014 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
1015 List<Coordinate> jp = js.get(jnums).getPoints();
1016 if (!jdone[jnums]) {
1017 String numtxt = Integer.toString(enums+1);
1018 if (enums != jnums) {
1019 numtxt += '/' + Integer.toString(jnums+1);
1020 }
1021 myprintln("* Different number of points for shape "+numtxt+" ("+ep.size()+" ! = "+jp.size()+"): "
1022 + getDescription(j));
1023 edone[enums] = true;
1024 jdone[jnums] = true;
1025 break;
1026 }
1027 }
1028 }
1029 }
1030 }
1031 }
1032
1033 void doMismatchingIcons() {
1034 myprintln("*** Mismatching icons: ***");
1035 doMismatching(this::compareIcons);
1036 }
1037
1038 void doMismatchingCategories() {
1039 myprintln("*** Mismatching categories: ***");
1040 doMismatching(this::compareCategories);
1041 }
1042
1043 void doMismatching(BiConsumer<ImageryInfo, JsonObject> comparator) {
1044 for (String url : eliUrls.keySet()) {
1045 if (josmUrls.containsKey(url)) {
1046 comparator.accept(josmUrls.get(url), eliUrls.get(url));
1047 }
1048 }
1049 }
1050
1051 void compareIcons(ImageryInfo j, JsonObject e) {
1052 String ij = getIcon(j);
1053 String ie = getIcon(e);
1054 boolean ijok = isNotBlank(ij);
1055 boolean ieok = isNotBlank(ie);
1056 if (ijok && !ieok) {
1057 if (!optionNoEli) {
1058 myprintln("+ No ELI icon: "+getDescription(j));
1059 }
1060 } else if (!ijok && ieok) {
1061 myprintln("- No JOSM icon: "+getDescription(j));
1062 } else if (ijok && ieok && !Objects.equals(ij, ie) && !(
1063 (ie.startsWith("https://osmlab.github.io/editor-layer-index/")
1064 || ie.startsWith("https://raw.githubusercontent.com/osmlab/editor-layer-index/")) &&
1065 ij.startsWith("data:"))) {
1066 String iehttps = ie.replace("http:", "https:");
1067 if (ij.equals(iehttps)) {
1068 myprintln("+ Different icons: "+getDescription(j));
1069 } else {
1070 myprintln("* Different icons: "+getDescription(j));
1071 }
1072 }
1073 }
1074
1075 void compareCategories(ImageryInfo j, JsonObject e) {
1076 String cj = getCategory(j);
1077 String ce = getCategory(e);
1078 boolean cjok = isNotBlank(cj);
1079 boolean ceok = isNotBlank(ce);
1080 if (cjok && !ceok) {
1081 if (!optionNoEli) {
1082 myprintln("+ No ELI category: "+getDescription(j));
1083 }
1084 } else if (!cjok && ceok) {
1085 myprintln("- No JOSM category: "+getDescription(j));
1086 } else if (cjok && ceok && !Objects.equals(cj, ce)) {
1087 myprintln("* Different categories ('"+ce+"' != '"+cj+"'): "+getDescription(j));
1088 }
1089 }
1090
1091 void doMiscellaneousChecks() {
1092 myprintln("*** Miscellaneous checks: ***");
1093 Map<String, ImageryInfo> josmIds = new HashMap<>();
1094 Collection<String> all = Projections.getAllProjectionCodes();
1095 DomainValidator dv = DomainValidator.getInstance();
1096 for (String url : josmUrls.keySet()) {
1097 ImageryInfo j = josmUrls.get(url);
1098 String id = getId(j);
1099 if ("wms".equals(getType(j))) {
1100 String urlLc = url.toLowerCase(Locale.ENGLISH);
1101 if (getProjections(j).isEmpty()) {
1102 myprintln("* WMS without projections: "+getDescription(j));
1103 } else {
1104 List<String> unsupported = new LinkedList<>();
1105 List<String> old = new LinkedList<>();
1106 for (String p : getProjectionsUnstripped(j)) {
1107 if ("CRS:84".equals(p)) {
1108 if (!urlLc.contains("version=1.3")) {
1109 myprintln("* CRS:84 without WMS 1.3: "+getDescription(j));
1110 }
1111 } else if (oldproj.containsKey(p)) {
1112 old.add(p);
1113 } else if (!all.contains(p) && !ignoreproj.contains(p)) {
1114 unsupported.add(p);
1115 }
1116 }
1117 if (!unsupported.isEmpty()) {
1118 myprintln("* Projections "+String.join(", ", unsupported)+" not supported by JOSM: "+getDescription(j));
1119 }
1120 for (String o : old) {
1121 myprintln("* Projection "+o+" is an old unsupported code and has been replaced by "+oldproj.get(o)+": "
1122 + getDescription(j));
1123 }
1124 }
1125 if (urlLc.contains("version=1.3") && !urlLc.contains("crs={proj}")) {
1126 myprintln("* WMS 1.3 with strange CRS specification: "+getDescription(j));
1127 } else if (urlLc.contains("version=1.1") && !urlLc.contains("srs={proj}")) {
1128 myprintln("* WMS 1.1 with strange SRS specification: "+getDescription(j));
1129 }
1130 }
1131 List<String> urls = new LinkedList<>();
1132 if (!"scanex".equals(getType(j))) {
1133 urls.add(url);
1134 }
1135 String jt = getPermissionReferenceUrl(j);
1136 if (isNotBlank(jt) && !"Public Domain".equalsIgnoreCase(jt))
1137 urls.add(jt);
1138 jt = getTermsOfUseUrl(j);
1139 if (isNotBlank(jt))
1140 urls.add(jt);
1141 jt = getAttributionUrl(j);
1142 if (isNotBlank(jt))
1143 urls.add(jt);
1144 jt = getIcon(j);
1145 if (isNotBlank(jt)) {
1146 if (!jt.startsWith("data:image/"))
1147 urls.add(jt);
1148 else {
1149 try {
1150 new ImageProvider(jt).get();
1151 } catch (RuntimeException e) {
1152 myprintln("* Strange Icon: "+getDescription(j));
1153 }
1154 }
1155 }
1156 Pattern patternU = Pattern.compile("^https?://([^/]+?)(:\\d+)?(/.*)?");
1157 for (String u : urls) {
1158 if (!patternU.matcher(u).matches() || u.matches(".*[ \t]+$")) {
1159 myprintln("* Strange URL '"+u+"': "+getDescription(j));
1160 } else {
1161 try {
1162 URL jurl = new URL(u.replaceAll("\\{switch:[^\\}]*\\}", "x"));
1163 String domain = jurl.getHost();
1164 int port = jurl.getPort();
1165 if (!(domain.matches("^\\d+\\.\\d+\\.\\d+\\.\\d+$")) && !dv.isValid(domain))
1166 myprintln("* Strange Domain '"+domain+"': "+getDescription(j));
1167 else if (80 == port || 443 == port) {
1168 myprintln("* Useless port '"+port+"': "+getDescription(j));
1169 }
1170 } catch (MalformedURLException e) {
1171 myprintln("* Malformed URL '"+u+"': "+getDescription(j)+" => "+e.getMessage());
1172 }
1173 }
1174 }
1175
1176 if (josmMirrors.containsKey(url)) {
1177 continue;
1178 }
1179 if (isBlank(id)) {
1180 myprintln("* No JOSM-ID: "+getDescription(j));
1181 } else if (josmIds.containsKey(id)) {
1182 myprintln("* JOSM-ID "+id+" not unique: "+getDescription(j));
1183 } else {
1184 josmIds.put(id, j);
1185 }
1186 String d = getDate(j);
1187 if (isNotBlank(d)) {
1188 Pattern patternD = Pattern.compile("^(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?)(;(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?))?$");
1189 Matcher m = patternD.matcher(d);
1190 if (!m.matches()) {
1191 myprintln("* JOSM-Date '"+d+"' is strange: "+getDescription(j));
1192 } else {
1193 try {
1194 Date first = verifyDate(m.group(2), m.group(4), m.group(6));
1195 Date second = verifyDate(m.group(9), m.group(11), m.group(13));
1196 if (second.compareTo(first) < 0) {
1197 myprintln("* JOSM-Date '"+d+"' is strange (second earlier than first): "+getDescription(j));
1198 }
1199 } catch (Exception e) {
1200 myprintln("* JOSM-Date '"+d+"' is strange ("+e.getMessage()+"): "+getDescription(j));
1201 }
1202 }
1203 }
1204 if (isNotBlank(getAttributionUrl(j)) && isBlank(getAttributionText(j))) {
1205 myprintln("* Attribution link without text: "+getDescription(j));
1206 }
1207 if (isNotBlank(getLogoUrl(j)) && isBlank(getLogoImage(j))) {
1208 myprintln("* Logo link without image: "+getDescription(j));
1209 }
1210 if (isNotBlank(getTermsOfUseText(j)) && isBlank(getTermsOfUseUrl(j))) {
1211 myprintln("* Terms of Use text without link: "+getDescription(j));
1212 }
1213 List<Shape> js = getShapes(j);
1214 if (!js.isEmpty()) {
1215 double minlat = 1000;
1216 double minlon = 1000;
1217 double maxlat = -1000;
1218 double maxlon = -1000;
1219 for (Shape s: js) {
1220 for (Coordinate p: s.getPoints()) {
1221 double lat = p.getLat();
1222 double lon = p.getLon();
1223 if (lat > maxlat) maxlat = lat;
1224 if (lon > maxlon) maxlon = lon;
1225 if (lat < minlat) minlat = lat;
1226 if (lon < minlon) minlon = lon;
1227 }
1228 }
1229 ImageryBounds b = j.getBounds();
1230 if (b.getMinLat() != minlat || b.getMinLon() != minlon || b.getMaxLat() != maxlat || b.getMaxLon() != maxlon) {
1231 myprintln("* Bounds do not match shape (is "+b.getMinLat()+","+b.getMinLon()+","+b.getMaxLat()+","+b.getMaxLon()
1232 + ", calculated <bounds min-lat='"+minlat+"' min-lon='"+minlon+"' max-lat='"+maxlat+"' max-lon='"+maxlon+"'>): "
1233 + getDescription(j));
1234 }
1235 }
1236 List<String> knownCategories = Arrays.asList(
1237 "photo", "elevation", "map", "historicmap", "osmbasedmap", "historicphoto", "qa", "other");
1238 String cat = getCategory(j);
1239 if (isBlank(cat)) {
1240 myprintln("* No category: "+getDescription(j));
1241 } else if (!knownCategories.contains(cat)) {
1242 myprintln("* Strange category "+cat+": "+getDescription(j));
1243 }
1244 }
1245 }
1246
1247 /*
1248 * Utility functions that allow uniform access for both ImageryInfo and JsonObject.
1249 */
1250
1251 static String getUrl(Object e) {
1252 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getUrl();
1253 return ((Map<String, JsonObject>) e).get("properties").getString("url");
1254 }
1255
1256 static String getUrlStripped(Object e) {
1257 return getUrl(e).replaceAll("\\?(apikey|access_token)=.*", "");
1258 }
1259
1260 static String getDate(Object e) {
1261 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getDate() != null ? ((ImageryInfo) e).getDate() : "";
1262 JsonObject p = ((Map<String, JsonObject>) e).get("properties");
1263 String start = p.containsKey("start_date") ? p.getString("start_date") : "";
1264 String end = p.containsKey("end_date") ? p.getString("end_date") : "";
1265 if (!start.isEmpty() && !end.isEmpty())
1266 return start+";"+end;
1267 else if (!start.isEmpty())
1268 return start+";-";
1269 else if (!end.isEmpty())
1270 return "-;"+end;
1271 return "";
1272 }
1273
1274 static Date verifyDate(String year, String month, String day) throws ParseException {
1275 String date;
1276 if (year == null) {
1277 date = "3000-01-01";
1278 } else {
1279 date = year + "-" + (month == null ? "01" : month) + "-" + (day == null ? "01" : day);
1280 }
1281 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");
1282 df.setLenient(false);
1283 return df.parse(date);
1284 }
1285
1286 static String getId(Object e) {
1287 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getId();
1288 return ((Map<String, JsonObject>) e).get("properties").getString("id");
1289 }
1290
1291 static String getName(Object e) {
1292 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getOriginalName();
1293 return ((Map<String, JsonObject>) e).get("properties").getString("name");
1294 }
1295
1296 static List<ImageryInfo> getMirrors(Object e) {
1297 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getMirrors();
1298 return Collections.emptyList();
1299 }
1300
1301 static List<String> getProjections(Object e) {
1302 List<String> r = new ArrayList<>();
1303 List<String> u = getProjectionsUnstripped(e);
1304 if (u != null) {
1305 for (String p : u) {
1306 if (!oldproj.containsKey(p) && !("CRS:84".equals(p) && !(getUrlStripped(e).matches("(?i)version=1\\.3")))) {
1307 r.add(p);
1308 }
1309 }
1310 }
1311 return r;
1312 }
1313
1314 static List<String> getProjectionsUnstripped(Object e) {
1315 List<String> r = null;
1316 if (e instanceof ImageryInfo) {
1317 r = ((ImageryInfo) e).getServerProjections();
1318 } else {
1319 JsonValue s = ((Map<String, JsonObject>) e).get("properties").get("available_projections");
1320 if (s != null) {
1321 r = new ArrayList<>();
1322 for (JsonValue p : s.asJsonArray()) {
1323 r.add(((JsonString) p).getString());
1324 }
1325 }
1326 }
1327 return r != null ? r : Collections.emptyList();
1328 }
1329
1330 static List<Shape> getShapes(Object e) {
1331 if (e instanceof ImageryInfo) {
1332 ImageryBounds bounds = ((ImageryInfo) e).getBounds();
1333 if (bounds != null) {
1334 return bounds.getShapes();
1335 }
1336 return Collections.emptyList();
1337 }
1338 JsonValue ex = ((Map<String, JsonValue>) e).get("geometry");
1339 if (ex != null && !JsonValue.NULL.equals(ex) && !ex.asJsonObject().isNull("coordinates")) {
1340 JsonArray poly = ex.asJsonObject().getJsonArray("coordinates");
1341 List<Shape> l = new ArrayList<>();
1342 for (JsonValue shapes: poly) {
1343 Shape s = new Shape();
1344 for (JsonValue point: shapes.asJsonArray()) {
1345 String lon = point.asJsonArray().getJsonNumber(0).toString();
1346 String lat = point.asJsonArray().getJsonNumber(1).toString();
1347 s.addPoint(lat, lon);
1348 }
1349 l.add(s);
1350 }
1351 return l;
1352 }
1353 return Collections.emptyList();
1354 }
1355
1356 static String getType(Object e) {
1357 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getImageryType().getTypeString();
1358 return ((Map<String, JsonObject>) e).get("properties").getString("type");
1359 }
1360
1361 static Integer getMinZoom(Object e) {
1362 if (e instanceof ImageryInfo) {
1363 int mz = ((ImageryInfo) e).getMinZoom();
1364 return mz == 0 ? null : mz;
1365 } else {
1366 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("min_zoom");
1367 if (num == null) return null;
1368 return num.intValue();
1369 }
1370 }
1371
1372 static Integer getMaxZoom(Object e) {
1373 if (e instanceof ImageryInfo) {
1374 int mz = ((ImageryInfo) e).getMaxZoom();
1375 return mz == 0 ? null : mz;
1376 } else {
1377 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("max_zoom");
1378 if (num == null) return null;
1379 return num.intValue();
1380 }
1381 }
1382
1383 static String getCountryCode(Object e) {
1384 if (e instanceof ImageryInfo) return "".equals(((ImageryInfo) e).getCountryCode()) ? null : ((ImageryInfo) e).getCountryCode();
1385 return ((Map<String, JsonObject>) e).get("properties").getString("country_code", null);
1386 }
1387
1388 static String getQuality(Object e) {
1389 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isBestMarked() ? "eli-best" : null;
1390 return (((Map<String, JsonObject>) e).get("properties").containsKey("best")
1391 && ((Map<String, JsonObject>) e).get("properties").getBoolean("best")) ? "eli-best" : null;
1392 }
1393
1394 static boolean getOverlay(Object e) {
1395 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isOverlay();
1396 return (((Map<String, JsonObject>) e).get("properties").containsKey("overlay")
1397 && ((Map<String, JsonObject>) e).get("properties").getBoolean("overlay"));
1398 }
1399
1400 static String getIcon(Object e) {
1401 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getIcon();
1402 return ((Map<String, JsonObject>) e).get("properties").getString("icon", null);
1403 }
1404
1405 static String getAttributionText(Object e) {
1406 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionText(0, null, null);
1407 try {
1408 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("text", null);
1409 } catch (NullPointerException ex) {
1410 return null;
1411 }
1412 }
1413
1414 static String getAttributionUrl(Object e) {
1415 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionLinkURL();
1416 try {
1417 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("url", null);
1418 } catch (NullPointerException ex) {
1419 return null;
1420 }
1421 }
1422
1423 static String getTermsOfUseText(Object e) {
1424 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseText();
1425 return null;
1426 }
1427
1428 static String getTermsOfUseUrl(Object e) {
1429 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseURL();
1430 return null;
1431 }
1432
1433 static String getCategory(Object e) {
1434 if (e instanceof ImageryInfo) {
1435 return ((ImageryInfo) e).getImageryCategoryOriginalString();
1436 }
1437 return ((Map<String, JsonObject>) e).get("properties").getString("category", null);
1438 }
1439
1440 static String getLogoImage(Object e) {
1441 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageRaw();
1442 return null;
1443 }
1444
1445 static String getLogoUrl(Object e) {
1446 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageURL();
1447 return null;
1448 }
1449
1450 static String getPermissionReferenceUrl(Object e) {
1451 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getPermissionReferenceURL();
1452 return ((Map<String, JsonObject>) e).get("properties").getString("license_url", null);
1453 }
1454
1455 static Map<String, Set<String>> getNoTileHeader(Object e) {
1456 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getNoTileHeaders();
1457 JsonObject nth = ((Map<String, JsonObject>) e).get("properties").getJsonObject("no_tile_header");
1458 return nth == null ? null : nth.keySet().stream().collect(Collectors.toMap(
1459 Function.identity(),
1460 k -> nth.getJsonArray(k).stream().map(x -> ((JsonString) x).getString()).collect(Collectors.toSet())));
1461 }
1462
1463 static Map<String, String> getDescriptions(Object e) {
1464 Map<String, String> res = new HashMap<>();
1465 if (e instanceof ImageryInfo) {
1466 String a = ((ImageryInfo) e).getDescription();
1467 if (a != null) res.put("en", a);
1468 } else {
1469 String a = ((Map<String, JsonObject>) e).get("properties").getString("description", null);
1470 if (a != null) res.put("en", a.replaceAll("''", "'"));
1471 }
1472 return res;
1473 }
1474
1475 static boolean getValidGeoreference(Object e) {
1476 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isGeoreferenceValid();
1477 return false;
1478 }
1479
1480 static boolean getDefault(Object e) {
1481 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isDefaultEntry();
1482 return ((Map<String, JsonObject>) e).get("properties").getBoolean("default", false);
1483 }
1484
1485 String getDescription(Object o) {
1486 String url = getUrl(o);
1487 String cc = getCountryCode(o);
1488 if (cc == null) {
1489 ImageryInfo j = josmUrls.get(url);
1490 if (j != null) cc = getCountryCode(j);
1491 if (cc == null) {
1492 JsonObject e = eliUrls.get(url);
1493 if (e != null) cc = getCountryCode(e);
1494 }
1495 }
1496 if (cc == null) {
1497 cc = "";
1498 } else {
1499 cc = "["+cc+"] ";
1500 }
1501 String name = getName(o);
1502 String id = getId(o);
1503 String d = cc;
1504 if (name != null && !name.isEmpty()) {
1505 d += name;
1506 if (id != null && !id.isEmpty())
1507 d += " ["+id+"]";
1508 } else if (url != null && !url.isEmpty())
1509 d += url;
1510 if (optionShorten) {
1511 if (d.length() > MAXLEN) d = d.substring(0, MAXLEN-1) + "...";
1512 }
1513 return d;
1514 }
1515}
Note: See TracBrowser for help on using the repository browser.