1 | // License: GPL. For details, see LICENSE file.
|
---|
2 |
|
---|
3 | import static java.nio.charset.StandardCharsets.UTF_8;
|
---|
4 | import static org.apache.commons.lang3.StringUtils.isBlank;
|
---|
5 | import static org.apache.commons.lang3.StringUtils.isNotBlank;
|
---|
6 |
|
---|
7 | import java.io.BufferedReader;
|
---|
8 | import java.io.BufferedWriter;
|
---|
9 | import java.io.IOException;
|
---|
10 | import java.io.OutputStreamWriter;
|
---|
11 | import java.io.Writer;
|
---|
12 | import java.lang.reflect.Field;
|
---|
13 | import java.net.MalformedURLException;
|
---|
14 | import java.net.URL;
|
---|
15 | import java.nio.charset.Charset;
|
---|
16 | import java.nio.file.Files;
|
---|
17 | import java.nio.file.Paths;
|
---|
18 | import java.text.DecimalFormat;
|
---|
19 | import java.text.ParseException;
|
---|
20 | import java.text.SimpleDateFormat;
|
---|
21 | import java.util.ArrayList;
|
---|
22 | import java.util.Arrays;
|
---|
23 | import java.util.Calendar;
|
---|
24 | import java.util.Collection;
|
---|
25 | import java.util.Collections;
|
---|
26 | import java.util.Date;
|
---|
27 | import java.util.HashMap;
|
---|
28 | import java.util.LinkedList;
|
---|
29 | import java.util.List;
|
---|
30 | import java.util.Locale;
|
---|
31 | import java.util.Map;
|
---|
32 | import java.util.Map.Entry;
|
---|
33 | import java.util.Objects;
|
---|
34 | import java.util.Set;
|
---|
35 | import java.util.function.BiConsumer;
|
---|
36 | import java.util.function.Function;
|
---|
37 | import java.util.regex.Matcher;
|
---|
38 | import java.util.regex.Pattern;
|
---|
39 | import java.util.stream.Collectors;
|
---|
40 |
|
---|
41 | import org.openstreetmap.gui.jmapviewer.Coordinate;
|
---|
42 | import org.openstreetmap.josm.data.Preferences;
|
---|
43 | import org.openstreetmap.josm.data.imagery.ImageryInfo;
|
---|
44 | import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
|
---|
45 | import org.openstreetmap.josm.data.imagery.Shape;
|
---|
46 | import org.openstreetmap.josm.data.preferences.JosmBaseDirectories;
|
---|
47 | import org.openstreetmap.josm.data.preferences.JosmUrls;
|
---|
48 | import org.openstreetmap.josm.data.projection.Projections;
|
---|
49 | import org.openstreetmap.josm.data.sources.SourceInfo;
|
---|
50 | import org.openstreetmap.josm.data.validation.routines.DomainValidator;
|
---|
51 | import org.openstreetmap.josm.io.imagery.ImageryReader;
|
---|
52 | import org.openstreetmap.josm.spi.preferences.Config;
|
---|
53 | import org.openstreetmap.josm.tools.ImageProvider;
|
---|
54 | import org.openstreetmap.josm.tools.JosmRuntimeException;
|
---|
55 | import org.openstreetmap.josm.tools.Logging;
|
---|
56 | import org.openstreetmap.josm.tools.OptionParser;
|
---|
57 | import org.openstreetmap.josm.tools.OptionParser.OptionCount;
|
---|
58 | import org.openstreetmap.josm.tools.ReflectionUtils;
|
---|
59 | import org.openstreetmap.josm.tools.Utils;
|
---|
60 | import org.xml.sax.SAXException;
|
---|
61 |
|
---|
62 | import jakarta.json.Json;
|
---|
63 | import jakarta.json.JsonArray;
|
---|
64 | import jakarta.json.JsonNumber;
|
---|
65 | import jakarta.json.JsonObject;
|
---|
66 | import jakarta.json.JsonReader;
|
---|
67 | import jakarta.json.JsonString;
|
---|
68 | import jakarta.json.JsonValue;
|
---|
69 |
|
---|
70 | /**
|
---|
71 | * Compare and analyse the differences of the editor layer index and the JOSM imagery list.
|
---|
72 | * The goal is to keep both lists in sync.
|
---|
73 | * <p>
|
---|
74 | * The <a href="https://github.com/osmlab/editor-layer-index">editor layer index</a> project
|
---|
75 | * provides also a version in the JOSM format, but the GEOJSON is the original source format, so we read that.
|
---|
76 | * <p>
|
---|
77 | * For running, the main JOSM binary needs to be in classpath, e.g.
|
---|
78 | * <p>
|
---|
79 | * {@code $ java -cp ../dist/josm-custom.jar SyncEditorLayerIndex}
|
---|
80 | * <p>
|
---|
81 | * Add option {@code -h} to show the available command line flags.
|
---|
82 | */
|
---|
83 | @SuppressWarnings("unchecked")
|
---|
84 | public class SyncEditorLayerIndex {
|
---|
85 |
|
---|
86 | private static final int MAXLEN = 140;
|
---|
87 |
|
---|
88 | private List<ImageryInfo> josmEntries;
|
---|
89 | private JsonArray eliEntries;
|
---|
90 | private JsonArray idEntries;
|
---|
91 | private JsonArray rapidEntries;
|
---|
92 |
|
---|
93 | private final Map<String, JsonObject> eliUrls = new HashMap<>();
|
---|
94 | private final Map<String, JsonObject> idUrls = new HashMap<>();
|
---|
95 | private final Map<String, JsonObject> rapidUrls = new HashMap<>();
|
---|
96 | private final Map<String, ImageryInfo> josmUrls = new HashMap<>();
|
---|
97 | private final Map<String, ImageryInfo> josmMirrors = new HashMap<>();
|
---|
98 | private static final Map<String, String> oldproj = new HashMap<>();
|
---|
99 | private static final List<String> ignoreproj = new LinkedList<>();
|
---|
100 |
|
---|
101 | private static String eliInputFile = "imagery_eli.geojson";
|
---|
102 | private static String idInputFile = "imagery_id.geojson";
|
---|
103 | private static String rapidInputFile = "imagery_rapid.geojson";
|
---|
104 | private static String josmInputFile = "imagery_josm.imagery.xml";
|
---|
105 | private static String ignoreInputFile = "imagery_josm.ignores.txt";
|
---|
106 | private static Writer outputStream;
|
---|
107 | private static String optionOutput;
|
---|
108 | private static boolean optionShorten;
|
---|
109 | private static boolean optionNoSkip;
|
---|
110 | private static boolean optionXhtmlBody;
|
---|
111 | private static boolean optionXhtml;
|
---|
112 | private static String optionEliXml;
|
---|
113 | private static String optionJosmXml;
|
---|
114 | private static String optionEncoding;
|
---|
115 | private static boolean optionNoEli;
|
---|
116 | private final Map<String, String> skip = new HashMap<>();
|
---|
117 | private final Map<String, String> skipStart = new HashMap<>();
|
---|
118 |
|
---|
119 | /**
|
---|
120 | * Main method.
|
---|
121 | * @param args program arguments
|
---|
122 | * @throws IOException if any I/O error occurs
|
---|
123 | * @throws ReflectiveOperationException if any reflective operation error occurs
|
---|
124 | * @throws SAXException if any SAX error occurs
|
---|
125 | */
|
---|
126 | public static void main(String[] args) throws IOException, SAXException, ReflectiveOperationException {
|
---|
127 | Locale.setDefault(Locale.ROOT);
|
---|
128 | parseCommandLineArguments(args);
|
---|
129 | Config.setUrlsProvider(JosmUrls.getInstance());
|
---|
130 | Preferences pref = new Preferences(JosmBaseDirectories.getInstance());
|
---|
131 | Config.setPreferencesInstance(pref);
|
---|
132 | pref.init(false);
|
---|
133 | SyncEditorLayerIndex script = new SyncEditorLayerIndex();
|
---|
134 | script.setupProj();
|
---|
135 | script.loadSkip();
|
---|
136 | script.start();
|
---|
137 | script.loadJosmEntries();
|
---|
138 | if (optionJosmXml != null) {
|
---|
139 | try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(optionJosmXml), UTF_8)) {
|
---|
140 | script.printentries(script.josmEntries, writer);
|
---|
141 | }
|
---|
142 | }
|
---|
143 | script.loadELIEntries();
|
---|
144 | script.loadELIUsers();
|
---|
145 | if (optionEliXml != null) {
|
---|
146 | try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(optionEliXml), UTF_8)) {
|
---|
147 | script.printentries(script.eliEntries, writer);
|
---|
148 | }
|
---|
149 | }
|
---|
150 | script.checkInOneButNotTheOther();
|
---|
151 | script.checkCommonEntries();
|
---|
152 | script.end();
|
---|
153 | if (outputStream != null) {
|
---|
154 | outputStream.close();
|
---|
155 | }
|
---|
156 | }
|
---|
157 |
|
---|
158 | /**
|
---|
159 | * Displays help on the console
|
---|
160 | */
|
---|
161 | private static void showHelp() {
|
---|
162 | System.out.println(getHelp());
|
---|
163 | System.exit(0);
|
---|
164 | }
|
---|
165 |
|
---|
166 | static String getHelp() {
|
---|
167 | return "usage: java -cp build SyncEditorLayerIndex\n" +
|
---|
168 | "-c,--encoding <encoding> output encoding (defaults to UTF-8 or cp850 on Windows)\n" +
|
---|
169 | "-e,--eli_input <eli_input> Input file for the editor layer index (geojson). " +
|
---|
170 | "Default is imagery_eli.geojson (current directory).\n" +
|
---|
171 | "-d,--id_input <id_input> Input file for the id index (geojson). " +
|
---|
172 | "Default is imagery_id.geojson (current directory).\n" +
|
---|
173 | "-h,--help show this help\n" +
|
---|
174 | "-i,--ignore_input <ignore_input> Input file for the ignore list. Default is imagery_josm.ignores.txt (current directory).\n" +
|
---|
175 | "-j,--josm_input <josm_input> Input file for the JOSM imagery list (xml). " +
|
---|
176 | "Default is imagery_josm.imagery.xml (current directory).\n" +
|
---|
177 | "-m,--noeli don't show output for ELI, Rapid or iD problems\n" +
|
---|
178 | "-n,--noskip don't skip known entries\n" +
|
---|
179 | "-o,--output <output> Output file, - prints to stdout (default: -)\n" +
|
---|
180 | "-p,--elixml <elixml> ELI entries for use in JOSM as XML file (incomplete)\n" +
|
---|
181 | "-q,--josmxml <josmxml> JOSM entries reoutput as XML file (incomplete)\n" +
|
---|
182 | "-r,--rapid_input <rapid_input> Input file for the rapid index (geojson). " +
|
---|
183 | "Default is imagery_rapid.geojson (current directory).\n" +
|
---|
184 | "-s,--shorten shorten the output, so it is easier to read in a console window\n" +
|
---|
185 | "-x,--xhtmlbody create XHTML body for display in a web page\n" +
|
---|
186 | "-X,--xhtml create XHTML for display in a web page\n";
|
---|
187 | }
|
---|
188 |
|
---|
189 | /**
|
---|
190 | * Parse command line arguments.
|
---|
191 | * @param args program arguments
|
---|
192 | * @throws IOException in case of I/O error
|
---|
193 | */
|
---|
194 | static void parseCommandLineArguments(String[] args) throws IOException {
|
---|
195 | new OptionParser("JOSM/ELI synchronization script")
|
---|
196 | .addFlagParameter("help", SyncEditorLayerIndex::showHelp)
|
---|
197 | .addShortAlias("help", "h")
|
---|
198 | .addArgumentParameter("output", OptionCount.OPTIONAL, x -> optionOutput = x)
|
---|
199 | .addShortAlias("output", "o")
|
---|
200 | .addArgumentParameter("eli_input", OptionCount.OPTIONAL, x -> eliInputFile = x)
|
---|
201 | .addShortAlias("eli_input", "e")
|
---|
202 | .addArgumentParameter("id_input", OptionCount.OPTIONAL, x -> idInputFile = x)
|
---|
203 | .addShortAlias("id_input", "d")
|
---|
204 | .addArgumentParameter("rapid_input", OptionCount.OPTIONAL, x -> rapidInputFile = x)
|
---|
205 | .addShortAlias("rapid_input", "r")
|
---|
206 | .addArgumentParameter("josm_input", OptionCount.OPTIONAL, x -> josmInputFile = x)
|
---|
207 | .addShortAlias("josm_input", "j")
|
---|
208 | .addArgumentParameter("ignore_input", OptionCount.OPTIONAL, x -> ignoreInputFile = x)
|
---|
209 | .addShortAlias("ignore_input", "i")
|
---|
210 | .addFlagParameter("shorten", () -> optionShorten = true)
|
---|
211 | .addShortAlias("shorten", "s")
|
---|
212 | .addFlagParameter("noskip", () -> optionNoSkip = true)
|
---|
213 | .addShortAlias("noskip", "n")
|
---|
214 | .addFlagParameter("xhtmlbody", () -> optionXhtmlBody = true)
|
---|
215 | .addShortAlias("xhtmlbody", "x")
|
---|
216 | .addFlagParameter("xhtml", () -> optionXhtml = true)
|
---|
217 | .addShortAlias("xhtml", "X")
|
---|
218 | .addArgumentParameter("elixml", OptionCount.OPTIONAL, x -> optionEliXml = x)
|
---|
219 | .addShortAlias("elixml", "p")
|
---|
220 | .addArgumentParameter("josmxml", OptionCount.OPTIONAL, x -> optionJosmXml = x)
|
---|
221 | .addShortAlias("josmxml", "q")
|
---|
222 | .addFlagParameter("noeli", () -> optionNoEli = true)
|
---|
223 | .addShortAlias("noeli", "m")
|
---|
224 | .addArgumentParameter("encoding", OptionCount.OPTIONAL, x -> optionEncoding = x)
|
---|
225 | .addShortAlias("encoding", "c")
|
---|
226 | .parseOptionsOrExit(Arrays.asList(args));
|
---|
227 |
|
---|
228 | if (optionOutput != null && !"-".equals(optionOutput)) {
|
---|
229 | outputStream = Files.newBufferedWriter(Paths.get(optionOutput), optionEncoding != null ? Charset.forName(optionEncoding) : UTF_8);
|
---|
230 | } else if (optionEncoding != null) {
|
---|
231 | outputStream = new OutputStreamWriter(System.out, optionEncoding);
|
---|
232 | }
|
---|
233 | }
|
---|
234 |
|
---|
235 | void setupProj() {
|
---|
236 | oldproj.put("EPSG:3359", "EPSG:3404");
|
---|
237 | oldproj.put("EPSG:3785", "EPSG:3857");
|
---|
238 | oldproj.put("EPSG:31297", "EPGS:31287");
|
---|
239 | oldproj.put("EPSG:31464", "EPSG:31468");
|
---|
240 | oldproj.put("EPSG:54004", "EPSG:3857");
|
---|
241 | oldproj.put("EPSG:102100", "EPSG:3857");
|
---|
242 | oldproj.put("EPSG:102113", "EPSG:3857");
|
---|
243 | oldproj.put("EPSG:900913", "EPGS:3857");
|
---|
244 | ignoreproj.add("EPSG:4267");
|
---|
245 | ignoreproj.add("EPSG:5221");
|
---|
246 | ignoreproj.add("EPSG:5514");
|
---|
247 | ignoreproj.add("EPSG:32019");
|
---|
248 | ignoreproj.add("EPSG:102066");
|
---|
249 | ignoreproj.add("EPSG:102067");
|
---|
250 | ignoreproj.add("EPSG:102685");
|
---|
251 | ignoreproj.add("EPSG:102711");
|
---|
252 | }
|
---|
253 |
|
---|
254 | void loadSkip() throws IOException {
|
---|
255 | final Pattern pattern = Pattern.compile("^\\|\\| *(ELI|Ignore) *\\|\\| *\\{\\{\\{(.+)}}} *\\|\\|");
|
---|
256 | try (BufferedReader fr = Files.newBufferedReader(Paths.get(ignoreInputFile), UTF_8)) {
|
---|
257 | String line;
|
---|
258 |
|
---|
259 | while ((line = fr.readLine()) != null) {
|
---|
260 | Matcher res = pattern.matcher(line);
|
---|
261 | if (res.matches()) {
|
---|
262 | String s = res.group(2);
|
---|
263 | if (s.endsWith("...")) {
|
---|
264 | s = s.substring(0, s.length() - 3);
|
---|
265 | if ("Ignore".equals(res.group(1))) {
|
---|
266 | skipStart.put(s, "green");
|
---|
267 | } else {
|
---|
268 | skipStart.put(s, "darkgoldenrod");
|
---|
269 | }
|
---|
270 | } else {
|
---|
271 | if ("Ignore".equals(res.group(1))) {
|
---|
272 | skip.put(s, "green");
|
---|
273 | } else {
|
---|
274 | skip.put(s, "darkgoldenrod");
|
---|
275 | }
|
---|
276 | }
|
---|
277 | }
|
---|
278 | }
|
---|
279 | }
|
---|
280 | }
|
---|
281 |
|
---|
282 | void myprintlnfinal(String s) {
|
---|
283 | if (outputStream != null) {
|
---|
284 | try {
|
---|
285 | outputStream.write(s + System.getProperty("line.separator"));
|
---|
286 | } catch (IOException e) {
|
---|
287 | throw new JosmRuntimeException(e);
|
---|
288 | }
|
---|
289 | } else {
|
---|
290 | System.out.println(s);
|
---|
291 | }
|
---|
292 | }
|
---|
293 |
|
---|
294 | String isSkipString(String s) {
|
---|
295 | if (skip.containsKey(s))
|
---|
296 | return skip.get(s);
|
---|
297 | for (Entry<String, String> str : skipStart.entrySet()) {
|
---|
298 | if (s.startsWith(str.getKey())) {
|
---|
299 | skipStart.remove(str.getKey());
|
---|
300 | return str.getValue();
|
---|
301 | }
|
---|
302 | }
|
---|
303 | return null;
|
---|
304 | }
|
---|
305 |
|
---|
306 | void myprintln(String s) {
|
---|
307 | String color;
|
---|
308 | final String escaped = s.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">");
|
---|
309 | if ((color = isSkipString(s)) != null) {
|
---|
310 | skip.remove(s);
|
---|
311 | if (optionXhtmlBody || optionXhtml) {
|
---|
312 | s = "<pre style=\"margin:3px;color:"+color+"\">"
|
---|
313 | + escaped +"</pre>";
|
---|
314 | }
|
---|
315 | if (!optionNoSkip) {
|
---|
316 | return;
|
---|
317 | }
|
---|
318 | } else if (optionXhtmlBody || optionXhtml) {
|
---|
319 | color =
|
---|
320 | s.startsWith("***") ? "black" :
|
---|
321 | ((s.startsWith("+ ") || s.startsWith("+++ ELI")) ? "blue" :
|
---|
322 | (s.startsWith("#") ? "indigo" :
|
---|
323 | (s.startsWith("!") ? "orange" :
|
---|
324 | (s.startsWith("~") ? "red" : "brown"))));
|
---|
325 | s = "<pre style=\"margin:3px;color:"+color+"\">"+ escaped +"</pre>";
|
---|
326 | }
|
---|
327 | if ((s.startsWith("+ ") || s.startsWith("+++ ELI") || s.startsWith("#")) && optionNoEli) {
|
---|
328 | return;
|
---|
329 | }
|
---|
330 | myprintlnfinal(s);
|
---|
331 | }
|
---|
332 |
|
---|
333 | void start() {
|
---|
334 | if (optionXhtml) {
|
---|
335 | myprintlnfinal(
|
---|
336 | "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n");
|
---|
337 | myprintlnfinal(
|
---|
338 | "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>"+
|
---|
339 | "<title>JOSM - ELI differences</title></head><body>\n");
|
---|
340 | }
|
---|
341 | }
|
---|
342 |
|
---|
343 | void end() {
|
---|
344 | for (String s : skip.keySet()) {
|
---|
345 | myprintln("+++ Obsolete skip entry: " + s);
|
---|
346 | }
|
---|
347 | for (String s : skipStart.keySet()) {
|
---|
348 | myprintln("+++ Obsolete skip entry: " + s + "...");
|
---|
349 | }
|
---|
350 | if (optionXhtml) {
|
---|
351 | myprintlnfinal("</body></html>\n");
|
---|
352 | }
|
---|
353 | }
|
---|
354 |
|
---|
355 | void loadELIEntries() throws IOException {
|
---|
356 | try (JsonReader jr = Json.createReader(Files.newBufferedReader(Paths.get(eliInputFile), UTF_8))) {
|
---|
357 | eliEntries = jr.readObject().getJsonArray("features");
|
---|
358 | }
|
---|
359 |
|
---|
360 | for (JsonValue e : eliEntries) {
|
---|
361 | String url = getUrlStripped(e);
|
---|
362 | if (url.contains("{z}")) {
|
---|
363 | myprintln("+++ ELI-URL uses {z} instead of {zoom}: "+getDescription(e));
|
---|
364 | url = url.replace("{z}", "{zoom}");
|
---|
365 | }
|
---|
366 | if (eliUrls.containsKey(url)) {
|
---|
367 | myprintln("+++ ELI-URL is not unique: "+url);
|
---|
368 | } else {
|
---|
369 | eliUrls.put(url, e.asJsonObject());
|
---|
370 | }
|
---|
371 | JsonArray s = e.asJsonObject().get("properties").asJsonObject().getJsonArray("available_projections");
|
---|
372 | if (s != null) {
|
---|
373 | String urlLc = url.toLowerCase(Locale.ENGLISH);
|
---|
374 | List<String> old = new LinkedList<>();
|
---|
375 | for (JsonValue p : s) {
|
---|
376 | String proj = ((JsonString) p).getString();
|
---|
377 | if (oldproj.containsKey(proj) || ("CRS:84".equals(proj) && !urlLc.contains("version=1.3"))) {
|
---|
378 | old.add(proj);
|
---|
379 | }
|
---|
380 | }
|
---|
381 | if (!old.isEmpty()) {
|
---|
382 | myprintln("+ ELI Projections "+String.join(", ", old)+" not useful: "+getDescription(e));
|
---|
383 | }
|
---|
384 | }
|
---|
385 | }
|
---|
386 | myprintln("*** Loaded "+eliEntries.size()+" entries (ELI). ***");
|
---|
387 | }
|
---|
388 | void loadELIUsers() throws IOException {
|
---|
389 | try (JsonReader jr = Json.createReader(Files.newBufferedReader(Paths.get(idInputFile), UTF_8))) {
|
---|
390 | idEntries = jr.readArray();
|
---|
391 | }
|
---|
392 | for (JsonValue e : idEntries) {
|
---|
393 | String url = getUrlStripped(e);
|
---|
394 | if (!eliUrls.containsKey(url))
|
---|
395 | myprintln("+++ iD-URL not in ELI: "+url);
|
---|
396 | idUrls.put(url, e.asJsonObject());
|
---|
397 | }
|
---|
398 | myprintln("*** Loaded "+idEntries.size()+" entries (iD). ***");
|
---|
399 | try (JsonReader jr = Json.createReader(Files.newBufferedReader(Paths.get(rapidInputFile), UTF_8))) {
|
---|
400 | rapidEntries = jr.readArray();
|
---|
401 | }
|
---|
402 | for (JsonValue e : rapidEntries) {
|
---|
403 | String url = getUrlStripped(e);
|
---|
404 | if (!eliUrls.containsKey(url))
|
---|
405 | myprintln("+++ Rapid-URL not in ELI: "+url);
|
---|
406 | rapidUrls.put(url, e.asJsonObject());
|
---|
407 | }
|
---|
408 | myprintln("*** Loaded "+rapidEntries.size()+" entries (Rapid). ***");
|
---|
409 | }
|
---|
410 |
|
---|
411 | String cdata(String s) {
|
---|
412 | return cdata(s, false);
|
---|
413 | }
|
---|
414 |
|
---|
415 | String cdata(String s, boolean escape) {
|
---|
416 | if (escape) {
|
---|
417 | return s.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">");
|
---|
418 | } else if (s.matches(".*[<>&].*"))
|
---|
419 | return "<![CDATA["+s+"]]>";
|
---|
420 | return s;
|
---|
421 | }
|
---|
422 |
|
---|
423 | String maininfo(Object entry, String offset) {
|
---|
424 | String t = getType(entry);
|
---|
425 | String res = offset + "<type>"+t+"</type>\n";
|
---|
426 | res += offset + "<url>"+cdata(getUrl(entry))+"</url>\n";
|
---|
427 | if (getMinZoom(entry) != null)
|
---|
428 | res += offset + "<min-zoom>"+getMinZoom(entry)+"</min-zoom>\n";
|
---|
429 | if (getMaxZoom(entry) != null)
|
---|
430 | res += offset + "<max-zoom>"+getMaxZoom(entry)+"</max-zoom>\n";
|
---|
431 | if ("wms".equals(t)) {
|
---|
432 | List<String> p = getProjections(entry);
|
---|
433 | if (p != null) {
|
---|
434 | res += offset + "<projections>\n";
|
---|
435 | for (String c : p) {
|
---|
436 | res += offset + " <code>"+c+"</code>\n";
|
---|
437 | }
|
---|
438 | res += offset + "</projections>\n";
|
---|
439 | }
|
---|
440 | }
|
---|
441 | return res;
|
---|
442 | }
|
---|
443 |
|
---|
444 | void printentries(List<?> entries, Writer stream) throws IOException {
|
---|
445 | DecimalFormat df = new DecimalFormat("#.#######");
|
---|
446 | df.setRoundingMode(java.math.RoundingMode.CEILING);
|
---|
447 | stream.write("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n");
|
---|
448 | stream.write("<imagery xmlns=\"http://josm.openstreetmap.de/maps-1.0\">\n");
|
---|
449 | for (Object e : entries) {
|
---|
450 | stream.write(" <entry"
|
---|
451 | + ("eli-best".equals(getQuality(e)) ? " eli-best=\"true\"" : "")
|
---|
452 | + (getOverlay(e) ? " overlay=\"true\"" : "")
|
---|
453 | + ">\n");
|
---|
454 | String t;
|
---|
455 | if (isNotBlank(t = getName(e)))
|
---|
456 | stream.write(" <name>"+cdata(t, true)+"</name>\n");
|
---|
457 | if (isNotBlank(t = getId(e)))
|
---|
458 | stream.write(" <id>"+t+"</id>\n");
|
---|
459 | if (isNotBlank(t = getCategory(e)))
|
---|
460 | stream.write(" <category>"+t+"</category>\n");
|
---|
461 | if (isNotBlank(t = getDate(e)))
|
---|
462 | stream.write(" <date>"+t+"</date>\n");
|
---|
463 | if (isNotBlank(t = getCountryCode(e)))
|
---|
464 | stream.write(" <country-code>"+t+"</country-code>\n");
|
---|
465 | if ((getDefault(e)))
|
---|
466 | stream.write(" <default>true</default>\n");
|
---|
467 | stream.write(maininfo(e, " "));
|
---|
468 | if (isNotBlank(t = getAttributionText(e)))
|
---|
469 | stream.write(" <attribution-text mandatory=\"true\">"+cdata(t, true)+"</attribution-text>\n");
|
---|
470 | if (isNotBlank(t = getAttributionUrl(e)))
|
---|
471 | stream.write(" <attribution-url>"+cdata(t)+"</attribution-url>\n");
|
---|
472 | if (isNotBlank(t = getLogoImage(e)))
|
---|
473 | stream.write(" <logo-image>"+cdata(t, true)+"</logo-image>\n");
|
---|
474 | if (isNotBlank(t = getLogoUrl(e)))
|
---|
475 | stream.write(" <logo-url>"+cdata(t)+"</logo-url>\n");
|
---|
476 | if (isNotBlank(t = getTermsOfUseText(e)))
|
---|
477 | stream.write(" <terms-of-use-text>"+cdata(t, true)+"</terms-of-use-text>\n");
|
---|
478 | if (isNotBlank(t = getTermsOfUseUrl(e)))
|
---|
479 | stream.write(" <terms-of-use-url>"+cdata(t)+"</terms-of-use-url>\n");
|
---|
480 | if (isNotBlank(t = getPermissionReferenceUrl(e)))
|
---|
481 | stream.write(" <permission-ref>"+cdata(t)+"</permission-ref>\n");
|
---|
482 | if (isNotBlank(t = getPrivacyPolicyUrl(e)))
|
---|
483 | stream.write(" <privacy-policy-url>"+cdata(t)+"</privacy-policy-url>\n");
|
---|
484 | if ((getValidGeoreference(e)))
|
---|
485 | stream.write(" <valid-georeference>true</valid-georeference>\n");
|
---|
486 | if (isNotBlank(t = getIcon(e)))
|
---|
487 | stream.write(" <icon>"+cdata(t)+"</icon>\n");
|
---|
488 | for (Entry<String, String> d : getDescriptions(e).entrySet()) {
|
---|
489 | stream.write(" <description lang=\""+d.getKey()+"\">"+cdata(d.getValue(), true)+"</description>\n");
|
---|
490 | }
|
---|
491 | for (ImageryInfo m : getMirrors(e)) {
|
---|
492 | stream.write(" <mirror>\n"+maininfo(m, " ")+" </mirror>\n");
|
---|
493 | }
|
---|
494 | double minlat = 1000;
|
---|
495 | double minlon = 1000;
|
---|
496 | double maxlat = -1000;
|
---|
497 | double maxlon = -1000;
|
---|
498 | String shapes = "";
|
---|
499 | String sep = "\n ";
|
---|
500 | try {
|
---|
501 | for (Shape s: getShapes(e)) {
|
---|
502 | shapes += " <shape>";
|
---|
503 | int i = 0;
|
---|
504 | for (Coordinate p: s.getPoints()) {
|
---|
505 | double lat = p.getLat();
|
---|
506 | double lon = p.getLon();
|
---|
507 | if (lat > maxlat) maxlat = lat;
|
---|
508 | if (lon > maxlon) maxlon = lon;
|
---|
509 | if (lat < minlat) minlat = lat;
|
---|
510 | if (lon < minlon) minlon = lon;
|
---|
511 | if ((i++ % 3) == 0) {
|
---|
512 | shapes += sep + " ";
|
---|
513 | }
|
---|
514 | shapes += "<point lat='"+df.format(lat)+"' lon='"+df.format(lon)+"'/>";
|
---|
515 | }
|
---|
516 | shapes += sep + "</shape>\n";
|
---|
517 | }
|
---|
518 | } catch (IllegalArgumentException illegalArgumentException) {
|
---|
519 | Logging.trace(illegalArgumentException);
|
---|
520 | }
|
---|
521 | if (!shapes.isEmpty()) {
|
---|
522 | stream.write(" <bounds min-lat='"+df.format(minlat)
|
---|
523 | +"' min-lon='"+df.format(minlon)
|
---|
524 | +"' max-lat='"+df.format(maxlat)
|
---|
525 | +"' max-lon='"+df.format(maxlon)+"'>\n");
|
---|
526 | stream.write(shapes + " </bounds>\n");
|
---|
527 | }
|
---|
528 | stream.write(" </entry>\n");
|
---|
529 | }
|
---|
530 | stream.write("</imagery>\n");
|
---|
531 | stream.close();
|
---|
532 | }
|
---|
533 |
|
---|
534 | void loadJosmEntries() throws IOException, SAXException, ReflectiveOperationException {
|
---|
535 | try (ImageryReader reader = new ImageryReader(josmInputFile)) {
|
---|
536 | josmEntries = reader.parse();
|
---|
537 | }
|
---|
538 |
|
---|
539 | for (ImageryInfo e : josmEntries) {
|
---|
540 | if (!e.isValid()) {
|
---|
541 | myprintln("~~~ JOSM-Entry missing fields (" + String.join(", ", e.getMissingFields()) + "): " + getDescription(e));
|
---|
542 | }
|
---|
543 | if (isBlank(getUrl(e))) {
|
---|
544 | myprintln("~~~ JOSM-Entry without URL: " + getDescription(e));
|
---|
545 | continue;
|
---|
546 | }
|
---|
547 | if (isBlank(e.getDate()) && e.getDate() != null) {
|
---|
548 | myprintln("~~~ JOSM-Entry with empty Date: " + getDescription(e));
|
---|
549 | continue;
|
---|
550 | }
|
---|
551 | if (isBlank(getName(e))) {
|
---|
552 | myprintln("~~~ JOSM-Entry without Name: " + getDescription(e));
|
---|
553 | continue;
|
---|
554 | }
|
---|
555 | String url = getUrlStripped(e);
|
---|
556 | if (url.contains("{z}")) {
|
---|
557 | myprintln("~~~ JOSM-URL uses {z} instead of {zoom}: "+getDescription(e));
|
---|
558 | url = url.replace("{z}", "{zoom}");
|
---|
559 | }
|
---|
560 | if (josmUrls.containsKey(url)) {
|
---|
561 | myprintln("~~~ JOSM-URL is not unique: "+url);
|
---|
562 | } else {
|
---|
563 | josmUrls.put(url, e);
|
---|
564 | }
|
---|
565 | for (ImageryInfo m : e.getMirrors()) {
|
---|
566 | url = getUrlStripped(m);
|
---|
567 | Field origNameField = SourceInfo.class.getDeclaredField("origName");
|
---|
568 | ReflectionUtils.setObjectsAccessible(origNameField);
|
---|
569 | origNameField.set(m, m.getOriginalName().replaceAll(" mirror server( \\d+)?", ""));
|
---|
570 | if (josmUrls.containsKey(url)) {
|
---|
571 | myprintln("~~~ JOSM-Mirror-URL is not unique: "+url);
|
---|
572 | } else {
|
---|
573 | josmUrls.put(url, m);
|
---|
574 | josmMirrors.put(url, m);
|
---|
575 | }
|
---|
576 | }
|
---|
577 | }
|
---|
578 | myprintln("*** Loaded "+josmEntries.size()+" entries (JOSM). ***");
|
---|
579 | }
|
---|
580 |
|
---|
581 | // catch reordered arguments, make them uppercase, and switches to WMS version 1.3.0
|
---|
582 | String unifyWMS(String url) {
|
---|
583 | String[] x = url.replaceAll("(?i)VERSION=[0-9.]+", "VERSION=x")
|
---|
584 | .replaceAll("(?i)SRS=", "CRS=")
|
---|
585 | .replaceAll("(?i)BBOX=", "BBOX=")
|
---|
586 | .replaceAll("(?i)FORMAT=", "FORMAT=")
|
---|
587 | .replaceAll("(?i)LAYERS=", "LAYERS=")
|
---|
588 | .replaceAll("(?i)MAP=", "MAP=")
|
---|
589 | .replaceAll("(?i)REQUEST=", "REQUEST=")
|
---|
590 | .replaceAll("(?i)SERVICE=", "SERVICE=")
|
---|
591 | .replaceAll("(?i)STYLES=", "STYLES=")
|
---|
592 | .replaceAll("(?i)TRANSPARENT=FALSE", "TRANSPARENT=FALSE")
|
---|
593 | .replaceAll("(?i)TRANSPARENT=TRUE", "TRANSPARENT=TRUE")
|
---|
594 | .replaceAll("(?i)WIDTH=", "WIDTH=")
|
---|
595 | .replaceAll("(?i)HEIGHT=", "HEIGHT=")
|
---|
596 | .split("\\?");
|
---|
597 | return x[0] +"?" + Arrays.stream(x[1].split("&"))
|
---|
598 | .filter(s -> !s.endsWith("=")) // filter empty params
|
---|
599 | .sorted()
|
---|
600 | .collect(Collectors.joining("&"));
|
---|
601 | }
|
---|
602 |
|
---|
603 | void checkInOneButNotTheOther() {
|
---|
604 | List<String> le = new LinkedList<>(eliUrls.keySet());
|
---|
605 | List<String> lj = new LinkedList<>(josmUrls.keySet());
|
---|
606 |
|
---|
607 | for (String url : new LinkedList<>(le)) {
|
---|
608 | if (lj.contains(url)) {
|
---|
609 | le.remove(url);
|
---|
610 | lj.remove(url);
|
---|
611 | }
|
---|
612 | }
|
---|
613 |
|
---|
614 | if (!le.isEmpty() && !lj.isEmpty()) {
|
---|
615 | List<String> ke = new LinkedList<>(le);
|
---|
616 | for (String urle : ke) {
|
---|
617 | JsonObject e = eliUrls.get(urle);
|
---|
618 | String ide = getId(e);
|
---|
619 | String urlhttps = urle.replace("http:", "https:");
|
---|
620 | if (lj.contains(urlhttps)) {
|
---|
621 | myprintln("+ Missing https: "+getDescription(e));
|
---|
622 | eliUrls.put(urlhttps, eliUrls.get(urle));
|
---|
623 | eliUrls.remove(urle);
|
---|
624 | le.remove(urle);
|
---|
625 | lj.remove(urlhttps);
|
---|
626 | } else if (isNotBlank(ide)) {
|
---|
627 | checkUrlsEquality(ide, e, urle, le, lj);
|
---|
628 | }
|
---|
629 | }
|
---|
630 | }
|
---|
631 |
|
---|
632 | myprintln("*** URLs found in ELI but not in JOSM ("+le.size()+"): ***");
|
---|
633 | Collections.sort(le);
|
---|
634 | if (!le.isEmpty()) {
|
---|
635 | for (String l : le) {
|
---|
636 | String e = "";
|
---|
637 | if(idUrls.get(l) != null && rapidUrls.get(l) != null)
|
---|
638 | e = " **iD+Rapid**";
|
---|
639 | else if(idUrls.get(l) != null)
|
---|
640 | e = " **iD**";
|
---|
641 | else if(rapidUrls.get(l) != null)
|
---|
642 | e = " **Rapid**";
|
---|
643 | myprintln("- " + getDescription(eliUrls.get(l)) + e);
|
---|
644 | }
|
---|
645 | }
|
---|
646 | myprintln("*** URLs found in JOSM but not in ELI ("+lj.size()+"): ***");
|
---|
647 | Collections.sort(lj);
|
---|
648 | if (!lj.isEmpty()) {
|
---|
649 | for (String l : lj) {
|
---|
650 | myprintln("+ " + getDescription(josmUrls.get(l)));
|
---|
651 | }
|
---|
652 | }
|
---|
653 | }
|
---|
654 |
|
---|
655 | void checkUrlsEquality(String ide, JsonObject e, String urle, List<String> le, List<String> lj) {
|
---|
656 | for (String urlj : new LinkedList<>(lj)) {
|
---|
657 | ImageryInfo j = josmUrls.get(urlj);
|
---|
658 | String idj = getId(j);
|
---|
659 |
|
---|
660 | if (checkUrlEquality(ide, "id", idj, e, j, urle, urlj, le, lj)) {
|
---|
661 | return;
|
---|
662 | }
|
---|
663 | Collection<String> old = j.getOldIds();
|
---|
664 | if (old != null) {
|
---|
665 | for (String oidj : old) {
|
---|
666 | if (checkUrlEquality(ide, "oldid", oidj, e, j, urle, urlj, le, lj)) {
|
---|
667 | return;
|
---|
668 | }
|
---|
669 | }
|
---|
670 | }
|
---|
671 | }
|
---|
672 | }
|
---|
673 |
|
---|
674 | boolean checkUrlEquality(
|
---|
675 | String ide, String idtype, String idj, JsonObject e, ImageryInfo j, String urle, String urlj, List<String> le, List<String> lj) {
|
---|
676 | if (ide.equals(idj) && Objects.equals(getType(j), getType(e))) {
|
---|
677 | if (getType(j).equals("wms") && unifyWMS(urle).equals(unifyWMS(urlj))) {
|
---|
678 | myprintln("# WMS-URL for "+idtype+" "+idj+" modified: "+getDescription(j));
|
---|
679 | } else {
|
---|
680 | myprintln("* URL for "+idtype+" "+idj+" differs ("+urle+"): "+getDescription(j));
|
---|
681 | }
|
---|
682 | le.remove(urle);
|
---|
683 | lj.remove(urlj);
|
---|
684 | // replace key for this entry with JOSM URL
|
---|
685 | eliUrls.remove(urle);
|
---|
686 | eliUrls.put(urlj, e);
|
---|
687 | return true;
|
---|
688 | }
|
---|
689 | return false;
|
---|
690 | }
|
---|
691 |
|
---|
692 | void checkCommonEntries() {
|
---|
693 | doSameUrlButDifferentName();
|
---|
694 | doSameUrlButDifferentId();
|
---|
695 | doSameUrlButDifferentType();
|
---|
696 | doSameUrlButDifferentZoomBounds();
|
---|
697 | doSameUrlButDifferentCountryCode();
|
---|
698 | doSameUrlButDifferentQuality();
|
---|
699 | doSameUrlButDifferentDates();
|
---|
700 | doSameUrlButDifferentInformation();
|
---|
701 | doMismatchingShapes();
|
---|
702 | doMismatchingIcons();
|
---|
703 | doMismatchingCategories();
|
---|
704 | doMiscellaneousChecks();
|
---|
705 | }
|
---|
706 |
|
---|
707 | void doSameUrlButDifferentName() {
|
---|
708 | myprintln("*** Same URL, but different name: ***");
|
---|
709 | for (String url : eliUrls.keySet()) {
|
---|
710 | JsonObject e = eliUrls.get(url);
|
---|
711 | if (!josmUrls.containsKey(url)) continue;
|
---|
712 | ImageryInfo j = josmUrls.get(url);
|
---|
713 | String ename = getName(e).replace("'", "\u2019");
|
---|
714 | String jname = getName(j).replace("'", "\u2019");
|
---|
715 | if (!ename.equals(jname)) {
|
---|
716 | myprintln("* Name differs ('"+getName(e)+"' != '"+getName(j)+"'): "+getUrl(j));
|
---|
717 | }
|
---|
718 | }
|
---|
719 | }
|
---|
720 |
|
---|
721 | void doSameUrlButDifferentId() {
|
---|
722 | myprintln("*** Same URL, but different Id: ***");
|
---|
723 | for (String url : eliUrls.keySet()) {
|
---|
724 | JsonObject e = eliUrls.get(url);
|
---|
725 | if (!josmUrls.containsKey(url)) continue;
|
---|
726 | ImageryInfo j = josmUrls.get(url);
|
---|
727 | String ename = getId(e);
|
---|
728 | String jname = getId(j);
|
---|
729 | if (!Objects.equals(ename, jname)) {
|
---|
730 | myprintln("# Id differs ('"+getId(e)+"' != '"+getId(j)+"'): "+getUrl(j));
|
---|
731 | }
|
---|
732 | }
|
---|
733 | }
|
---|
734 |
|
---|
735 | void doSameUrlButDifferentType() {
|
---|
736 | myprintln("*** Same URL, but different type: ***");
|
---|
737 | for (String url : eliUrls.keySet()) {
|
---|
738 | JsonObject e = eliUrls.get(url);
|
---|
739 | if (!josmUrls.containsKey(url)) continue;
|
---|
740 | ImageryInfo j = josmUrls.get(url);
|
---|
741 | if (!Objects.equals(getType(e), getType(j))) {
|
---|
742 | myprintln("* Type differs ("+getType(e)+" != "+getType(j)+"): "+getName(j)+" - "+getUrl(j));
|
---|
743 | }
|
---|
744 | }
|
---|
745 | }
|
---|
746 |
|
---|
747 | void doSameUrlButDifferentZoomBounds() {
|
---|
748 | myprintln("*** Same URL, but different zoom bounds: ***");
|
---|
749 | for (String url : eliUrls.keySet()) {
|
---|
750 | JsonObject e = eliUrls.get(url);
|
---|
751 | if (!josmUrls.containsKey(url)) continue;
|
---|
752 | ImageryInfo j = josmUrls.get(url);
|
---|
753 |
|
---|
754 | Integer eMinZoom = getMinZoom(e);
|
---|
755 | Integer jMinZoom = getMinZoom(j);
|
---|
756 | /* dont warn for entries copied from the base of the mirror */
|
---|
757 | if (eMinZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
|
---|
758 | jMinZoom = null;
|
---|
759 | if (!Objects.equals(eMinZoom, jMinZoom) && !(Objects.equals(eMinZoom, 0) && jMinZoom == null)) {
|
---|
760 | myprintln("* Minzoom differs ("+eMinZoom+" != "+jMinZoom+"): "+getDescription(j));
|
---|
761 | }
|
---|
762 | Integer eMaxZoom = getMaxZoom(e);
|
---|
763 | Integer jMaxZoom = getMaxZoom(j);
|
---|
764 | /* dont warn for entries copied from the base of the mirror */
|
---|
765 | if (eMaxZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
|
---|
766 | jMaxZoom = null;
|
---|
767 | if (!Objects.equals(eMaxZoom, jMaxZoom)) {
|
---|
768 | myprintln("* Maxzoom differs ("+eMaxZoom+" != "+jMaxZoom+"): "+getDescription(j));
|
---|
769 | }
|
---|
770 | }
|
---|
771 | }
|
---|
772 |
|
---|
773 | void doSameUrlButDifferentCountryCode() {
|
---|
774 | myprintln("*** Same URL, but different country code: ***");
|
---|
775 | for (String url : eliUrls.keySet()) {
|
---|
776 | JsonObject e = eliUrls.get(url);
|
---|
777 | if (!josmUrls.containsKey(url)) continue;
|
---|
778 | ImageryInfo j = josmUrls.get(url);
|
---|
779 | String cce = getCountryCode(e);
|
---|
780 | if ("ZZ".equals(cce)) { /* special ELI country code */
|
---|
781 | cce = null;
|
---|
782 | }
|
---|
783 | if (cce != null && !cce.equals(getCountryCode(j))) {
|
---|
784 | myprintln("* Country code differs ("+getCountryCode(e)+" != "+getCountryCode(j)+"): "+getDescription(j));
|
---|
785 | }
|
---|
786 | }
|
---|
787 | }
|
---|
788 |
|
---|
789 | void doSameUrlButDifferentQuality() {
|
---|
790 | myprintln("*** Same URL, but different quality: ***");
|
---|
791 | for (String url : eliUrls.keySet()) {
|
---|
792 | JsonObject e = eliUrls.get(url);
|
---|
793 | if (!josmUrls.containsKey(url)) {
|
---|
794 | String q = getQuality(e);
|
---|
795 | if ("eli-best".equals(q)) {
|
---|
796 | myprintln("- Quality best entry not in JOSM for "+getDescription(e));
|
---|
797 | }
|
---|
798 | continue;
|
---|
799 | }
|
---|
800 | ImageryInfo j = josmUrls.get(url);
|
---|
801 | if (!Objects.equals(getQuality(e), getQuality(j))) {
|
---|
802 | myprintln("* Quality differs ("+getQuality(e)+" != "+getQuality(j)+"): "+getDescription(j));
|
---|
803 | }
|
---|
804 | }
|
---|
805 | }
|
---|
806 |
|
---|
807 | void doSameUrlButDifferentDates() {
|
---|
808 | myprintln("*** Same URL, but different dates: ***");
|
---|
809 | Pattern pattern = Pattern.compile("^(.*;)(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?$");
|
---|
810 | for (String url : eliUrls.keySet()) {
|
---|
811 | String ed = getDate(eliUrls.get(url));
|
---|
812 | if (!josmUrls.containsKey(url)) continue;
|
---|
813 | ImageryInfo j = josmUrls.get(url);
|
---|
814 | String jd = getDate(j);
|
---|
815 | // The forms 2015;- or -;2015 or 2015;2015 are handled equal to 2015
|
---|
816 | String ef = ed.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
|
---|
817 | // ELI has a strange and inconsistent used end_date definition, so we try again with subtraction by one
|
---|
818 | String ed2 = ed;
|
---|
819 | Matcher m = pattern.matcher(ed);
|
---|
820 | if (m.matches()) {
|
---|
821 | Calendar cal = Calendar.getInstance();
|
---|
822 | cal.set(Integer.parseInt(m.group(2)),
|
---|
823 | m.group(4) == null ? 0 : Integer.parseInt(m.group(4))-1,
|
---|
824 | m.group(6) == null ? 1 : Integer.parseInt(m.group(6)));
|
---|
825 | cal.add(Calendar.DAY_OF_MONTH, -1);
|
---|
826 | ed2 = m.group(1) + cal.get(Calendar.YEAR);
|
---|
827 | if (m.group(4) != null)
|
---|
828 | ed2 += "-" + String.format("%02d", cal.get(Calendar.MONTH)+1);
|
---|
829 | if (m.group(6) != null)
|
---|
830 | ed2 += "-" + String.format("%02d", cal.get(Calendar.DAY_OF_MONTH));
|
---|
831 | }
|
---|
832 | String ef2 = ed2.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
|
---|
833 | if (!ed.equals(jd) && !ef.equals(jd) && !ed2.equals(jd) && !ef2.equals(jd)) {
|
---|
834 | String t = "'"+ed+"'";
|
---|
835 | if (!ed.equals(ef)) {
|
---|
836 | t += " or '"+ef+"'";
|
---|
837 | }
|
---|
838 | if (jd.isEmpty()) {
|
---|
839 | myprintln("- Missing JOSM date ("+t+"): "+getDescription(j));
|
---|
840 | } else if (!ed.isEmpty()) {
|
---|
841 | myprintln("* Date differs ('"+t+"' != '"+jd+"'): "+getDescription(j));
|
---|
842 | } else if (!optionNoEli) {
|
---|
843 | myprintln("+ Missing ELI date ('"+jd+"'): "+getDescription(j));
|
---|
844 | }
|
---|
845 | }
|
---|
846 | }
|
---|
847 | }
|
---|
848 |
|
---|
849 | void doSameUrlButDifferentInformation() {
|
---|
850 | myprintln("*** Same URL, but different information: ***");
|
---|
851 | for (String url : eliUrls.keySet()) {
|
---|
852 | if (!josmUrls.containsKey(url)) continue;
|
---|
853 | JsonObject e = eliUrls.get(url);
|
---|
854 | ImageryInfo j = josmUrls.get(url);
|
---|
855 |
|
---|
856 | compareDescriptions(e, j);
|
---|
857 | comparePrivacyPolicyUrls(e, j);
|
---|
858 | comparePermissionReferenceUrls(e, j);
|
---|
859 | compareAttributionUrls(e, j);
|
---|
860 | compareAttributionTexts(e, j);
|
---|
861 | compareProjections(e, j);
|
---|
862 | compareDefaults(e, j);
|
---|
863 | compareOverlays(e, j);
|
---|
864 | compareNoTileHeaders(e, j);
|
---|
865 | }
|
---|
866 | }
|
---|
867 |
|
---|
868 | void compareDescriptions(JsonObject e, ImageryInfo j) {
|
---|
869 | String et = getDescriptions(e).getOrDefault("en", "");
|
---|
870 | String jt = getDescriptions(j).getOrDefault("en", "");
|
---|
871 | if (!et.equals(jt)) {
|
---|
872 | if (jt.isEmpty()) {
|
---|
873 | myprintln("- Missing JOSM description ("+et+"): "+getDescription(j));
|
---|
874 | } else if (!et.isEmpty()) {
|
---|
875 | myprintln("* Description differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
|
---|
876 | } else if (!optionNoEli) {
|
---|
877 | myprintln("+ Missing ELI description ('"+jt+"'): "+getDescription(j));
|
---|
878 | }
|
---|
879 | }
|
---|
880 | }
|
---|
881 |
|
---|
882 | void comparePrivacyPolicyUrls(JsonObject e, ImageryInfo j) {
|
---|
883 | String et = getPrivacyPolicyUrl(e);
|
---|
884 | String jt = getPrivacyPolicyUrl(j);
|
---|
885 | if (!Objects.equals(et, jt)) {
|
---|
886 | if (isBlank(jt)) {
|
---|
887 | myprintln("- Missing JOSM privacy policy URL ("+et+"): "+getDescription(j));
|
---|
888 | } else if (isNotBlank(et)) {
|
---|
889 | myprintln("* Privacy policy URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
|
---|
890 | } else if (!optionNoEli) {
|
---|
891 | myprintln("+ Missing ELI privacy policy URL ('"+jt+"'): "+getDescription(j));
|
---|
892 | }
|
---|
893 | }
|
---|
894 | }
|
---|
895 |
|
---|
896 | void comparePermissionReferenceUrls(JsonObject e, ImageryInfo j) {
|
---|
897 | String et = getPermissionReferenceUrl(e);
|
---|
898 | String jt = getPermissionReferenceUrl(j);
|
---|
899 | String jt2 = getTermsOfUseUrl(j);
|
---|
900 | if (isBlank(jt)) jt = jt2;
|
---|
901 | if (!Objects.equals(et, jt)) {
|
---|
902 | if (isBlank(jt)) {
|
---|
903 | myprintln("- Missing JOSM license URL ("+et+"): "+getDescription(j));
|
---|
904 | } else if (isNotBlank(et)) {
|
---|
905 | String ethttps = et.replace("http:", "https:");
|
---|
906 | if (isBlank(jt2) || !(jt2.equals(ethttps) || jt2.equals(et+"/") || jt2.equals(ethttps+"/"))) {
|
---|
907 | if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
|
---|
908 | myprintln("+ License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
|
---|
909 | } else {
|
---|
910 | String ja = getAttributionUrl(j);
|
---|
911 | if (ja != null && (ja.equals(et) || ja.equals(ethttps) || ja.equals(et+"/") || ja.equals(ethttps+"/"))) {
|
---|
912 | myprintln("+ ELI License URL in JOSM Attribution: "+getDescription(j));
|
---|
913 | } else {
|
---|
914 | myprintln("* License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
|
---|
915 | }
|
---|
916 | }
|
---|
917 | }
|
---|
918 | } else if (!optionNoEli) {
|
---|
919 | myprintln("+ Missing ELI license URL ('"+jt+"'): "+getDescription(j));
|
---|
920 | }
|
---|
921 | }
|
---|
922 | }
|
---|
923 |
|
---|
924 | void compareAttributionUrls(JsonObject e, ImageryInfo j) {
|
---|
925 | String et = getAttributionUrl(e);
|
---|
926 | String jt = getAttributionUrl(j);
|
---|
927 | if (!Objects.equals(et, jt)) {
|
---|
928 | if (isBlank(jt)) {
|
---|
929 | myprintln("- Missing JOSM attribution URL ("+et+"): "+getDescription(j));
|
---|
930 | } else if (isNotBlank(et)) {
|
---|
931 | String ethttps = et.replace("http:", "https:");
|
---|
932 | if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
|
---|
933 | myprintln("+ Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
|
---|
934 | } else {
|
---|
935 | myprintln("* Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
|
---|
936 | }
|
---|
937 | } else if (!optionNoEli) {
|
---|
938 | myprintln("+ Missing ELI attribution URL ('"+jt+"'): "+getDescription(j));
|
---|
939 | }
|
---|
940 | }
|
---|
941 | }
|
---|
942 |
|
---|
943 | void compareAttributionTexts(JsonObject e, ImageryInfo j) {
|
---|
944 | String et = getAttributionText(e);
|
---|
945 | String jt = getAttributionText(j);
|
---|
946 | if (!Objects.equals(et, jt)) {
|
---|
947 | if (isBlank(jt)) {
|
---|
948 | myprintln("- Missing JOSM attribution text ("+et+"): "+getDescription(j));
|
---|
949 | } else if (isNotBlank(et)) {
|
---|
950 | myprintln("* Attribution text differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
|
---|
951 | } else if (!optionNoEli) {
|
---|
952 | myprintln("+ Missing ELI attribution text ('"+jt+"'): "+getDescription(j));
|
---|
953 | }
|
---|
954 | }
|
---|
955 | }
|
---|
956 |
|
---|
957 | void compareProjections(JsonObject e, ImageryInfo j) {
|
---|
958 | String et = getProjections(e).stream().sorted().collect(Collectors.joining(" "));
|
---|
959 | String jt = getProjections(j).stream().sorted().collect(Collectors.joining(" "));
|
---|
960 | if (!Objects.equals(et, jt)) {
|
---|
961 | if (isBlank(jt)) {
|
---|
962 | String t = getType(e);
|
---|
963 | if ("wms_endpoint".equals(t) || "tms".equals(t)) {
|
---|
964 | myprintln("+ ELI projections for type "+t+": "+getDescription(j));
|
---|
965 | } else {
|
---|
966 | myprintln("- Missing JOSM projections ("+et+"): "+getDescription(j));
|
---|
967 | }
|
---|
968 | } else if (isNotBlank(et)) {
|
---|
969 | if ("EPSG:3857 EPSG:4326".equals(et) || "EPSG:3857".equals(et) || "EPSG:4326".equals(et)) {
|
---|
970 | myprintln("+ ELI has minimal projections ('"+et+"' != '"+jt+"'): "+getDescription(j));
|
---|
971 | } else {
|
---|
972 | myprintln("* Projections differ ('"+et+"' != '"+jt+"'): "+getDescription(j));
|
---|
973 | }
|
---|
974 | } else if (!optionNoEli && !"tms".equals(getType(e))) {
|
---|
975 | myprintln("+ Missing ELI projections ('"+jt+"'): "+getDescription(j));
|
---|
976 | }
|
---|
977 | }
|
---|
978 | }
|
---|
979 |
|
---|
980 | void compareDefaults(JsonObject e, ImageryInfo j) {
|
---|
981 | boolean ed = getDefault(e);
|
---|
982 | boolean jd = getDefault(j);
|
---|
983 | if (ed != jd) {
|
---|
984 | if (!jd) {
|
---|
985 | myprintln("- Missing JOSM default: "+getDescription(j));
|
---|
986 | } else if (!optionNoEli) {
|
---|
987 | myprintln("+ Missing ELI default: "+getDescription(j));
|
---|
988 | }
|
---|
989 | }
|
---|
990 | }
|
---|
991 |
|
---|
992 | void compareOverlays(JsonObject e, ImageryInfo j) {
|
---|
993 | boolean eo = getOverlay(e);
|
---|
994 | boolean jo = getOverlay(j);
|
---|
995 | if (eo != jo) {
|
---|
996 | if (!jo) {
|
---|
997 | myprintln("- Missing JOSM overlay flag: "+getDescription(j));
|
---|
998 | } else if (!optionNoEli) {
|
---|
999 | myprintln("+ Missing ELI overlay flag: "+getDescription(j));
|
---|
1000 | }
|
---|
1001 | }
|
---|
1002 | }
|
---|
1003 |
|
---|
1004 | void compareNoTileHeaders(JsonObject e, ImageryInfo j) {
|
---|
1005 | Map<String, Set<String>> eh = getNoTileHeader(e);
|
---|
1006 | Map<String, Set<String>> jh = getNoTileHeader(j);
|
---|
1007 | if (!Objects.equals(eh, jh)) {
|
---|
1008 | if (Utils.isEmpty(jh)) {
|
---|
1009 | myprintln("- Missing JOSM no tile headers ("+eh+"): "+getDescription(j));
|
---|
1010 | } else if (!Utils.isEmpty(eh)) {
|
---|
1011 | myprintln("* No tile headers differ ('"+eh+"' != '"+jh+"'): "+getDescription(j));
|
---|
1012 | } else if (!optionNoEli) {
|
---|
1013 | myprintln("+ Missing ELI no tile headers ('"+jh+"'): "+getDescription(j));
|
---|
1014 | }
|
---|
1015 | }
|
---|
1016 | }
|
---|
1017 |
|
---|
1018 | void doMismatchingShapes() {
|
---|
1019 | myprintln("*** Mismatching shapes: ***");
|
---|
1020 | for (String url : josmUrls.keySet()) {
|
---|
1021 | ImageryInfo j = josmUrls.get(url);
|
---|
1022 | int num = 1;
|
---|
1023 | for (Shape shape : getShapes(j)) {
|
---|
1024 | List<Coordinate> p = shape.getPoints();
|
---|
1025 | if (!p.get(0).equals(p.get(p.size()-1))) {
|
---|
1026 | myprintln("~~~ JOSM shape "+num+" unclosed: "+getDescription(j));
|
---|
1027 | }
|
---|
1028 | for (int nump = 1; nump < p.size(); ++nump) {
|
---|
1029 | if (Objects.equals(p.get(nump-1), p.get(nump))) {
|
---|
1030 | myprintln("~~~ JOSM shape "+num+" double point at "+(nump-1)+": "+getDescription(j));
|
---|
1031 | }
|
---|
1032 | }
|
---|
1033 | ++num;
|
---|
1034 | }
|
---|
1035 | }
|
---|
1036 | for (String url : eliUrls.keySet()) {
|
---|
1037 | JsonObject e = eliUrls.get(url);
|
---|
1038 | int num = 1;
|
---|
1039 | List<Shape> s = null;
|
---|
1040 | try {
|
---|
1041 | s = getShapes(e);
|
---|
1042 | for (Shape shape : s) {
|
---|
1043 | List<Coordinate> p = shape.getPoints();
|
---|
1044 | if (!p.get(0).equals(p.get(p.size()-1)) && !optionNoEli) {
|
---|
1045 | myprintln("+++ ELI shape "+num+" unclosed: "+getDescription(e));
|
---|
1046 | }
|
---|
1047 | for (int nump = 1; nump < p.size(); ++nump) {
|
---|
1048 | if (Objects.equals(p.get(nump-1), p.get(nump))) {
|
---|
1049 | myprintln("+++ ELI shape "+num+" double point at "+(nump-1)+": "+getDescription(e));
|
---|
1050 | }
|
---|
1051 | }
|
---|
1052 | ++num;
|
---|
1053 | }
|
---|
1054 | } catch (IllegalArgumentException err) {
|
---|
1055 | String desc = getDescription(e);
|
---|
1056 | myprintln("+++ ELI shape contains invalid data for "+desc+": "+err.getMessage());
|
---|
1057 | }
|
---|
1058 | if (s == null || !josmUrls.containsKey(url)) {
|
---|
1059 | continue;
|
---|
1060 | }
|
---|
1061 | ImageryInfo j = josmUrls.get(url);
|
---|
1062 | List<Shape> js = getShapes(j);
|
---|
1063 | if (s.isEmpty() && !js.isEmpty()) {
|
---|
1064 | if (!optionNoEli) {
|
---|
1065 | myprintln("+ No ELI shape: "+getDescription(j));
|
---|
1066 | }
|
---|
1067 | } else if (js.isEmpty() && !s.isEmpty()) {
|
---|
1068 | // don't report boundary like 5 point shapes as difference
|
---|
1069 | if (s.size() != 1 || s.get(0).getPoints().size() != 5) {
|
---|
1070 | myprintln("- No JOSM shape: "+getDescription(j));
|
---|
1071 | }
|
---|
1072 | } else if (s.size() != js.size()) {
|
---|
1073 | myprintln("* Different number of shapes ("+s.size()+" != "+js.size()+"): "+getDescription(j));
|
---|
1074 | } else {
|
---|
1075 | boolean[] edone = new boolean[s.size()];
|
---|
1076 | boolean[] jdone = new boolean[js.size()];
|
---|
1077 | for (int enums = 0; enums < s.size(); ++enums) {
|
---|
1078 | List<Coordinate> ep = s.get(enums).getPoints();
|
---|
1079 | for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
|
---|
1080 | List<Coordinate> jp = js.get(jnums).getPoints();
|
---|
1081 | if (ep.size() == jp.size() && !jdone[jnums]) {
|
---|
1082 | boolean err = false;
|
---|
1083 | for (int nump = 0; nump < ep.size() && !err; ++nump) {
|
---|
1084 | Coordinate ept = ep.get(nump);
|
---|
1085 | Coordinate jpt = jp.get(nump);
|
---|
1086 | if (differentCoordinate(ept.getLat(), jpt.getLat()) || differentCoordinate(ept.getLon(), jpt.getLon()))
|
---|
1087 | err = true;
|
---|
1088 | }
|
---|
1089 | if (!err) {
|
---|
1090 | edone[enums] = true;
|
---|
1091 | jdone[jnums] = true;
|
---|
1092 | break;
|
---|
1093 | }
|
---|
1094 | }
|
---|
1095 | }
|
---|
1096 | }
|
---|
1097 | for (int enums = 0; enums < s.size(); ++enums) {
|
---|
1098 | List<Coordinate> ep = s.get(enums).getPoints();
|
---|
1099 | for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
|
---|
1100 | List<Coordinate> jp = js.get(jnums).getPoints();
|
---|
1101 | if (ep.size() == jp.size() && !jdone[jnums]) {
|
---|
1102 | boolean err = false;
|
---|
1103 | for (int nump = 0; nump < ep.size() && !err; ++nump) {
|
---|
1104 | Coordinate ept = ep.get(nump);
|
---|
1105 | Coordinate jpt = jp.get(nump);
|
---|
1106 | if (differentCoordinate(ept.getLat(), jpt.getLat()) || differentCoordinate(ept.getLon(), jpt.getLon())) {
|
---|
1107 | String numtxt = Integer.toString(enums+1);
|
---|
1108 | if (enums != jnums) {
|
---|
1109 | numtxt += '/' + Integer.toString(jnums+1);
|
---|
1110 | }
|
---|
1111 | myprintln("* Different coordinate for point "+(nump+1)+" of shape "+numtxt+": "+getDescription(j));
|
---|
1112 | break;
|
---|
1113 | }
|
---|
1114 | }
|
---|
1115 | edone[enums] = true;
|
---|
1116 | jdone[jnums] = true;
|
---|
1117 | break;
|
---|
1118 | }
|
---|
1119 | }
|
---|
1120 | }
|
---|
1121 | for (int enums = 0; enums < s.size(); ++enums) {
|
---|
1122 | List<Coordinate> ep = s.get(enums).getPoints();
|
---|
1123 | for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
|
---|
1124 | List<Coordinate> jp = js.get(jnums).getPoints();
|
---|
1125 | if (!jdone[jnums]) {
|
---|
1126 | String numtxt = Integer.toString(enums+1);
|
---|
1127 | if (enums != jnums) {
|
---|
1128 | numtxt += '/' + Integer.toString(jnums+1);
|
---|
1129 | }
|
---|
1130 | myprintln("* Different number of points for shape "+numtxt+" ("+ep.size()+" ! = "+jp.size()+"): "
|
---|
1131 | + getDescription(j));
|
---|
1132 | edone[enums] = true;
|
---|
1133 | jdone[jnums] = true;
|
---|
1134 | break;
|
---|
1135 | }
|
---|
1136 | }
|
---|
1137 | }
|
---|
1138 | }
|
---|
1139 | }
|
---|
1140 | }
|
---|
1141 |
|
---|
1142 | private boolean differentCoordinate(double v1, double v2) {
|
---|
1143 | double epsilon = 0.00001;
|
---|
1144 | return Math.abs(v1 - v2) > epsilon;
|
---|
1145 | }
|
---|
1146 |
|
---|
1147 | void doMismatchingIcons() {
|
---|
1148 | myprintln("*** Mismatching icons: ***");
|
---|
1149 | doMismatching(this::compareIcons);
|
---|
1150 | }
|
---|
1151 |
|
---|
1152 | void doMismatchingCategories() {
|
---|
1153 | myprintln("*** Mismatching categories: ***");
|
---|
1154 | doMismatching(this::compareCategories);
|
---|
1155 | }
|
---|
1156 |
|
---|
1157 | void doMismatching(BiConsumer<ImageryInfo, JsonObject> comparator) {
|
---|
1158 | for (String url : eliUrls.keySet()) {
|
---|
1159 | if (josmUrls.containsKey(url)) {
|
---|
1160 | comparator.accept(josmUrls.get(url), eliUrls.get(url));
|
---|
1161 | }
|
---|
1162 | }
|
---|
1163 | }
|
---|
1164 |
|
---|
1165 | void compareIcons(ImageryInfo j, JsonObject e) {
|
---|
1166 | String ij = getIcon(j);
|
---|
1167 | String ie = getIcon(e);
|
---|
1168 | boolean ijok = isNotBlank(ij);
|
---|
1169 | boolean ieok = isNotBlank(ie);
|
---|
1170 | if (ijok && !ieok) {
|
---|
1171 | if (!optionNoEli) {
|
---|
1172 | myprintln("+ No ELI icon: "+getDescription(j));
|
---|
1173 | }
|
---|
1174 | } else if (!ijok && ieok) {
|
---|
1175 | myprintln("- No JOSM icon: "+getDescription(j));
|
---|
1176 | } else if (ijok && ieok && !Objects.equals(ij, ie) && !(
|
---|
1177 | (ie.startsWith("https://osmlab.github.io/editor-layer-index/")
|
---|
1178 | || ie.startsWith("https://raw.githubusercontent.com/osmlab/editor-layer-index/")) &&
|
---|
1179 | ij.startsWith("data:"))) {
|
---|
1180 | String iehttps = ie.replace("http:", "https:");
|
---|
1181 | if (ij.equals(iehttps)) {
|
---|
1182 | myprintln("+ Different icons: "+getDescription(j));
|
---|
1183 | } else {
|
---|
1184 | myprintln("* Different icons: "+getDescription(j));
|
---|
1185 | }
|
---|
1186 | }
|
---|
1187 | }
|
---|
1188 |
|
---|
1189 | void compareCategories(ImageryInfo j, JsonObject e) {
|
---|
1190 | String cj = getCategory(j);
|
---|
1191 | String ce = getCategory(e);
|
---|
1192 | boolean cjok = isNotBlank(cj);
|
---|
1193 | boolean ceok = isNotBlank(ce);
|
---|
1194 | if (cjok && !ceok) {
|
---|
1195 | if (!optionNoEli) {
|
---|
1196 | myprintln("+ No ELI category: "+getDescription(j));
|
---|
1197 | }
|
---|
1198 | } else if (!cjok && ceok) {
|
---|
1199 | myprintln("- No JOSM category: "+getDescription(j));
|
---|
1200 | } else if (cjok && ceok && !Objects.equals(cj, ce)) {
|
---|
1201 | myprintln("* Different categories ('"+ce+"' != '"+cj+"'): "+getDescription(j));
|
---|
1202 | }
|
---|
1203 | }
|
---|
1204 |
|
---|
1205 | void doMiscellaneousChecks() {
|
---|
1206 | myprintln("*** Miscellaneous checks: ***");
|
---|
1207 | Map<String, ImageryInfo> josmIds = new HashMap<>();
|
---|
1208 | Collection<String> all = Projections.getAllProjectionCodes();
|
---|
1209 | DomainValidator dv = DomainValidator.getInstance();
|
---|
1210 | for (String url : josmUrls.keySet()) {
|
---|
1211 | ImageryInfo j = josmUrls.get(url);
|
---|
1212 | String id = getId(j);
|
---|
1213 | if ("wms".equals(getType(j))) {
|
---|
1214 | String urlLc = url.toLowerCase(Locale.ENGLISH);
|
---|
1215 | if (getProjections(j).isEmpty()) {
|
---|
1216 | myprintln("~ WMS without projections: "+getDescription(j));
|
---|
1217 | } else {
|
---|
1218 | List<String> unsupported = new LinkedList<>();
|
---|
1219 | List<String> old = new LinkedList<>();
|
---|
1220 | for (String p : getProjectionsUnstripped(j)) {
|
---|
1221 | if ("CRS:84".equals(p)) {
|
---|
1222 | if (!urlLc.contains("version=1.3")) {
|
---|
1223 | myprintln("~ CRS:84 without WMS 1.3: "+getDescription(j));
|
---|
1224 | }
|
---|
1225 | } else if (oldproj.containsKey(p)) {
|
---|
1226 | old.add(p);
|
---|
1227 | } else if (!all.contains(p) && !ignoreproj.contains(p)) {
|
---|
1228 | unsupported.add(p);
|
---|
1229 | }
|
---|
1230 | }
|
---|
1231 | if (!unsupported.isEmpty()) {
|
---|
1232 | myprintln("~ Projections "+String.join(", ", unsupported)+" not supported by JOSM: "+getDescription(j));
|
---|
1233 | }
|
---|
1234 | for (String o : old) {
|
---|
1235 | myprintln("~ Projection "+o+" is an old unsupported code and has been replaced by "+oldproj.get(o)+": "
|
---|
1236 | + getDescription(j));
|
---|
1237 | }
|
---|
1238 | }
|
---|
1239 | if (urlLc.contains("version=1.3") && !urlLc.contains("crs={proj}")) {
|
---|
1240 | myprintln("~ WMS 1.3 with strange CRS specification: "+getDescription(j));
|
---|
1241 | } else if (urlLc.contains("version=1.1") && !urlLc.contains("srs={proj}")) {
|
---|
1242 | myprintln("~ WMS 1.1 with strange SRS specification: "+getDescription(j));
|
---|
1243 | }
|
---|
1244 | }
|
---|
1245 | List<String> urls = new LinkedList<>();
|
---|
1246 | if (!"scanex".equals(getType(j))) {
|
---|
1247 | urls.add(url);
|
---|
1248 | }
|
---|
1249 | String jt = getPermissionReferenceUrl(j);
|
---|
1250 | if (isNotBlank(jt) && !"Public Domain".equalsIgnoreCase(jt))
|
---|
1251 | urls.add(jt);
|
---|
1252 | jt = getTermsOfUseUrl(j);
|
---|
1253 | if (isNotBlank(jt))
|
---|
1254 | urls.add(jt);
|
---|
1255 | jt = getAttributionUrl(j);
|
---|
1256 | if (isNotBlank(jt))
|
---|
1257 | urls.add(jt);
|
---|
1258 | jt = getIcon(j);
|
---|
1259 | if (isNotBlank(jt)) {
|
---|
1260 | if (!jt.startsWith("data:image/"))
|
---|
1261 | urls.add(jt);
|
---|
1262 | else {
|
---|
1263 | try {
|
---|
1264 | new ImageProvider(jt).get();
|
---|
1265 | } catch (RuntimeException e) {
|
---|
1266 | myprintln("~ Strange Icon: "+getDescription(j));
|
---|
1267 | }
|
---|
1268 | }
|
---|
1269 | }
|
---|
1270 | Pattern patternU = Pattern.compile("^https?://([^/]+?)(:\\d+)?(/.*)?");
|
---|
1271 | for (String u : urls) {
|
---|
1272 | if (!patternU.matcher(u).matches() || u.matches(".*[ \t]+$")) {
|
---|
1273 | myprintln("~ Strange URL '"+u+"': "+getDescription(j));
|
---|
1274 | } else {
|
---|
1275 | try {
|
---|
1276 | URL jurl = new URL(u.replaceAll("\\{switch:[^}]*}", "x"));
|
---|
1277 | String domain = jurl.getHost();
|
---|
1278 | int port = jurl.getPort();
|
---|
1279 | if (!(domain.matches("^\\d+\\.\\d+\\.\\d+\\.\\d+$")) && !dv.isValid(domain))
|
---|
1280 | myprintln("~ Strange Domain '"+domain+"': "+getDescription(j));
|
---|
1281 | else if (80 == port || 443 == port) {
|
---|
1282 | myprintln("~ Useless port '"+port+"': "+getDescription(j));
|
---|
1283 | }
|
---|
1284 | } catch (MalformedURLException e) {
|
---|
1285 | myprintln("~ Malformed URL '"+u+"': "+getDescription(j)+" => "+e.getMessage());
|
---|
1286 | }
|
---|
1287 | }
|
---|
1288 | }
|
---|
1289 |
|
---|
1290 | if (josmMirrors.containsKey(url)) {
|
---|
1291 | continue;
|
---|
1292 | }
|
---|
1293 | if (isBlank(id)) {
|
---|
1294 | myprintln("~ No JOSM-ID: "+getDescription(j));
|
---|
1295 | } else if (josmIds.containsKey(id)) {
|
---|
1296 | myprintln("~ JOSM-ID "+id+" not unique: "+getDescription(j));
|
---|
1297 | } else {
|
---|
1298 | josmIds.put(id, j);
|
---|
1299 | }
|
---|
1300 | String d = getDate(j);
|
---|
1301 | if (isNotBlank(d)) {
|
---|
1302 | Pattern patternD = Pattern.compile("^(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?)(;(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?))?$");
|
---|
1303 | Matcher m = patternD.matcher(d);
|
---|
1304 | if (!m.matches()) {
|
---|
1305 | myprintln("~ JOSM-Date '"+d+"' is strange: "+getDescription(j));
|
---|
1306 | } else {
|
---|
1307 | try {
|
---|
1308 | Date first = verifyDate(m.group(2), m.group(4), m.group(6));
|
---|
1309 | Date second = verifyDate(m.group(9), m.group(11), m.group(13));
|
---|
1310 | if (second.compareTo(first) < 0) {
|
---|
1311 | myprintln("~ JOSM-Date '"+d+"' is strange (second earlier than first): "+getDescription(j));
|
---|
1312 | }
|
---|
1313 | } catch (Exception e) {
|
---|
1314 | myprintln("~ JOSM-Date '"+d+"' is strange ("+e.getMessage()+"): "+getDescription(j));
|
---|
1315 | }
|
---|
1316 | }
|
---|
1317 | }
|
---|
1318 | if (isNotBlank(getAttributionUrl(j)) && isBlank(getAttributionText(j))) {
|
---|
1319 | myprintln("~ Attribution link without text: "+getDescription(j));
|
---|
1320 | }
|
---|
1321 | if (isNotBlank(getLogoUrl(j)) && isBlank(getLogoImage(j))) {
|
---|
1322 | myprintln("~ Logo link without image: "+getDescription(j));
|
---|
1323 | }
|
---|
1324 | if (isNotBlank(getTermsOfUseText(j)) && isBlank(getTermsOfUseUrl(j))) {
|
---|
1325 | myprintln("~ Terms of Use text without link: "+getDescription(j));
|
---|
1326 | }
|
---|
1327 | List<Shape> js = getShapes(j);
|
---|
1328 | if (!js.isEmpty()) {
|
---|
1329 | double minlat = 1000;
|
---|
1330 | double minlon = 1000;
|
---|
1331 | double maxlat = -1000;
|
---|
1332 | double maxlon = -1000;
|
---|
1333 | for (Shape s: js) {
|
---|
1334 | for (Coordinate p: s.getPoints()) {
|
---|
1335 | double lat = p.getLat();
|
---|
1336 | double lon = p.getLon();
|
---|
1337 | if (lat > maxlat) maxlat = lat;
|
---|
1338 | if (lon > maxlon) maxlon = lon;
|
---|
1339 | if (lat < minlat) minlat = lat;
|
---|
1340 | if (lon < minlon) minlon = lon;
|
---|
1341 | }
|
---|
1342 | }
|
---|
1343 | ImageryBounds b = j.getBounds();
|
---|
1344 | if (differentCoordinate(b.getMinLat(), minlat)
|
---|
1345 | || differentCoordinate(b.getMinLon(), minlon)
|
---|
1346 | || differentCoordinate(b.getMaxLat(), maxlat)
|
---|
1347 | || differentCoordinate(b.getMaxLon(), maxlon)) {
|
---|
1348 | myprintln("~ Bounds do not match shape (is "+b.getMinLat()+","+b.getMinLon()+","+b.getMaxLat()+","+b.getMaxLon()
|
---|
1349 | + ", calculated <bounds min-lat='"+minlat+"' min-lon='"+minlon+"' max-lat='"+maxlat+"' max-lon='"+maxlon+"'>): "
|
---|
1350 | + getDescription(j));
|
---|
1351 | }
|
---|
1352 | }
|
---|
1353 | List<String> knownCategories = Arrays.asList(
|
---|
1354 | "photo", "elevation", "map", "historicmap", "osmbasedmap", "historicphoto", "qa", "other");
|
---|
1355 | String cat = getCategory(j);
|
---|
1356 | if (isBlank(cat)) {
|
---|
1357 | myprintln("~ No category: "+getDescription(j));
|
---|
1358 | } else if (!knownCategories.contains(cat)) {
|
---|
1359 | myprintln("~ Strange category "+cat+": "+getDescription(j));
|
---|
1360 | }
|
---|
1361 | }
|
---|
1362 | }
|
---|
1363 |
|
---|
1364 | /*
|
---|
1365 | * Utility functions that allow uniform access for both ImageryInfo and JsonObject.
|
---|
1366 | */
|
---|
1367 |
|
---|
1368 | static String getUrl(Object e) {
|
---|
1369 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).getUrl();
|
---|
1370 | JsonObject p = ((Map<String, JsonObject>) e).get("properties");
|
---|
1371 | if (p != null)
|
---|
1372 | return p.getString("url");
|
---|
1373 | else
|
---|
1374 | return ((JsonObject)e).getString("template");
|
---|
1375 | }
|
---|
1376 |
|
---|
1377 | static String getUrlStripped(Object e) {
|
---|
1378 | return getUrl(e).replaceAll("\\?(apikey|access_token)=.*", "");
|
---|
1379 | }
|
---|
1380 |
|
---|
1381 | static String getDate(Object e) {
|
---|
1382 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).getDate() != null ? ((ImageryInfo) e).getDate() : "";
|
---|
1383 | JsonObject p = ((Map<String, JsonObject>) e).get("properties");
|
---|
1384 | String start = p.containsKey("start_date") ? p.getString("start_date") : "";
|
---|
1385 | String end = p.containsKey("end_date") ? p.getString("end_date") : "";
|
---|
1386 | if (!start.isEmpty() && !end.isEmpty())
|
---|
1387 | return start+";"+end;
|
---|
1388 | else if (!start.isEmpty())
|
---|
1389 | return start+";-";
|
---|
1390 | else if (!end.isEmpty())
|
---|
1391 | return "-;"+end;
|
---|
1392 | return "";
|
---|
1393 | }
|
---|
1394 |
|
---|
1395 | static Date verifyDate(String year, String month, String day) throws ParseException {
|
---|
1396 | String date;
|
---|
1397 | if (year == null) {
|
---|
1398 | date = "3000-01-01";
|
---|
1399 | } else {
|
---|
1400 | date = year + "-" + (month == null ? "01" : month) + "-" + (day == null ? "01" : day);
|
---|
1401 | }
|
---|
1402 | SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");
|
---|
1403 | df.setLenient(false);
|
---|
1404 | return df.parse(date);
|
---|
1405 | }
|
---|
1406 |
|
---|
1407 | static String getId(Object e) {
|
---|
1408 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).getId();
|
---|
1409 | return ((Map<String, JsonObject>) e).get("properties").getString("id");
|
---|
1410 | }
|
---|
1411 |
|
---|
1412 | static String getName(Object e) {
|
---|
1413 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).getOriginalName();
|
---|
1414 | return ((Map<String, JsonObject>) e).get("properties").getString("name");
|
---|
1415 | }
|
---|
1416 |
|
---|
1417 | static List<ImageryInfo> getMirrors(Object e) {
|
---|
1418 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).getMirrors();
|
---|
1419 | return Collections.emptyList();
|
---|
1420 | }
|
---|
1421 |
|
---|
1422 | static List<String> getProjections(Object e) {
|
---|
1423 | List<String> r = new ArrayList<>();
|
---|
1424 | List<String> u = getProjectionsUnstripped(e);
|
---|
1425 | for (String p : u) {
|
---|
1426 | if (!oldproj.containsKey(p) && !("CRS:84".equals(p) && !(getUrlStripped(e).matches("(?i)version=1\\.3")))) {
|
---|
1427 | r.add(p);
|
---|
1428 | }
|
---|
1429 | }
|
---|
1430 | return r;
|
---|
1431 | }
|
---|
1432 |
|
---|
1433 | static List<String> getProjectionsUnstripped(Object e) {
|
---|
1434 | List<String> r = null;
|
---|
1435 | if (e instanceof ImageryInfo) {
|
---|
1436 | r = ((ImageryInfo) e).getServerProjections();
|
---|
1437 | } else {
|
---|
1438 | JsonValue s = ((Map<String, JsonObject>) e).get("properties").get("available_projections");
|
---|
1439 | if (s != null) {
|
---|
1440 | r = new ArrayList<>();
|
---|
1441 | for (JsonValue p : s.asJsonArray()) {
|
---|
1442 | r.add(((JsonString) p).getString());
|
---|
1443 | }
|
---|
1444 | }
|
---|
1445 | }
|
---|
1446 | return r != null ? r : Collections.emptyList();
|
---|
1447 | }
|
---|
1448 |
|
---|
1449 | static void addJsonShapes(List<Shape> l, JsonArray a) {
|
---|
1450 | if (a.get(0).asJsonArray().get(0) instanceof JsonArray) {
|
---|
1451 | for (JsonValue sub: a.asJsonArray()) {
|
---|
1452 | addJsonShapes(l, sub.asJsonArray());
|
---|
1453 | }
|
---|
1454 | } else {
|
---|
1455 | Shape s = new Shape();
|
---|
1456 | for (JsonValue point: a.asJsonArray()) {
|
---|
1457 | JsonArray ar = point.asJsonArray();
|
---|
1458 | String lon = ar.getJsonNumber(0).toString();
|
---|
1459 | String lat = ar.getJsonNumber(1).toString();
|
---|
1460 | s.addPoint(lat, lon);
|
---|
1461 | }
|
---|
1462 | l.add(s);
|
---|
1463 | }
|
---|
1464 | }
|
---|
1465 |
|
---|
1466 | static List<Shape> getShapes(Object e) {
|
---|
1467 | if (e instanceof ImageryInfo) {
|
---|
1468 | ImageryBounds bounds = ((ImageryInfo) e).getBounds();
|
---|
1469 | if (bounds != null) {
|
---|
1470 | return bounds.getShapes();
|
---|
1471 | }
|
---|
1472 | return Collections.emptyList();
|
---|
1473 | }
|
---|
1474 | JsonValue ex = ((Map<String, JsonValue>) e).get("geometry");
|
---|
1475 | if (ex != null && !JsonValue.NULL.equals(ex) && !ex.asJsonObject().isNull("coordinates")) {
|
---|
1476 | JsonArray poly = ex.asJsonObject().getJsonArray("coordinates");
|
---|
1477 | List<Shape> l = new ArrayList<>();
|
---|
1478 | for (JsonValue shapes: poly) {
|
---|
1479 | addJsonShapes(l, shapes.asJsonArray());
|
---|
1480 | }
|
---|
1481 | return l;
|
---|
1482 | }
|
---|
1483 | return Collections.emptyList();
|
---|
1484 | }
|
---|
1485 |
|
---|
1486 | static String getType(Object e) {
|
---|
1487 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).getImageryType().getTypeString();
|
---|
1488 | return ((Map<String, JsonObject>) e).get("properties").getString("type");
|
---|
1489 | }
|
---|
1490 |
|
---|
1491 | static Integer getMinZoom(Object e) {
|
---|
1492 | if (e instanceof ImageryInfo) {
|
---|
1493 | int mz = ((ImageryInfo) e).getMinZoom();
|
---|
1494 | return mz == 0 ? null : mz;
|
---|
1495 | } else {
|
---|
1496 | JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("min_zoom");
|
---|
1497 | if (num == null) return null;
|
---|
1498 | return num.intValue();
|
---|
1499 | }
|
---|
1500 | }
|
---|
1501 |
|
---|
1502 | static Integer getMaxZoom(Object e) {
|
---|
1503 | if (e instanceof ImageryInfo) {
|
---|
1504 | int mz = ((ImageryInfo) e).getMaxZoom();
|
---|
1505 | return mz == 0 ? null : mz;
|
---|
1506 | } else {
|
---|
1507 | JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("max_zoom");
|
---|
1508 | if (num == null) return null;
|
---|
1509 | return num.intValue();
|
---|
1510 | }
|
---|
1511 | }
|
---|
1512 |
|
---|
1513 | static String getCountryCode(Object e) {
|
---|
1514 | if (e instanceof ImageryInfo) return "".equals(((ImageryInfo) e).getCountryCode()) ? null : ((ImageryInfo) e).getCountryCode();
|
---|
1515 | return ((Map<String, JsonObject>) e).get("properties").getString("country_code", null);
|
---|
1516 | }
|
---|
1517 |
|
---|
1518 | static String getQuality(Object e) {
|
---|
1519 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).isBestMarked() ? "eli-best" : null;
|
---|
1520 | return (((Map<String, JsonObject>) e).get("properties").containsKey("best")
|
---|
1521 | && ((Map<String, JsonObject>) e).get("properties").getBoolean("best")) ? "eli-best" : null;
|
---|
1522 | }
|
---|
1523 |
|
---|
1524 | static boolean getOverlay(Object e) {
|
---|
1525 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).isOverlay();
|
---|
1526 | return (((Map<String, JsonObject>) e).get("properties").containsKey("overlay")
|
---|
1527 | && ((Map<String, JsonObject>) e).get("properties").getBoolean("overlay"));
|
---|
1528 | }
|
---|
1529 |
|
---|
1530 | static String getIcon(Object e) {
|
---|
1531 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).getIcon();
|
---|
1532 | return ((Map<String, JsonObject>) e).get("properties").getString("icon", null);
|
---|
1533 | }
|
---|
1534 |
|
---|
1535 | static String getAttributionText(Object e) {
|
---|
1536 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionText(0, null, null);
|
---|
1537 | try {
|
---|
1538 | return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("text", null);
|
---|
1539 | } catch (NullPointerException ex) {
|
---|
1540 | return null;
|
---|
1541 | }
|
---|
1542 | }
|
---|
1543 |
|
---|
1544 | static String getAttributionUrl(Object e) {
|
---|
1545 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionLinkURL();
|
---|
1546 | try {
|
---|
1547 | return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("url", null);
|
---|
1548 | } catch (NullPointerException ex) {
|
---|
1549 | return null;
|
---|
1550 | }
|
---|
1551 | }
|
---|
1552 |
|
---|
1553 | static String getTermsOfUseText(Object e) {
|
---|
1554 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseText();
|
---|
1555 | return null;
|
---|
1556 | }
|
---|
1557 |
|
---|
1558 | static String getTermsOfUseUrl(Object e) {
|
---|
1559 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseURL();
|
---|
1560 | return null;
|
---|
1561 | }
|
---|
1562 |
|
---|
1563 | static String getCategory(Object e) {
|
---|
1564 | if (e instanceof ImageryInfo) {
|
---|
1565 | return ((ImageryInfo) e).getImageryCategoryOriginalString();
|
---|
1566 | }
|
---|
1567 | return ((Map<String, JsonObject>) e).get("properties").getString("category", null);
|
---|
1568 | }
|
---|
1569 |
|
---|
1570 | static String getLogoImage(Object e) {
|
---|
1571 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageRaw();
|
---|
1572 | return null;
|
---|
1573 | }
|
---|
1574 |
|
---|
1575 | static String getLogoUrl(Object e) {
|
---|
1576 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageURL();
|
---|
1577 | return null;
|
---|
1578 | }
|
---|
1579 |
|
---|
1580 | static String getPermissionReferenceUrl(Object e) {
|
---|
1581 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).getPermissionReferenceURL();
|
---|
1582 | return ((Map<String, JsonObject>) e).get("properties").getString("license_url", null);
|
---|
1583 | }
|
---|
1584 |
|
---|
1585 | static String getPrivacyPolicyUrl(Object e) {
|
---|
1586 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).getPrivacyPolicyURL();
|
---|
1587 | return ((Map<String, JsonObject>) e).get("properties").getString("privacy_policy_url", null);
|
---|
1588 | }
|
---|
1589 |
|
---|
1590 | static Map<String, Set<String>> getNoTileHeader(Object e) {
|
---|
1591 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).getNoTileHeaders();
|
---|
1592 | JsonObject nth = ((Map<String, JsonObject>) e).get("properties").getJsonObject("no_tile_header");
|
---|
1593 | return nth == null ? null : nth.keySet().stream().collect(Collectors.toMap(
|
---|
1594 | Function.identity(),
|
---|
1595 | k -> nth.getJsonArray(k).stream().map(x -> ((JsonString) x).getString()).collect(Collectors.toSet())));
|
---|
1596 | }
|
---|
1597 |
|
---|
1598 | static Map<String, String> getDescriptions(Object e) {
|
---|
1599 | Map<String, String> res = new HashMap<>();
|
---|
1600 | if (e instanceof ImageryInfo) {
|
---|
1601 | String a = ((ImageryInfo) e).getDescription();
|
---|
1602 | if (a != null) res.put("en", a);
|
---|
1603 | } else {
|
---|
1604 | String a = ((Map<String, JsonObject>) e).get("properties").getString("description", null);
|
---|
1605 | if (a != null) res.put("en", a.replaceAll("''", "'"));
|
---|
1606 | }
|
---|
1607 | return res;
|
---|
1608 | }
|
---|
1609 |
|
---|
1610 | static boolean getValidGeoreference(Object e) {
|
---|
1611 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).isGeoreferenceValid();
|
---|
1612 | return false;
|
---|
1613 | }
|
---|
1614 |
|
---|
1615 | static boolean getDefault(Object e) {
|
---|
1616 | if (e instanceof ImageryInfo) return ((ImageryInfo) e).isDefaultEntry();
|
---|
1617 | return ((Map<String, JsonObject>) e).get("properties").getBoolean("default", false);
|
---|
1618 | }
|
---|
1619 |
|
---|
1620 | String getDescription(Object o) {
|
---|
1621 | String url = getUrl(o);
|
---|
1622 | String cc = getCountryCode(o);
|
---|
1623 | if (cc == null) {
|
---|
1624 | ImageryInfo j = josmUrls.get(url);
|
---|
1625 | if (j != null) cc = getCountryCode(j);
|
---|
1626 | if (cc == null) {
|
---|
1627 | JsonObject e = eliUrls.get(url);
|
---|
1628 | if (e != null) cc = getCountryCode(e);
|
---|
1629 | }
|
---|
1630 | }
|
---|
1631 | if (cc == null) {
|
---|
1632 | cc = "";
|
---|
1633 | } else {
|
---|
1634 | cc = "["+cc+"] ";
|
---|
1635 | }
|
---|
1636 | String name = getName(o);
|
---|
1637 | String id = getId(o);
|
---|
1638 | String d = cc;
|
---|
1639 | if (!Utils.isEmpty(name)) {
|
---|
1640 | d += name;
|
---|
1641 | if (!Utils.isEmpty(id))
|
---|
1642 | d += " ["+id+"]";
|
---|
1643 | } else if (!Utils.isEmpty(url))
|
---|
1644 | d += url;
|
---|
1645 | if (optionShorten) {
|
---|
1646 | if (d.length() > MAXLEN) d = d.substring(0, MAXLEN-1) + "...";
|
---|
1647 | }
|
---|
1648 | return d;
|
---|
1649 | }
|
---|
1650 | }
|
---|