source: josm/trunk/scripts/SyncEditorLayerIndex.groovy@ 11599

Last change on this file since 11599 was 11599, checked in by Klumbumbus, 7 years ago

see #12706, see #12313 - enable eli-best comparison script
  • Property svn:eol-style set to native
File size: 21.6 KB
RevLine 
[7726]1// License: GPL. For details, see LICENSE file.
2/**
3 * Compare and analyse the differences of the editor imagery index and the JOSM imagery list.
4 * The goal is to keep both lists in sync.
5 *
6 * The editor imagery index project (https://github.com/osmlab/editor-imagery-index)
7 * provides also a version in the JOSM format, but the JSON is the original source
8 * format, so we read that.
9 *
10 * How to run:
11 * -----------
12 *
13 * Main JOSM binary needs to be in classpath, e.g.
14 *
[11260]15 * $ groovy -cp ../dist/josm-custom.jar SyncEditorImageryIndex.groovy
[9667]16 *
[7726]17 * Add option "-h" to show the available command line flags.
18 */
19import javax.json.Json
20import javax.json.JsonArray
21import javax.json.JsonObject
22import javax.json.JsonReader
[11412]23import javax.json.JsonValue
[7726]24
[9953]25import org.openstreetmap.josm.data.imagery.ImageryInfo
[11410]26import org.openstreetmap.josm.data.imagery.Shape
[7726]27import org.openstreetmap.josm.io.imagery.ImageryReader
28
[9880]29class SyncEditorImageryIndex {
[7726]30
31 List<ImageryInfo> josmEntries;
[11582]32 JsonArray eliEntries;
[7726]33
[11582]34 def eliUrls = new HashMap<String, JsonObject>()
[7726]35 def josmUrls = new HashMap<String, ImageryInfo>()
[11420]36 def josmMirrors = new HashMap<String, ImageryInfo>()
[9667]37
[11582]38 static String eliInputFile = 'imagery.geojson'
[7726]39 static String josmInputFile = 'maps.xml'
[11238]40 static String ignoreInputFile = 'maps_ignores.txt'
[9505]41 static FileWriter outputFile = null
42 static BufferedWriter outputStream = null
[11582]43 def skip = [:]
[9505]44
[7726]45 static def options
[9658]46
[7726]47 /**
48 * Main method.
49 */
50 static main(def args) {
51 parse_command_line_arguments(args)
[9880]52 def script = new SyncEditorImageryIndex()
[9505]53 script.loadSkip()
[9658]54 script.start()
[7726]55 script.loadJosmEntries()
[11582]56 script.loadELIEntries()
[7726]57 script.checkInOneButNotTheOther()
58 script.checkCommonEntries()
[9658]59 script.end()
[9505]60 if(outputStream != null) {
61 outputStream.close();
62 }
63 if(outputFile != null) {
64 outputFile.close();
65 }
[7726]66 }
[9653]67
[7726]68 /**
69 * Parse command line arguments.
70 */
71 static void parse_command_line_arguments(args) {
[9658]72 def cli = new CliBuilder(width: 160)
[9505]73 cli.o(longOpt:'output', args:1, argName: "output", "Output file, - prints to stdout (default: -)")
[11582]74 cli.e(longOpt:'eli_input', args:1, argName:"eli_input", "Input file for the editor imagery index (json). Default is $eliInputFile (current directory).")
[9505]75 cli.j(longOpt:'josm_input', args:1, argName:"josm_input", "Input file for the JOSM imagery list (xml). Default is $josmInputFile (current directory).")
[11238]76 cli.i(longOpt:'ignore_input', args:1, argName:"ignore_input", "Input file for the ignore list. Default is $ignoreInputFile (current directory).")
[7726]77 cli.s(longOpt:'shorten', "shorten the output, so it is easier to read in a console window")
[9505]78 cli.n(longOpt:'noskip', argName:"noskip", "don't skip known entries")
[9658]79 cli.x(longOpt:'xhtmlbody', argName:"xhtmlbody", "create XHTML body for display in a web page")
80 cli.X(longOpt:'xhtml', argName:"xhtml", "create XHTML for display in a web page")
[11582]81 cli.m(longOpt:'nomissingeli', argName:"nomissingeli", "don't show missing editor imagery index entries")
[7726]82 cli.h(longOpt:'help', "show this help")
83 options = cli.parse(args)
84
85 if (options.h) {
86 cli.usage()
87 System.exit(0)
88 }
[11582]89 if (options.eli_input) {
90 eliInputFile = options.eli_input
[7726]91 }
92 if (options.josm_input) {
93 josmInputFile = options.josm_input
94 }
[11238]95 if (options.ignore_input) {
96 ignoreInputFile = options.ignore_input
97 }
[9505]98 if (options.output && options.output != "-") {
99 outputFile = new FileWriter(options.output)
100 outputStream = new BufferedWriter(outputFile)
101 }
[7726]102 }
103
[9505]104 void loadSkip() {
[11238]105 FileReader fr = new FileReader(ignoreInputFile)
106 def line
107
108 while((line = fr.readLine()) != null) {
[11582]109 def res = (line =~ /^\|\| *(ELI|Ignore) *\|\| *\{\{\{(.+)\}\}\} *\|\|/)
[11238]110 if(res.count)
111 {
[11582]112 if(res[0][1].equals("Ignore")) {
113 skip[res[0][2]] = "green"
[11238]114 } else {
[11582]115 skip[res[0][2]] = "darkgoldenrod"
[11238]116 }
117 }
[11234]118 }
[11238]119 }
[9653]120
[9658]121 void myprintlnfinal(String s) {
122 if(outputStream != null) {
123 outputStream.write(s);
124 outputStream.newLine();
125 } else {
126 println s;
127 }
128 }
129
[9505]130 void myprintln(String s) {
[11582]131 if(skip.containsKey(s)) {
132 String color = skip.get(s)
133 skip.remove(s)
[9658]134 if(options.xhtmlbody || options.xhtml) {
[11582]135 s = "<pre style=\"margin:3px;color:"+color+"\">"+s.replaceAll("&","&amp;").replaceAll("<","&lt;").replaceAll(">","&gt;")+"</pre>"
[9658]136 }
[9662]137 if (!options.noskip) {
138 return;
139 }
[9658]140 } else if(options.xhtmlbody || options.xhtml) {
[11582]141 String color = s.startsWith("***") ? "black" : ((s.startsWith("+ ") || s.startsWith("+++ ELI")) ? "blue" : "red")
[9658]142 s = "<pre style=\"margin:3px;color:"+color+"\">"+s.replaceAll("&","&amp;").replaceAll("<","&lt;").replaceAll(">","&gt;")+"</pre>"
[9505]143 }
[9658]144 myprintlnfinal(s)
145 }
146
147 void start() {
148 if (options.xhtml) {
149 myprintlnfinal "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n"
[11582]150 myprintlnfinal "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/><title>JOSM - ELI differences</title></head><body>\n"
[9505]151 }
152 }
[9653]153
[9658]154 void end() {
[11582]155 for (def s: skip.keySet()) {
[9658]156 myprintln "+++ Obsolete skip entry: " + s
157 }
158 if (options.xhtml) {
159 myprintlnfinal "</body></html>\n"
160 }
161 }
162
[11582]163 void loadELIEntries() {
164 FileReader fr = new FileReader(eliInputFile)
[7726]165 JsonReader jr = Json.createReader(fr)
[11582]166 eliEntries = jr.readObject().get("features")
[7726]167 jr.close()
[9653]168
[11582]169 for (def e : eliEntries) {
[7726]170 def url = getUrl(e)
[9653]171 if (url.contains("{z}")) {
[11582]172 myprintln "+++ ELI-URL uses {z} instead of {zoom}: "+url
[9653]173 url = url.replace("{z}","{zoom}")
174 }
[11582]175 if (eliUrls.containsKey(url)) {
176 myprintln "+++ ELI-URL is not unique: "+url
[9505]177 } else {
[11582]178 eliUrls.put(url, e)
[9505]179 }
[7726]180 }
[11582]181 myprintln "*** Loaded ${eliEntries.size()} entries (ELI). ***"
[7726]182 }
183
184 void loadJosmEntries() {
185 def reader = new ImageryReader(josmInputFile)
186 josmEntries = reader.parse()
[9667]187
[7726]188 for (def e : josmEntries) {
189 def url = getUrl(e)
[9658]190 if (url.contains("{z}")) {
191 myprintln "+++ JOSM-URL uses {z} instead of {zoom}: "+url
192 url = url.replace("{z}","{zoom}")
193 }
[9505]194 if (josmUrls.containsKey(url)) {
195 myprintln "+++ JOSM-URL is not unique: "+url
196 } else {
[11420]197 josmUrls.put(url, e)
[7726]198 }
[9658]199 for (def m : e.getMirrors()) {
200 url = getUrl(m)
[11574]201 m.origName = m.getOriginalName().replaceAll(" mirror server( \\d+)?","")
[9658]202 if (josmUrls.containsKey(url)) {
203 myprintln "+++ JOSM-Mirror-URL is not unique: "+url
204 } else {
[11420]205 josmUrls.put(url, m)
206 josmMirrors.put(url, m)
[9658]207 }
208 }
[7726]209 }
[9505]210 myprintln "*** Loaded ${josmEntries.size()} entries (JOSM). ***"
[7726]211 }
212
213 List inOneButNotTheOther(Map m1, Map m2) {
214 def l = []
215 for (def url : m1.keySet()) {
216 if (!m2.containsKey(url)) {
217 def name = getName(m1.get(url))
218 l += " "+getDescription(m1.get(url))
219 }
220 }
221 l.sort()
222 }
[9667]223
[7726]224 void checkInOneButNotTheOther() {
[11582]225 def l1 = inOneButNotTheOther(eliUrls, josmUrls)
226 myprintln "*** URLs found in ELI but not in JOSM (${l1.size()}): ***"
[9658]227 if (!l1.isEmpty()) {
[11412]228 for (def l : l1) {
229 myprintln "-" + l
230 }
[7726]231 }
232
[11582]233 if (options.nomissingeli)
[9505]234 return
[11582]235 def l2 = inOneButNotTheOther(josmUrls, eliUrls)
236 myprintln "*** URLs found in JOSM but not in ELI (${l2.size()}): ***"
[9658]237 if (!l2.isEmpty()) {
[11412]238 for (def l : l2) {
[9658]239 myprintln "+" + l
[11412]240 }
[7726]241 }
242 }
[9667]243
[7726]244 void checkCommonEntries() {
[9505]245 myprintln "*** Same URL, but different name: ***"
[11582]246 for (def url : eliUrls.keySet()) {
247 def e = eliUrls.get(url)
[7726]248 if (!josmUrls.containsKey(url)) continue
249 def j = josmUrls.get(url)
250 if (!getName(e).equals(getName(j))) {
[11582]251 myprintln "* Name differs ('${getName(e)}' != '${getName(j)}'): $url"
[7726]252 }
253 }
[9667]254
[9505]255 myprintln "*** Same URL, but different type: ***"
[11582]256 for (def url : eliUrls.keySet()) {
257 def e = eliUrls.get(url)
[7726]258 if (!josmUrls.containsKey(url)) continue
259 def j = josmUrls.get(url)
260 if (!getType(e).equals(getType(j))) {
[11582]261 myprintln "* Type differs (${getType(e)} != ${getType(j)}): ${getName(j)} - $url"
[7726]262 }
263 }
[9667]264
[9505]265 myprintln "*** Same URL, but different zoom bounds: ***"
[11582]266 for (def url : eliUrls.keySet()) {
267 def e = eliUrls.get(url)
[7726]268 if (!josmUrls.containsKey(url)) continue
269 def j = josmUrls.get(url)
270
271 Integer eMinZoom = getMinZoom(e)
272 Integer jMinZoom = getMinZoom(j)
[9518]273 if (eMinZoom != jMinZoom && !(eMinZoom == 0 && jMinZoom == null)) {
[11582]274 myprintln "* Minzoom differs (${eMinZoom} != ${jMinZoom}): ${getDescription(j)}"
[7726]275 }
276 Integer eMaxZoom = getMaxZoom(e)
277 Integer jMaxZoom = getMaxZoom(j)
278 if (eMaxZoom != jMaxZoom) {
[11582]279 myprintln "* Maxzoom differs (${eMaxZoom} != ${jMaxZoom}): ${getDescription(j)}"
[7726]280 }
281 }
[9667]282
[9505]283 myprintln "*** Same URL, but different country code: ***"
[11582]284 for (def url : eliUrls.keySet()) {
285 def e = eliUrls.get(url)
[7726]286 if (!josmUrls.containsKey(url)) continue
287 def j = josmUrls.get(url)
288 if (!getCountryCode(e).equals(getCountryCode(j))) {
[11582]289 myprintln "* Country code differs (${getCountryCode(e)} != ${getCountryCode(j)}): ${getDescription(j)}"
[7726]290 }
291 }
[11599]292 myprintln "*** Same URL, but different quality: ***"
[11582]293 for (def url : eliUrls.keySet()) {
294 def e = eliUrls.get(url)
[9515]295 if (!josmUrls.containsKey(url)) {
296 def q = getQuality(e)
[11582]297 if("eli-best".equals(q)) {
298 myprintln "- Quality best entry not in JOSM for ${getDescription(e)}"
[9515]299 }
300 continue
301 }
[9505]302 def j = josmUrls.get(url)
303 if (!getQuality(e).equals(getQuality(j))) {
[11582]304 myprintln "* Quality differs (${getQuality(e)} != ${getQuality(j)}): ${getDescription(j)}"
[9505]305 }
[11599]306 }
[11573]307 /*myprintln "*** Same URL, but different dates: ***"
[11582]308 for (def url : eliUrls.keySet()) {
309 def e = eliUrls.get(url)
[11573]310 if (!josmUrls.containsKey(url)) continue
311 def j = josmUrls.get(url)
312 if (!getDate(e).equals(getDate(j))) {
[11582]313 myprintln "* Date differs ('${getDate(e)}' != '${getDate(j)}'): ${getDescription(j)}"
[11573]314 }
315 }*/
[11410]316 myprintln "*** Mismatching shapes: ***"
317 for (def url : josmUrls.keySet()) {
318 def j = josmUrls.get(url)
319 def num = 1
320 for (def shape : getShapes(j)) {
321 def p = shape.getPoints()
322 if(!p[0].equals(p[p.size()-1])) {
323 myprintln "+++ JOSM shape $num unclosed: ${getDescription(j)}"
324 }
325 ++num
326 }
327 }
[11582]328 for (def url : eliUrls.keySet()) {
329 def e = eliUrls.get(url)
[11410]330 def num = 1
331 def s = getShapes(e)
332 for (def shape : s) {
333 def p = shape.getPoints()
[11582]334 if(!p[0].equals(p[p.size()-1]) && !options.nomissingeli) {
335 myprintln "+++ ELI shape $num unclosed: ${getDescription(e)}"
[11410]336 }
337 ++num
338 }
339 if (!josmUrls.containsKey(url)) {
340 continue
341 }
342 def j = josmUrls.get(url)
[11411]343 def js = getShapes(j)
[11414]344 if(!s.size() && js.size()) {
[11582]345 if(!options.nomissingeli) {
346 myprintln "+ No ELI shape: ${getDescription(j)}"
[11414]347 }
[11413]348 } else if(!js.size() && s.size()) {
[11415]349 // don't report boundary like 5 point shapes as difference
350 if (s.size() != 1 || s[0].getPoints().size() != 5) {
351 myprintln "- No JOSM shape: ${getDescription(j)}"
352 }
[11414]353 } else if(s.size() != js.size()) {
354 myprintln "* Different number of shapes (${s.size()} != ${js.size()}): ${getDescription(j)}"
[11413]355 } else {
[11414]356 for(def nums = 0; nums < s.size(); ++nums) {
357 def ep = s[nums].getPoints()
358 def jp = js[nums].getPoints()
359 if(ep.size() != jp.size()) {
360 myprintln "* Different number of points for shape ${nums+1} (${ep.size()} ! = ${jp.size()})): ${getDescription(j)}"
361 } else {
362 for(def nump = 0; nump < ep.size(); ++nump) {
363 def ept = ep[nump]
364 def jpt = jp[nump]
365 if(Math.abs(ept.getLat()-jpt.getLat()) > 0.000001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.000001) {
366 myprintln "* Different coordinate for point ${nump+1} of shape ${nums+1}: ${getDescription(j)}"
367 nump = ep.size()
368 num = s.size()
[11413]369 }
370 }
371 }
[11411]372 }
[11410]373 }
374 }
[11420]375 myprintln "*** Mismatching icons: ***"
[11582]376 for (def url : eliUrls.keySet()) {
377 def e = eliUrls.get(url)
[11420]378 if (!josmUrls.containsKey(url)) {
379 continue
380 }
381 def j = josmUrls.get(url)
382 def ij = getIcon(j)
383 def ie = getIcon(e)
384 if(ij != null && ie == null) {
[11582]385 if(!options.nomissingeli) {
386 myprintln "+ No ELI icon: ${getDescription(j)}"
[11420]387 }
388 } else if(ij == null && ie != null) {
389 myprintln "- No JOSM icon: ${getDescription(j)}"
390 } else if(!ij.equals(ie)) {
391 myprintln "* Different icons: ${getDescription(j)}"
392 }
393 }
394 myprintln "*** Miscellaneous checks: ***"
395 def josmIds = new HashMap<String, ImageryInfo>()
396 for (def url : josmUrls.keySet()) {
397 def j = josmUrls.get(url)
398 def id = getId(j)
399 if(josmMirrors.containsKey(url)) {
400 continue;
401 }
402 if(id == null) {
403 myprintln "* No JOSM-ID: ${getDescription(j)}"
404 } else if(josmIds.containsKey(id)) {
405 myprintln "* JOSM-ID ${id} not unique: ${getDescription(j)}"
406 } else {
407 josmIds.put(id, j);
408 }
[11572]409 def d = getDate(j)
[11573]410 if(!d.isEmpty()) {
[11572]411 def reg = (d =~ /^(\d\d\d\d)(-(\d\d)(-(\d\d))?)?(;(\d\d\d\d)(-(\d\d)(-(\d\d))?)?)?/)
412 if(reg == null || reg.count != 1) {
413 myprintln "* JOSM-Date '${d}' is strange: ${getDescription(j)}"
414 } else {
415 try {
416 def first = verifyDate(reg[0][1],reg[0][3],reg[0][5]);
417 def second = verifyDate(reg[0][7],reg[0][9],reg[0][11]);
418 if(second.compareTo(first) < 0) {
419 myprintln "* JOSM-Date '${d}' is strange (second earlier than first): ${getDescription(j)}"
420 }
421 }
422 catch (Exception e) {
423 myprintln "* JOSM-Date '${d}' is strange (${e.getMessage()}): ${getDescription(j)}"
424 }
425 }
426 }
[11422]427 def js = getShapes(j)
428 if(js.size()) {
429 def minlat = 1000;
430 def minlon = 1000;
431 def maxlat = -1000;
432 def maxlon = -1000;
433 for(def s: js) {
434 for(def p: s.getPoints()) {
435 def lat = p.getLat();
436 def lon = p.getLon();
437 if(lat > maxlat) maxlat = lat;
438 if(lon > maxlon) maxlon = lon;
439 if(lat < minlat) minlat = lat;
440 if(lon < minlon) minlon = lon;
441 }
442 }
443 def b = j.getBounds();
444 if(b.getMinLat() != minlat || b.getMinLon() != minlon || b.getMaxLat() != maxlat || b.getMaxLon() != maxlon) {
[11423]445 myprintln "* Bounds do not match shape (is ${b.getMinLat()},${b.getMinLon()},${b.getMaxLat()},${b.getMaxLon()}, calculated <bounds min-lat='${minlat}' min-lon='${minlon}' max-lat='${maxlat}' max-lon='${maxlon}'>): ${getDescription(j)}"
[11422]446 }
447 }
[11420]448 }
[7726]449 }
[9667]450
[7726]451 /**
452 * Utility functions that allow uniform access for both ImageryInfo and JsonObject.
453 */
454 static String getUrl(Object e) {
455 if (e instanceof ImageryInfo) return e.url
[11412]456 return e.get("properties").getString("url")
[7726]457 }
[11572]458 static String getDate(Object e) {
[11573]459 if (e instanceof ImageryInfo) return e.date ? e.date : ""
460 def p = e.get("properties")
461 def start = p.containsKey("start_date") ? p.getString("start_date") : ""
462 def end = p.containsKey("end_date") ? p.getString("end_date") : ""
463 if(!start.isEmpty() && !end.isEmpty())
[11572]464 return start+";"+end
[11573]465 else if(!start.isEmpty())
[11572]466 return start
467 else
468 return end
469 }
470 static Date verifyDate(String year, String month, String day) {
471 def date
472 if(year == null)
473 date = "3000-01-01"
474 else
475 date = year + "-" + (month == null ? "01" : month) + "-" + (day == null ? "01" : day)
476 def df = new java.text.SimpleDateFormat("yyyy-MM-dd")
477 df.setLenient(false)
478 return df.parse(date)
479 }
[11420]480 static String getId(Object e) {
481 if (e instanceof ImageryInfo) return e.getId()
482 return e.get("properties").getString("id")
483 }
[7726]484 static String getName(Object e) {
[10517]485 if (e instanceof ImageryInfo) return e.getOriginalName()
[11412]486 return e.get("properties").getString("name")
[7726]487 }
[11410]488 static List<Shape> getShapes(Object e) {
489 if (e instanceof ImageryInfo) {
[11411]490 def bounds = e.getBounds();
491 if(bounds != null) {
492 return bounds.getShapes();
493 }
494 return []
[11410]495 }
[11412]496 if(!e.isNull("geometry")) {
497 def ex = e.get("geometry")
498 if(ex != null && !ex.isNull("coordinates")) {
499 def poly = ex.get("coordinates")
[11410]500 List<Shape> l = []
501 for(def shapes: poly) {
502 def s = new Shape()
503 for(def point: shapes) {
504 def lon = point[0].toString()
505 def lat = point[1].toString()
506 s.addPoint(lat, lon)
507 }
508 l.add(s)
509 }
510 return l
511 }
512 }
513 return []
514 }
[7726]515 static String getType(Object e) {
516 if (e instanceof ImageryInfo) return e.getImageryType().getTypeString()
[11412]517 return e.get("properties").getString("type")
[7726]518 }
519 static Integer getMinZoom(Object e) {
520 if (e instanceof ImageryInfo) {
521 int mz = e.getMinZoom()
522 return mz == 0 ? null : mz
523 } else {
[11412]524 def num = e.get("properties").getJsonNumber("min_zoom")
[7726]525 if (num == null) return null
526 return num.intValue()
527 }
528 }
529 static Integer getMaxZoom(Object e) {
530 if (e instanceof ImageryInfo) {
531 int mz = e.getMaxZoom()
532 return mz == 0 ? null : mz
533 } else {
[11412]534 def num = e.get("properties").getJsonNumber("max_zoom")
[7726]535 if (num == null) return null
536 return num.intValue()
537 }
538 }
539 static String getCountryCode(Object e) {
540 if (e instanceof ImageryInfo) return "".equals(e.getCountryCode()) ? null : e.getCountryCode()
[11412]541 return e.get("properties").getString("country_code", null)
[7726]542 }
[9505]543 static String getQuality(Object e) {
[11575]544 if (e instanceof ImageryInfo) return e.isBestMarked() ? "eli-best" : null
545 return e.get("properties").get("best") ? "eli-best" : null
[9505]546 }
[11420]547 static String getIcon(Object e) {
548 if (e instanceof ImageryInfo) return e.getIcon()
549 return e.get("properties").getString("icon", null)
550 }
[7726]551 String getDescription(Object o) {
552 def url = getUrl(o)
553 def cc = getCountryCode(o)
554 if (cc == null) {
555 def j = josmUrls.get(url)
556 if (j != null) cc = getCountryCode(j)
557 if (cc == null) {
[11582]558 def e = eliUrls.get(url)
[7726]559 if (e != null) cc = getCountryCode(e)
560 }
561 }
562 if (cc == null) {
563 cc = ''
564 } else {
565 cc = "[$cc] "
566 }
567 def d = cc + getName(o) + " - " + getUrl(o)
568 if (options.shorten) {
569 def MAXLEN = 140
570 if (d.length() > MAXLEN) d = d.substring(0, MAXLEN-1) + "..."
571 }
572 return d
573 }
574}
Note: See TracBrowser for help on using the repository browser.