source: josm/trunk/test/unit/org/openstreetmap/josm/gui/preferences/imagery/ImageryPreferenceTestIT.java@ 15456

Last change on this file since 15456 was 15456, checked in by wiktorn, 5 years ago

Allow zoom in TMS layer up to 24.

See: #18064
  • Property svn:eol-style set to native
File size: 17.2 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.gui.preferences.imagery;
3
4import static org.junit.Assert.assertFalse;
5import static org.junit.Assert.assertTrue;
6import static org.junit.Assume.assumeTrue;
7
8import java.io.ByteArrayInputStream;
9import java.io.IOException;
10import java.net.URL;
11import java.nio.charset.StandardCharsets;
12import java.util.ArrayList;
13import java.util.Collections;
14import java.util.List;
15import java.util.Locale;
16import java.util.Map;
17import java.util.Objects;
18import java.util.Optional;
19import java.util.TreeMap;
20import java.util.concurrent.TimeUnit;
21import java.util.stream.Collectors;
22
23import javax.imageio.ImageIO;
24
25import org.apache.commons.jcs.access.CacheAccess;
26import org.junit.AfterClass;
27import org.junit.BeforeClass;
28import org.junit.ClassRule;
29import org.junit.Test;
30import org.junit.runner.RunWith;
31import org.junit.runners.Parameterized.Parameters;
32import org.openstreetmap.gui.jmapviewer.Coordinate;
33import org.openstreetmap.gui.jmapviewer.TileXY;
34import org.openstreetmap.gui.jmapviewer.interfaces.ICoordinate;
35import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTileSource;
36import org.openstreetmap.gui.jmapviewer.tilesources.BingAerialTileSource;
37import org.openstreetmap.gui.jmapviewer.tilesources.ScanexTileSource;
38import org.openstreetmap.josm.TestUtils;
39import org.openstreetmap.josm.actions.AddImageryLayerAction;
40import org.openstreetmap.josm.actions.AddImageryLayerAction.LayerSelection;
41import org.openstreetmap.josm.data.Bounds;
42import org.openstreetmap.josm.data.coor.LatLon;
43import org.openstreetmap.josm.data.imagery.CoordinateConversion;
44import org.openstreetmap.josm.data.imagery.ImageryInfo;
45import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
46import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryType;
47import org.openstreetmap.josm.data.imagery.ImageryLayerInfo;
48import org.openstreetmap.josm.data.imagery.JosmTemplatedTMSTileSource;
49import org.openstreetmap.josm.data.imagery.LayerDetails;
50import org.openstreetmap.josm.data.imagery.Shape;
51import org.openstreetmap.josm.data.imagery.TMSCachedTileLoaderJob;
52import org.openstreetmap.josm.data.imagery.TemplatedWMSTileSource;
53import org.openstreetmap.josm.data.imagery.TileJobOptions;
54import org.openstreetmap.josm.data.imagery.WMTSTileSource;
55import org.openstreetmap.josm.data.imagery.WMTSTileSource.WMTSGetCapabilitiesException;
56import org.openstreetmap.josm.data.projection.Projection;
57import org.openstreetmap.josm.data.projection.ProjectionRegistry;
58import org.openstreetmap.josm.data.projection.Projections;
59import org.openstreetmap.josm.io.imagery.WMSImagery.WMSGetCapabilitiesException;
60import org.openstreetmap.josm.testutils.JOSMTestRules;
61import org.openstreetmap.josm.testutils.ParallelParameterized;
62import org.openstreetmap.josm.tools.HttpClient;
63import org.openstreetmap.josm.tools.HttpClient.Response;
64import org.openstreetmap.josm.tools.Logging;
65import org.openstreetmap.josm.tools.Utils;
66
67import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
68
69/**
70 * Integration tests of {@link ImageryPreference} class.
71 */
72@RunWith(ParallelParameterized.class)
73public class ImageryPreferenceTestIT {
74
75 private static final String ERROR_SEP = " -> ";
76 private static final LatLon GREENWICH = new LatLon(51.47810, -0.00170);
77 private static final int DEFAULT_ZOOM = 12;
78
79 /**
80 * Setup rule
81 */
82 @ClassRule
83 @SuppressFBWarnings(value = "URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
84 public static JOSMTestRules test = new JOSMTestRules().https().i18n().preferences().projection().projectionNadGrids()
85 .timeout((int) TimeUnit.MINUTES.toMillis(40)).parameters();
86
87 /** Entry to test */
88 private final ImageryInfo info;
89 private final Map<String, Map<ImageryInfo, List<String>>> errors = Collections.synchronizedMap(new TreeMap<>());
90 private final Map<String, Map<ImageryInfo, List<String>>> ignoredErrors = Collections.synchronizedMap(new TreeMap<>());
91 private static final Map<String, byte[]> workingURLs = Collections.synchronizedMap(new TreeMap<>());
92
93 private static TMSCachedTileLoaderJob helper;
94 private static final List<String> errorsToIgnore = new ArrayList<>();
95 private static final List<String> notIgnoredErrors = new ArrayList<>();
96
97 /**
98 * Setup test
99 * @throws IOException in case of I/O error
100 */
101 @BeforeClass
102 public static void beforeClass() throws IOException {
103 helper = new TMSCachedTileLoaderJob(null, null, new CacheAccess<>(null), new TileJobOptions(0, 0, null, 0), null);
104 errorsToIgnore.addAll(TestUtils.getIgnoredErrorMessages(ImageryPreferenceTestIT.class));
105 notIgnoredErrors.addAll(errorsToIgnore);
106 }
107
108 /**
109 * Cleanup test
110 */
111 @AfterClass
112 public static void afterClass() {
113 for (String e : notIgnoredErrors) {
114 Logging.warn("Ignore line unused: " + e);
115 }
116 }
117
118 /**
119 * Returns list of imagery entries to test.
120 * @return list of imagery entries to test
121 */
122 @Parameters(name = "{0}")
123 public static List<Object[]> data() {
124 ImageryLayerInfo.instance.load(false);
125 return ImageryLayerInfo.instance.getDefaultLayers()
126 .stream()
127 .map(x -> new Object[] {x.getId(), x})
128 .collect(Collectors.toList());
129 }
130
131 /**
132 * Constructs a new {@code ImageryPreferenceTestIT} instance.
133 * @param id entry ID, used only to name tests
134 * @param info entry to test
135 */
136 public ImageryPreferenceTestIT(String id, ImageryInfo info) {
137 this.info = Objects.requireNonNull(info);
138 }
139
140 private boolean addError(ImageryInfo info, String error) {
141 String errorMsg = error.replace('\n', ' ');
142 if (notIgnoredErrors.contains(errorMsg))
143 notIgnoredErrors.remove(errorMsg);
144 return addError(isIgnoredError(errorMsg) ? ignoredErrors : errors, info, errorMsg);
145 }
146
147 private static boolean isIgnoredError(String errorMsg) {
148 int idx = errorMsg.lastIndexOf(ERROR_SEP);
149 return isIgnoredSubstring(errorMsg) || (idx > -1 && isIgnoredSubstring(errorMsg.substring(idx + ERROR_SEP.length())));
150 }
151
152 private static boolean isIgnoredSubstring(String substring) {
153 return errorsToIgnore.parallelStream().anyMatch(x -> substring.contains(x));
154 }
155
156 private static boolean addError(Map<String, Map<ImageryInfo, List<String>>> map, ImageryInfo info, String errorMsg) {
157 return map.computeIfAbsent(info.getCountryCode(), x -> Collections.synchronizedMap(new TreeMap<>()))
158 .computeIfAbsent(info, x -> Collections.synchronizedList(new ArrayList<>()))
159 .add(errorMsg);
160 }
161
162 private Optional<byte[]> checkUrl(ImageryInfo info, String url) {
163 if (url != null && !url.isEmpty()) {
164 if (workingURLs.containsKey(url)) {
165 return Optional.of(workingURLs.get(url));
166 }
167 try {
168 Response response = HttpClient.create(new URL(url))
169 .setHeaders(info.getCustomHttpHeaders())
170 .setConnectTimeout((int) TimeUnit.MINUTES.toMillis(1))
171 .setReadTimeout((int) TimeUnit.MINUTES.toMillis(5))
172 .connect();
173 if (response.getResponseCode() >= 400) {
174 addError(info, url + " -> HTTP " + response.getResponseCode());
175 } else if (response.getResponseCode() >= 300) {
176 Logging.warn(url + " -> HTTP " + response.getResponseCode());
177 }
178 try {
179 byte[] data = Utils.readBytesFromStream(response.getContent());
180 if (response.getResponseCode() < 300) {
181 workingURLs.put(url, data);
182 }
183 return Optional.of(data);
184 } catch (IOException e) {
185 if (response.getResponseCode() < 300) {
186 addError(info, url + ERROR_SEP + e);
187 }
188 } finally {
189 response.disconnect();
190 }
191 } catch (IOException e) {
192 addError(info, url + ERROR_SEP + e);
193 }
194 }
195 return Optional.empty();
196 }
197
198 private void checkLinkUrl(ImageryInfo info, String url) {
199 checkUrl(info, url).filter(x -> x.length == 0).ifPresent(x -> addError(info, url + " -> returned empty contents"));
200 }
201
202 private String checkTileUrl(ImageryInfo info, AbstractTileSource tileSource, ICoordinate center, int zoom)
203 throws IOException {
204 TileXY xy = tileSource.latLonToTileXY(center, zoom);
205 for (int i = 0; i < 3; i++) {
206 try {
207 String url = tileSource.getTileUrl(zoom, xy.getXIndex(), xy.getYIndex());
208 Optional<byte[]> optional = checkUrl(info, url);
209 String error = "";
210 if (optional.isPresent()) {
211 byte[] data = optional.get();
212 try (ByteArrayInputStream bais = new ByteArrayInputStream(data)) {
213 if (ImageIO.read(bais) == null) {
214 error = addImageError(info, url, data, zoom, "did not return an image");
215 }
216 } catch (IOException e) {
217 error = addImageError(info, url, data, zoom, e.toString());
218 Logging.trace(e);
219 }
220 }
221 return error;
222 } catch (IOException e) {
223 // Try up to three times max to allow Bing source to initialize itself
224 // and avoid random network errors
225 Logging.trace(e);
226 if (i == 2) {
227 throw e;
228 }
229 try {
230 Thread.sleep(500);
231 } catch (InterruptedException ex) {
232 Logging.warn(ex);
233 }
234 }
235 }
236 return "";
237 }
238
239 private static String zoomMarker(int zoom) {
240 return " -> zoom " + zoom + ERROR_SEP;
241 }
242
243 private String addImageError(ImageryInfo info, String url, byte[] data, int zoom, String defaultMessage) {
244 // Check if we have received an error message
245 String error = helper.detectErrorMessage(new String(data, StandardCharsets.UTF_8));
246 String errorMsg = url + zoomMarker(zoom) + (error != null ? error.split("\\n")[0] : defaultMessage);
247 addError(info, errorMsg);
248 return errorMsg;
249 }
250
251 private static LatLon getPointInShape(Shape shape) {
252 final Coordinate p1 = shape.getPoints().get(0);
253 final Bounds bounds = new Bounds(p1.getLat(), p1.getLon(), p1.getLat(), p1.getLon());
254 shape.getPoints().forEach(p -> bounds.extend(p.getLat(), p.getLon()));
255
256 final double w = bounds.getWidth();
257 final double h = bounds.getHeight();
258
259 final double x2 = bounds.getMinLon() + (w / 2.0);
260 final double y2 = bounds.getMinLat() + (h / 2.0);
261
262 final LatLon center = new LatLon(y2, x2);
263
264 // check to see if center is inside shape
265 if (shape.contains(center)) {
266 return center;
267 }
268
269 // if center position (C) is not inside shape, try naively some other positions as follows:
270 final double x1 = bounds.getMinLon() + (.25 * w);
271 final double x3 = bounds.getMinLon() + (.75 * w);
272 final double y1 = bounds.getMinLat() + (.25 * h);
273 final double y3 = bounds.getMinLat() + (.75 * h);
274 // +-----------+
275 // | 5 1 6 |
276 // | 4 C 2 |
277 // | 8 3 7 |
278 // +-----------+
279 for (LatLon candidate : new LatLon[] {
280 new LatLon(y1, x2),
281 new LatLon(y2, x3),
282 new LatLon(y3, x2),
283 new LatLon(y2, x1),
284 new LatLon(y1, x1),
285 new LatLon(y1, x3),
286 new LatLon(y3, x3),
287 new LatLon(y3, x1)
288 }) {
289 if (shape.contains(candidate)) {
290 return candidate;
291 }
292 }
293 return center;
294 }
295
296 private static LatLon getCenter(ImageryBounds bounds) {
297 List<Shape> shapes = bounds.getShapes();
298 return shapes != null && !shapes.isEmpty() ? getPointInShape(shapes.get(0)) : bounds.getCenter();
299 }
300
301 private void checkEntry(ImageryInfo info) {
302 Logging.info("Checking "+ info);
303
304 if (info.getAttributionImageRaw() != null && info.getAttributionImage() == null) {
305 addError(info, "Can't fetch attribution image: " + info.getAttributionImageRaw());
306 }
307
308 checkLinkUrl(info, info.getAttributionImageURL());
309 checkLinkUrl(info, info.getAttributionLinkURL());
310 String eula = info.getEulaAcceptanceRequired();
311 if (eula != null) {
312 checkLinkUrl(info, eula.replaceAll("\\{lang\\}", ""));
313 }
314 checkLinkUrl(info, info.getPermissionReferenceURL());
315 checkLinkUrl(info, info.getTermsOfUseURL());
316
317 try {
318 ImageryBounds bounds = info.getBounds();
319 // Some imagery sources do not define tiles at (0,0). So pickup Greenwich Royal Observatory for global sources
320 ICoordinate center = CoordinateConversion.llToCoor(bounds != null ? getCenter(bounds) : GREENWICH);
321 AbstractTileSource tileSource = getTileSource(info);
322 // test min zoom and try to detect the correct value in case of error
323 int maxZoom = info.getMaxZoom() > 0 ? Math.min(DEFAULT_ZOOM, info.getMaxZoom()) : DEFAULT_ZOOM;
324 for (int zoom = info.getMinZoom(); zoom < maxZoom; zoom++) {
325 if (!isZoomError(checkTileUrl(info, tileSource, center, zoom))) {
326 break;
327 }
328 }
329 // checking max zoom for real is complex, see https://josm.openstreetmap.de/ticket/16073#comment:27
330 if (info.getMaxZoom() > 0 && info.getImageryType() != ImageryType.SCANEX) {
331 checkTileUrl(info, tileSource, center, Utils.clamp(DEFAULT_ZOOM, info.getMinZoom() + 1, info.getMaxZoom()));
332 }
333 } catch (IOException | RuntimeException | WMSGetCapabilitiesException | WMTSGetCapabilitiesException e) {
334 addError(info, info.getUrl() + ERROR_SEP + e.toString());
335 }
336
337 for (ImageryInfo mirror : info.getMirrors()) {
338 checkEntry(mirror);
339 }
340 }
341
342 private static boolean isZoomError(String error) {
343 String[] parts = error.split(ERROR_SEP);
344 String lastPart = parts.length > 0 ? parts[parts.length - 1].toLowerCase(Locale.ENGLISH) : "";
345 return lastPart.contains("bbox")
346 || lastPart.contains("bounding box");
347 }
348
349 private static Projection getProjection(ImageryInfo info) {
350 for (String code : info.getServerProjections()) {
351 Projection proj = Projections.getProjectionByCode(code);
352 if (proj != null) {
353 return proj;
354 }
355 }
356 return ProjectionRegistry.getProjection();
357 }
358
359 @SuppressWarnings("fallthrough")
360 private static AbstractTileSource getTileSource(ImageryInfo info)
361 throws IOException, WMTSGetCapabilitiesException, WMSGetCapabilitiesException {
362 switch (info.getImageryType()) {
363 case BING:
364 return new BingAerialTileSource(info);
365 case SCANEX:
366 return new ScanexTileSource(info);
367 case TMS:
368 return new JosmTemplatedTMSTileSource(info);
369 case WMS_ENDPOINT:
370 info = convertWmsEndpointToWms(info); // fall-through
371 case WMS:
372 return new TemplatedWMSTileSource(info, getProjection(info));
373 case WMTS:
374 return new WMTSTileSource(info, getProjection(info));
375 default:
376 throw new UnsupportedOperationException(info.toString());
377 }
378 }
379
380 private static ImageryInfo convertWmsEndpointToWms(ImageryInfo info) throws IOException, WMSGetCapabilitiesException {
381 return Optional.ofNullable(AddImageryLayerAction.getWMSLayerInfo(
382 info, wms -> new LayerSelection(firstLeafLayer(wms.getLayers()), wms.getPreferredFormat(), true)))
383 .orElseThrow(() -> new IllegalStateException("Unable to convert WMS_ENDPOINT to WMS"));
384 }
385
386 private static List<LayerDetails> firstLeafLayer(List<LayerDetails> layers) {
387 for (LayerDetails layer : layers) {
388 boolean hasNoChildren = layer.getChildren().isEmpty();
389 if (hasNoChildren && layer.getName() != null) {
390 return Collections.singletonList(layer);
391 } else if (!hasNoChildren) {
392 return firstLeafLayer(layer.getChildren());
393 }
394 }
395 throw new IllegalArgumentException("Unable to find a valid WMS layer");
396 }
397
398 private static String format(Map<String, Map<ImageryInfo, List<String>>> map) {
399 return map.toString().replaceAll("\\}, ", "\n\\}, ").replaceAll(", ImageryInfo\\{", "\n ,ImageryInfo\\{");
400 }
401
402 /**
403 * Test that available imagery entry is valid.
404 */
405 @Test
406 public void testImageryEntryValidity() {
407 checkEntry(info);
408 assertTrue(format(errors), errors.isEmpty());
409 assertFalse(workingURLs.isEmpty());
410 assumeTrue(format(ignoredErrors), ignoredErrors.isEmpty());
411 }
412}
Note: See TracBrowser for help on using the repository browser.