source: josm/trunk/src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java@ 19050

Last change on this file since 19050 was 19050, checked in by taylor.smock, 6 weeks ago

Revert most var changes from r19048, fix most new compile warnings and checkstyle issues

Also, document why various ErrorProne checks were originally disabled and fix
generic SonarLint issues.
  • Property svn:eol-style set to native
File size: 15.4 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.imagery;
3
4import static org.openstreetmap.josm.tools.I18n.tr;
5
6import java.io.ByteArrayInputStream;
7import java.io.IOException;
8import java.net.SocketTimeoutException;
9import java.net.URL;
10import java.nio.charset.StandardCharsets;
11import java.util.HashSet;
12import java.util.List;
13import java.util.Locale;
14import java.util.Map;
15import java.util.Map.Entry;
16import java.util.Optional;
17import java.util.Set;
18import java.util.concurrent.ConcurrentHashMap;
19import java.util.concurrent.ConcurrentMap;
20import java.util.concurrent.ThreadPoolExecutor;
21import java.util.concurrent.TimeUnit;
22import java.util.regex.Matcher;
23import java.util.regex.Pattern;
24
25import org.apache.commons.jcs3.access.behavior.ICacheAccess;
26import org.apache.commons.jcs3.engine.behavior.ICache;
27import org.openstreetmap.gui.jmapviewer.Tile;
28import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
29import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
30import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
31import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource;
32import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
33import org.openstreetmap.josm.data.cache.CacheEntry;
34import org.openstreetmap.josm.data.cache.CacheEntryAttributes;
35import org.openstreetmap.josm.data.cache.ICachedLoaderListener;
36import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob;
37import org.openstreetmap.josm.data.imagery.vectortile.VectorTile;
38import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MVTFile;
39import org.openstreetmap.josm.data.preferences.LongProperty;
40import org.openstreetmap.josm.tools.HttpClient;
41import org.openstreetmap.josm.tools.Logging;
42import org.openstreetmap.josm.tools.Utils;
43
44/**
45 * Class bridging TMS requests to JCS cache requests
46 *
47 * @author Wiktor Niesiobędzki
48 * @since 8168
49 */
50public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, BufferedImageCacheEntry> implements TileJob, ICachedLoaderListener {
51 /** General maximum expires for tiles. Might be overridden by imagery settings */
52 public static final LongProperty MAXIMUM_EXPIRES = new LongProperty("imagery.generic.maximum_expires", TimeUnit.DAYS.toMillis(30));
53 /** General minimum expires for tiles. Might be overridden by imagery settings */
54 public static final LongProperty MINIMUM_EXPIRES = new LongProperty("imagery.generic.minimum_expires", TimeUnit.HOURS.toMillis(1));
55 static final Pattern SERVICE_EXCEPTION_PATTERN = Pattern.compile("(?s).+<ServiceException[^>]*>(.+)</ServiceException>.+");
56 static final Pattern CDATA_PATTERN = Pattern.compile("(?s)\\s*<!\\[CDATA\\[(.+)\\]\\]>\\s*");
57 static final Pattern JSON_PATTERN = Pattern.compile("\\{\"message\":\"(.+)\"\\}");
58 protected final Tile tile;
59 private volatile URL url;
60 private final TileJobOptions options;
61
62 // we need another deduplication of Tile Loader listeners, as for each submit, new TMSCachedTileLoaderJob was created
63 // that way, we reduce calls to tileLoadingFinished, and general CPU load due to surplus Map repaints
64 private static final ConcurrentMap<String, Set<TileLoaderListener>> inProgress = new ConcurrentHashMap<>();
65
66 /**
67 * Constructor for creating a job, to get a specific tile from cache
68 * @param listener Tile loader listener
69 * @param tile to be fetched from cache
70 * @param cache object
71 * @param options for job (such as http headers, timeouts etc.)
72 * @param downloadExecutor that will be executing the jobs
73 */
74 public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile,
75 ICacheAccess<String, BufferedImageCacheEntry> cache,
76 TileJobOptions options,
77 ThreadPoolExecutor downloadExecutor) {
78 super(cache, options, downloadExecutor);
79 this.tile = tile;
80 this.options = options;
81 if (listener != null) {
82 inProgress.computeIfAbsent(getCacheKey(), k -> new HashSet<>()).add(listener);
83 }
84 }
85
86 @Override
87 public String getCacheKey() {
88 if (tile != null) {
89 TileSource tileSource = tile.getTileSource();
90 return Optional.ofNullable(tileSource.getName()).orElse("").replace(ICache.NAME_COMPONENT_DELIMITER, "_")
91 + ICache.NAME_COMPONENT_DELIMITER
92 + tileSource.getTileId(tile.getZoom(), tile.getXtile(), tile.getYtile());
93 }
94 return null;
95 }
96
97 /*
98 * this doesn't needs to be synchronized, as it's not that costly to keep only one execution
99 * in parallel, but URL creation and Tile.getUrl() are costly and are not needed when fetching
100 * data from cache, that's why URL creation is postponed until it's needed
101 *
102 * We need to have static url value for TileLoaderJob, as for some TileSources we might get different
103 * URL's each call we made (servers switching), and URL's are used below as a key for duplicate detection
104 *
105 */
106 @Override
107 public URL getUrl() throws IOException {
108 if (url == null) {
109 synchronized (this) {
110 if (url == null) {
111 String sUrl = tile.getUrl();
112 if (!"".equals(sUrl)) {
113 url = new URL(sUrl);
114 }
115 }
116 }
117 }
118 return url;
119 }
120
121 @Override
122 public boolean isObjectLoadable() {
123 if (cacheData != null) {
124 byte[] content = cacheData.getContent();
125 try {
126 return content.length > 0 || cacheData.getImage() != null || isNoTileAtZoom();
127 } catch (IOException e) {
128 Logging.logWithStackTrace(Logging.LEVEL_WARN, e, "JCS TMS - error loading from cache for tile {0}: {1}",
129 tile.getKey(), e.getMessage());
130 }
131 }
132 return false;
133 }
134
135 @Override
136 protected boolean isResponseLoadable(Map<String, List<String>> headers, int statusCode, byte[] content) {
137 attributes.setMetadata(tile.getTileSource().getMetadata(headers));
138 if (tile.getTileSource().isNoTileAtZoom(headers, statusCode, content)) {
139 attributes.setNoTileAtZoom(true);
140 return false; // do no try to load data from no-tile at zoom, cache empty object instead
141 }
142 if (isNotImage(headers, statusCode)) {
143 String message = detectErrorMessage(new String(content, StandardCharsets.UTF_8));
144 if (!Utils.isEmpty(message)) {
145 tile.setError(message);
146 }
147 return false;
148 }
149 return super.isResponseLoadable(headers, statusCode, content);
150 }
151
152 private boolean isNotImage(Map<String, List<String>> headers, int statusCode) {
153 if (statusCode == 200 && headers.containsKey("Content-Type") && !headers.get("Content-Type").isEmpty()) {
154 String contentType = headers.get("Content-Type").stream().findAny().orElse(null);
155 if (contentType != null && !contentType.startsWith("image") && !MVTFile.MIMETYPE.contains(contentType.toLowerCase(Locale.ROOT))) {
156 Logging.warn("Image not returned for tile: " + url + " content type was: " + contentType);
157 // not an image - do not store response in cache, so next time it will be queried again from the server
158 return true;
159 }
160 }
161 return false;
162 }
163
164 @Override
165 protected boolean cacheAsEmpty(Map<String, List<String>> headerFields, int responseCode) {
166 if (isNotImage(headerFields, responseCode)) {
167 return false;
168 }
169 return isNoTileAtZoom() || super.cacheAsEmpty(headerFields, responseCode);
170 }
171
172 @Override
173 public void submit(boolean force) {
174 tile.initLoading();
175 try {
176 super.submit(this, force);
177 } catch (IOException | IllegalArgumentException e) {
178 // if we fail to submit the job, mark tile as loaded and set error message
179 Logging.log(Logging.LEVEL_WARN, e);
180 tile.finishLoading();
181 tile.setError(e.getMessage());
182 }
183 }
184
185 @Override
186 public void loadingFinished(CacheEntry object, CacheEntryAttributes attributes, LoadResult result) {
187 this.attributes = attributes; // as we might get notification from other object than our selfs, pass attributes along
188 Set<TileLoaderListener> listeners = inProgress.remove(getCacheKey());
189 boolean status = result == LoadResult.SUCCESS;
190
191 try {
192 tile.finishLoading(); // whatever happened set that loading has finished
193 // set tile metadata
194 if (this.attributes != null) {
195 for (Entry<String, String> e: this.attributes.getMetadata().entrySet()) {
196 tile.putValue(e.getKey(), e.getValue());
197 }
198 }
199
200 switch (result) {
201 case SUCCESS:
202 handleNoTileAtZoom();
203 if (attributes != null) {
204 int httpStatusCode = attributes.getResponseCode();
205 if (httpStatusCode >= 400 && !isNoTileAtZoom()) {
206 status = false;
207 handleError(attributes);
208 }
209 }
210 status &= tryLoadTileImage(object); //try to keep returned image as background
211 break;
212 case FAILURE:
213 handleError(attributes);
214 tryLoadTileImage(object);
215 break;
216 case CANCELED:
217 tile.loadingCanceled();
218 // do nothing
219 }
220
221 // always check, if there is some listener interested in fact, that tile has finished loading
222 if (listeners != null) { // listeners might be null, if some other thread notified already about success
223 for (TileLoaderListener l: listeners) {
224 l.tileLoadingFinished(tile, status);
225 }
226 }
227 } catch (IOException e) {
228 Logging.warn("JCS TMS - error loading object for tile {0}: {1}", tile.getKey(), e.getMessage());
229 tile.setError(e);
230 tile.setLoaded(false);
231 if (listeners != null) { // listeners might be null, if some other thread notified already about success
232 for (TileLoaderListener l: listeners) {
233 l.tileLoadingFinished(tile, false);
234 }
235 }
236 }
237 }
238
239 private void handleError(CacheEntryAttributes attributes) {
240 if (tile.hasError() && tile.getErrorMessage() != null) {
241 // tile has already set error message, don't overwrite it
242 return;
243 }
244 if (attributes != null) {
245 int httpStatusCode = attributes.getResponseCode();
246 if (attributes.getErrorMessage() == null) {
247 tile.setError(tr("HTTP error {0} when loading tiles", httpStatusCode));
248 } else {
249 tile.setError(tr("Error downloading tiles: {0}", attributes.getErrorMessage()));
250 }
251 if (httpStatusCode >= 500 && httpStatusCode != 599) {
252 // httpStatusCode = 599 is set by JCSCachedTileLoaderJob on IOException
253 tile.setLoaded(false); // treat 500 errors as temporary and try to load it again
254 }
255 // treat SocketTimeoutException as transient error
256 attributes.getException()
257 .filter(x -> x.isAssignableFrom(SocketTimeoutException.class))
258 .ifPresent(x -> tile.setLoaded(false));
259 } else {
260 tile.setError(tr("Problem loading tile"));
261 // treat unknown errors as permanent and do not try to load tile again
262 }
263 }
264
265 /**
266 * For TMS use BaseURL as settings discovery, so for different paths, we will have different settings (useful for developer servers)
267 *
268 * @return base URL of TMS or server url as defined in super class
269 */
270 @Override
271 protected String getServerKey() {
272 TileSource ts = tile.getSource();
273 if (ts instanceof AbstractTMSTileSource) {
274 return ((AbstractTMSTileSource) ts).getBaseUrl();
275 }
276 return super.getServerKey();
277 }
278
279 @Override
280 protected BufferedImageCacheEntry createCacheEntry(byte[] content) {
281 return new BufferedImageCacheEntry(content);
282 }
283
284 @Override
285 public void submit() {
286 submit(false);
287 }
288
289 @Override
290 protected CacheEntryAttributes parseHeaders(HttpClient.Response urlConn) {
291 CacheEntryAttributes ret = super.parseHeaders(urlConn);
292 // keep the expiration time between MINIMUM_EXPIRES and MAXIMUM_EXPIRES, so we will cache the tiles
293 // at least for some short period of time, but not too long
294 long minimumExpiryTime = TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime());
295 long nowPlusMin = now + Math.max(MINIMUM_EXPIRES.get(), minimumExpiryTime);
296 if (ret.getExpirationTime() < nowPlusMin) {
297 ret.setExpirationTime(nowPlusMin);
298 }
299 long nowPlusMax = now + Math.max(MAXIMUM_EXPIRES.get(), minimumExpiryTime);
300 if (ret.getExpirationTime() > nowPlusMax) {
301 ret.setExpirationTime(nowPlusMax);
302 }
303 return ret;
304 }
305
306 private boolean handleNoTileAtZoom() {
307 if (isNoTileAtZoom()) {
308 Logging.debug("JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile);
309 tile.setError(tr("No tiles at this zoom level"));
310 tile.putValue("tile-info", "no-tile");
311 return true;
312 }
313 return false;
314 }
315
316 private boolean isNoTileAtZoom() {
317 if (attributes == null) {
318 Logging.warn("Cache attributes are null");
319 }
320 return attributes != null && attributes.isNoTileAtZoom();
321 }
322
323 private boolean tryLoadTileImage(CacheEntry object) throws IOException {
324 if (object != null) {
325 byte[] content = object.getContent();
326 if (content.length > 0 || tile instanceof VectorTile) {
327 try (ByteArrayInputStream in = new ByteArrayInputStream(content)) {
328 tile.loadImage(in);
329 if ((!(tile instanceof VectorTile) && tile.getImage() == null)
330 || ((tile instanceof VectorTile) && !tile.isLoaded())) {
331 String s = new String(content, StandardCharsets.UTF_8);
332 Matcher m = SERVICE_EXCEPTION_PATTERN.matcher(s);
333 if (m.matches()) {
334 String message = Utils.strip(m.group(1));
335 tile.setError(message);
336 Logging.error(message);
337 Logging.debug(s);
338 } else {
339 tile.setError(tr("Could not load image from tile server"));
340 }
341 return false;
342 }
343 } catch (UnsatisfiedLinkError | SecurityException e) {
344 throw new IOException(e);
345 }
346 }
347 }
348 return true;
349 }
350
351 @Override
352 public String detectErrorMessage(String data) {
353 Matcher xml = SERVICE_EXCEPTION_PATTERN.matcher(data);
354 Matcher json = JSON_PATTERN.matcher(data);
355 return xml.matches() ? removeCdata(Utils.strip(xml.group(1)))
356 : json.matches() ? Utils.strip(json.group(1))
357 : super.detectErrorMessage(data);
358 }
359
360 private static String removeCdata(String msg) {
361 Matcher m = CDATA_PATTERN.matcher(msg);
362 return m.matches() ? Utils.strip(m.group(1)) : msg;
363 }
364}
Note: See TracBrowser for help on using the repository browser.