Changeset 8602 in josm


Ignore:
Timestamp:
2015-07-14T21:58:01+02:00 (9 years ago)
Author:
wiktorn
Message:

Properly handle file based tile sources.

  • move getThreadFactory(String name) to Utils class, so it's easily usable across JOSM
  • rollback changes in [8485]
  • detect that we are working with filesystem TileSource in AbstractTileSourceLayer and if so, do not use JCS as caching mechanism, as tiles are already local
  • change return value of getTileSourceInfo in AbstractTileSourceLayer to AbstractTMSTileSource, as this is anyway - lowest we can work with
  • add test data for testing file base tile sources

closes: #11548

Location:
trunk
Files:
75 added
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/org/openstreetmap/josm/data/cache/JCSCachedTileLoaderJob.java

    r8568 r8602  
    1414import java.util.concurrent.ConcurrentHashMap;
    1515import java.util.concurrent.ConcurrentMap;
    16 import java.util.concurrent.Executors;
    1716import java.util.concurrent.LinkedBlockingDeque;
    18 import java.util.concurrent.ThreadFactory;
    1917import java.util.concurrent.ThreadPoolExecutor;
    2018import java.util.concurrent.TimeUnit;
    … …  
    7977            // make queue of LIFO type - so recently requested tiles will be loaded first (assuming that these are which user is waiting to see)
    8078            new LinkedBlockingDeque<Runnable>(),
    81             getNamedThreadFactory("JCS downloader")
     79            Utils.getNamedThreadFactory("JCS downloader")
    8280            );
    8381
    84     public static ThreadFactory getNamedThreadFactory(final String name) {
    85         return new ThreadFactory() {
    86             @Override
    87             public Thread newThread(Runnable r) {
    88                 Thread t = Executors.defaultThreadFactory().newThread(r);
    89                 t.setName(name);
    90                 return t;
    91             }
    92         };
    93     }
     82
    9483
    9584    private static ConcurrentMap<String, Set<ICachedLoaderListener>> inProgress = new ConcurrentHashMap<>();
    … …  
    312301            }
    313302
    314             URLConnection urlConn = getURLConnection();
     303            HttpURLConnection urlConn = getURLConnection();
    315304
    316305            if (isObjectLoadable()  &&
    … …  
    321310                urlConn.addRequestProperty("If-None-Match", attributes.getEtag());
    322311            }
    323             if (responseCode(urlConn) == 304) {
     312            if (urlConn.getResponseCode() == 304) {
    324313                // If isModifiedSince or If-None-Match has been set
    325314                // and the server answers with a HTTP 304 = "Not Modified"
    326315                log.log(Level.FINE, "JCS - IfModifiedSince/Etag test: local version is up to date: {0}", getUrl());
    327316                return true;
    328             } else if (isObjectLoadable()) {
    329                 // we have an object in cache, but we haven't received 304 resposne code
    330                 // check if we should use HEAD request to verify
    331                 if ((attributes.getEtag() != null && attributes.getEtag().equals(urlConn.getRequestProperty("ETag"))) ||
    332                         attributes.getLastModification() == urlConn.getLastModified()) {
    333                     // we sent ETag or If-Modified-Since, but didn't get 304 response code
    334                     // for further requests - use HEAD
    335                     String serverKey = getServerKey();
    336                     log.log(Level.INFO, "JCS - Host: {0} found not to return 304 codes for If-Modifed-Since or If-None-Match headers",
    337                             serverKey);
    338                     useHead.put(serverKey, Boolean.TRUE);
    339                 }
    340             }
     317            } else if (isObjectLoadable() // we have an object in cache, but we haven't received 304 resposne code
     318                    && (
     319                            (attributes.getEtag() != null && attributes.getEtag().equals(urlConn.getRequestProperty("ETag"))) ||
     320                            attributes.getLastModification() == urlConn.getLastModified())
     321                    ) {
     322                // we sent ETag or If-Modified-Since, but didn't get 304 response code
     323                // for further requests - use HEAD
     324                String serverKey = getServerKey();
     325                log.log(Level.INFO, "JCS - Host: {0} found not to return 304 codes for If-Modifed-Since or If-None-Match headers",
     326                        serverKey);
     327                useHead.put(serverKey, Boolean.TRUE);
     328            }
     329
    341330
    342331            attributes = parseHeaders(urlConn);
    343332
    344333            for (int i = 0; i < 5; ++i) {
    345                 if (responseCode(urlConn) == 503) {
     334                if (urlConn.getResponseCode() == 503) {
    346335                    Thread.sleep(5000+(new Random()).nextInt(5000));
    347336                    continue;
    348337                }
    349338
    350                 attributes.setResponseCode(responseCode(urlConn));
     339                attributes.setResponseCode(urlConn.getResponseCode());
    351340                byte[] raw = Utils.readBytesFromStream(urlConn.getInputStream());
    352341
    353                 if (isResponseLoadable(urlConn.getHeaderFields(), responseCode(urlConn), raw)) {
     342                if (isResponseLoadable(urlConn.getHeaderFields(), urlConn.getResponseCode(), raw)) {
    354343                    // we need to check cacheEmpty, so for cases, when data is returned, but we want to store
    355344                    // as empty (eg. empty tile images) to save some space
    … …  
    446435    }
    447436
    448     private URLConnection getURLConnection() throws IOException {
    449         URLConnection urlConn = getUrl().openConnection();
     437    private HttpURLConnection getURLConnection() throws IOException {
     438        HttpURLConnection urlConn = (HttpURLConnection) getUrl().openConnection();
    450439        urlConn.setRequestProperty("Accept", "text/html, image/png, image/jpeg, image/gif, */*");
    451440        urlConn.setReadTimeout(readTimeout); // 30 seconds read timeout
    … …  
    461450
    462451    private boolean isCacheValidUsingHead() throws IOException {
    463         URLConnection urlConn = getUrl().openConnection();
    464         if (urlConn instanceof HttpURLConnection) {
    465             ((HttpURLConnection) urlConn).setRequestMethod("HEAD");
    466             long lastModified = urlConn.getLastModified();
    467             return (attributes.getEtag() != null && attributes.getEtag().equals(urlConn.getRequestProperty("ETag"))) ||
    468                     (lastModified != 0 && lastModified <= attributes.getLastModification());
    469         }
    470         // for other URL connections, do not use HEAD requests for cache validation
    471         return false;
     452        HttpURLConnection urlConn = getURLConnection();
     453        urlConn.setRequestMethod("HEAD");
     454        long lastModified = urlConn.getLastModified();
     455        return (attributes.getEtag() != null && attributes.getEtag().equals(urlConn.getRequestProperty("ETag"))) ||
     456                (lastModified != 0 && lastModified <= attributes.getLastModification());
    472457    }
    473458
    … …  
    499484        finishLoading(LoadResult.CANCELED);
    500485    }
    501 
    502     /*
    503      * Temporary fix for file URLs. Returns response code for HttpURLConnections or 200 for all other
    504      */
    505     private int responseCode(URLConnection urlConn) throws IOException {
    506         if (urlConn instanceof HttpURLConnection) {
    507             return ((HttpURLConnection) urlConn).getResponseCode();
    508         } else {
    509             return 200;
    510         }
    511     }
    512486}

  • trunk/src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoader.java

    r8598 r8602  
    1717import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
    1818import org.openstreetmap.josm.data.cache.HostLimitQueue;
    19 import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob;
    2019import org.openstreetmap.josm.data.preferences.IntegerProperty;
     20import org.openstreetmap.josm.tools.Utils;
    2121
    2222/**
    … …  
    8585                TimeUnit.SECONDS,
    8686                new HostLimitQueue(HOST_LIMIT.get().intValue()),
    87                 JCSCachedTileLoaderJob.getNamedThreadFactory(name)
     87                Utils.getNamedThreadFactory(name)
    8888                );
    8989    }

  • trunk/src/org/openstreetmap/josm/gui/layer/AbstractTileSourceLayer.java

    r8601 r8602  
    2020import java.io.File;
    2121import java.io.IOException;
     22import java.net.MalformedURLException;
     23import java.net.URL;
    2224import java.text.SimpleDateFormat;
    2325import java.util.ArrayList;
    … …  
    5658import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
    5759import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
     60import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource;
    5861import org.openstreetmap.josm.Main;
    5962import org.openstreetmap.josm.actions.RenameLayerAction;
    … …  
    129132
    130133    protected TileCache tileCache;
    131     protected TileSource tileSource;
     134    protected AbstractTMSTileSource tileSource;
    132135    protected TileLoader tileLoader;
    133136
    … …  
    151154     * @throws IllegalArgumentException when Imagery is not supported by layer
    152155     */
    153     protected abstract TileSource getTileSource(ImageryInfo info) throws IllegalArgumentException;
     156    protected abstract AbstractTMSTileSource getTileSource(ImageryInfo info) throws IllegalArgumentException;
    154157
    155158    protected Map<String, String> getHeaders(TileSource tileSource) {
    … …  
    160163    }
    161164
    162     protected void initTileSource(TileSource tileSource) {
     165    protected void initTileSource(AbstractTMSTileSource tileSource) {
    163166        attribution.initialize(tileSource);
    164167
    … …  
    173176            tileCache = new MemoryTileCache();
    174177        }
     178
     179        try {
     180            if ("file".equalsIgnoreCase(new URL(tileSource.getBaseUrl()).getProtocol())) {
     181                tileLoader = new OsmTileLoader(this);
     182                tileCache = new MemoryTileCache();
     183            }
     184        } catch (MalformedURLException e) {
     185            // ignore, assume that this is not a file
     186        }
     187
     188
    175189        if (tileLoader == null)
    176190            tileLoader = new OsmTileLoader(this);

  • trunk/src/org/openstreetmap/josm/gui/layer/TMSLayer.java

    r8598 r8602  
    66import org.apache.commons.jcs.access.CacheAccess;
    77import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader;
    8 import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
     8import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource;
    99import org.openstreetmap.gui.jmapviewer.tilesources.ScanexTileSource;
    1010import org.openstreetmap.gui.jmapviewer.tilesources.TMSTileSource;
    … …  
    6666     */
    6767    @Override
    68     protected TileSource getTileSource(ImageryInfo info) throws IllegalArgumentException {
     68    protected AbstractTMSTileSource getTileSource(ImageryInfo info) throws IllegalArgumentException {
    6969        return getTileSourceStatic(info);
    7070    }
    … …  
    9696     * @throws IllegalArgumentException if url from imagery info is null or invalid
    9797     */
    98     public static TileSource getTileSourceStatic(ImageryInfo info) throws IllegalArgumentException {
     98    public static AbstractTMSTileSource getTileSourceStatic(ImageryInfo info) throws IllegalArgumentException {
    9999        if (info.getImageryType() == ImageryType.TMS) {
    100100            TemplatedTMSTileSource.checkUrl(info.getUrl());

  • trunk/src/org/openstreetmap/josm/gui/layer/WMSLayer.java

    r8598 r8602  
    1818import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader;
    1919import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
     20import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource;
    2021import org.openstreetmap.josm.Main;
    2122import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
    … …  
    7374
    7475    @Override
    75     protected TileSource getTileSource(ImageryInfo info) {
     76    protected AbstractTMSTileSource getTileSource(ImageryInfo info) {
    7677        if (info.getImageryType() == ImageryType.WMS && info.getUrl() != null) {
    7778            TemplatedWMSTileSource.checkUrl(info.getUrl());

  • trunk/src/org/openstreetmap/josm/gui/layer/WMTSLayer.java

    r8598 r8602  
    88import org.openstreetmap.gui.jmapviewer.interfaces.ICoordinate;
    99import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader;
    10 import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
     10import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource;
    1111import org.openstreetmap.josm.Main;
    1212import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
    … …  
    4848
    4949    @Override
    50     protected TileSource getTileSource(ImageryInfo info) {
     50    protected AbstractTMSTileSource getTileSource(ImageryInfo info) {
    5151        try {
    5252            if (info.getImageryType() == ImageryType.WMTS && info.getUrl() != null) {

  • trunk/src/org/openstreetmap/josm/tools/Utils.java

    r8574 r8602  
    4646import java.util.concurrent.ExecutorService;
    4747import java.util.concurrent.Executors;
     48import java.util.concurrent.ThreadFactory;
    4849import java.util.regex.Matcher;
    4950import java.util.regex.Pattern;
    … …  
    13701371     * Returns the initial capacity to pass to the HashMap / HashSet constructor
    13711372     * when it is initialized with a known number of entries.
    1372      * 
     1373     *
    13731374     * When a HashMap is filled with entries, the underlying array is copied over
    13741375     * to a larger one multiple times. To avoid this process when the number of
    … …  
    13871388     * Returns the initial capacity to pass to the HashMap / HashSet constructor
    13881389     * when it is initialized with a known number of entries.
    1389      * 
     1390     *
    13901391     * When a HashMap is filled with entries, the underlying array is copied over
    13911392     * to a larger one multiple times. To avoid this process when the number of
    … …  
    13931394     * given to the HashMap constructor. This method returns a suitable value
    13941395     * that avoids rehashing but doesn't waste memory.
    1395      * 
     1396     *
    13961397     * Assumes default load factor (0.75).
    13971398     * @param nEntries the number of entries expected
    … …  
    14011402        return hashMapInitialCapacity(nEntries, 0.75f);
    14021403    }
     1404
     1405    /**
     1406     * @param name to be set for the threads
     1407     * @return Thread Factory returning named threads
     1408     */
     1409    public static ThreadFactory getNamedThreadFactory(final String name) {
     1410        return new ThreadFactory() {
     1411            @Override
     1412            public Thread newThread(Runnable r) {
     1413                Thread t = Executors.defaultThreadFactory().newThread(r);
     1414                t.setName(name);
     1415                return t;
     1416            }
     1417        };
     1418    }
    14031419}
Note: See TracChangeset for help on using the changeset viewer.