Changeset 8397 in josm for trunk/src

Timestamp:

2015-05-19T20:54:26+02:00 (9 years ago)

Author:

wiktorn

Message:

addresses #11437 - introduce infinite queue for tile loading and clear the queue when user pans the map or changes the zoom. Fixing hosts limit is last problem, thay may incur additional load

Location:

trunk/src/org/openstreetmap/josm

Files:

• data/cache/JCSCachedTileLoaderJob.java (modified) (7 diffs)
• data/imagery/TMSCachedTileLoader.java (modified) (5 diffs)
• data/imagery/TMSCachedTileLoaderJob.java (modified) (4 diffs)
• gui/layer/TMSLayer.java (modified) (37 diffs)

trunk/src/org/openstreetmap/josm/data/cache/JCSCachedTileLoaderJob.java

@@ -63 +63 @@
 
     public static class LIFOQueue extends LinkedBlockingDeque<Runnable> {
+        public LIFOQueue() {
+            super();
+        }
+
         public LIFOQueue(int capacity) {
             super(capacity);
@@ -91 +95 @@
      * and performance (we do want to have something to offer to worker threads before tasks will be resubmitted by class consumer)
      */
-    private static Executor DOWNLOAD_JOB_DISPATCHER = new ThreadPoolExecutor(
+    private static Executor DEFAULT_DOWNLOAD_JOB_DISPATCHER = new ThreadPoolExecutor(
             2, // we have a small queue, so threads will be quickly started (threads are started only, when queue is full)
             THREAD_LIMIT.get().intValue(), // do not this number of threads
@@ -114 +118 @@
     private int readTimeout;
     private Map<String, String> headers;
+    private Executor downloadJobExecutor;
 
     /**
@@ -120 +125 @@
      * @param readTimeout
      * @param connectTimeout
+     * @param downloadJobExecutor
      */
     public JCSCachedTileLoaderJob(ICacheAccess<K,V> cache,
             int connectTimeout, int readTimeout,
-            Map<String, String> headers) {
+            Map<String, String> headers,
+            Executor downloadJobExecutor) {
 
         this.cache = cache;
@@ -130 +137 @@
         this.readTimeout = readTimeout;
         this.headers = headers;
+        this.downloadJobExecutor = downloadJobExecutor;
+    }
+
+    /**
+     * @param cache cache instance that we will work on
+     * @param headers
+     * @param readTimeout
+     * @param connectTimeout
+     */
+    public JCSCachedTileLoaderJob(ICacheAccess<K, V> cache,
+            int connectTimeout, int readTimeout,
+            Map<String, String> headers) {
+        this(cache, connectTimeout, readTimeout,
+                headers, DEFAULT_DOWNLOAD_JOB_DISPATCHER);
     }
 
@@ -240 +261 @@
      */
     protected Executor getDownloadExecutor() {
-        return DOWNLOAD_JOB_DISPATCHER;
-    }
-
+        return downloadJobExecutor;
+    }
 
     public void run() {
@@ -330 +350 @@
                 return true;
             }
+
             HttpURLConnection urlConn = getURLConnection();
 

trunk/src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoader.java

@@ -4 +4 @@
 import java.io.IOException;
 import java.util.Map;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.jcs.access.behavior.ICacheAccess;
@@ -15 +17 @@
 import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
 import org.openstreetmap.josm.data.cache.JCSCacheManager;
+import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob.LIFOQueue;
 import org.openstreetmap.josm.data.preferences.IntegerProperty;
 
@@ -30 +33 @@
     private Map<String, String> headers;
     private TileLoaderListener listener;
-    public static final String PREFERENCE_PREFIX   = "imagery.tms.cache.";
+    private static final String PREFERENCE_PREFIX   = "imagery.tms.cache.";
     // average tile size is about 20kb
     public static final IntegerProperty MAX_OBJECTS_ON_DISK = new IntegerProperty(PREFERENCE_PREFIX + "max_objects_disk", 25000); // 25000 is around 500MB under this assumptions
 
+    /**
+     * overrides the THREAD_LIMIT in superclass, as we want to have separate limit and pool for TMS
+     */
+    public static final IntegerProperty THREAD_LIMIT = new IntegerProperty("imagery.tms.tmsloader.maxjobs", 25);
+
+    /**
+     * separate from JCS thread pool for TMS loader, so we can have different thread pools for default JCS
+     * and for TMS imagery
+     */
+    private static ThreadPoolExecutor DEFAULT_DOWNLOAD_JOB_DISPATCHER = getThreadPoolExecutor();
+
+    private static ThreadPoolExecutor getThreadPoolExecutor() {
+        return new ThreadPoolExecutor(
+                THREAD_LIMIT.get().intValue(), // keep the thread number constant
+                THREAD_LIMIT.get().intValue(), // do not this number of threads
+                30, // keepalive for thread
+                TimeUnit.SECONDS,
+                // make queue of LIFO type - so recently requested tiles will be loaded first (assuming that these are which user is waiting to see)
+                new LIFOQueue()
+                    /* keep the queue size fairly small, we do not want to
+                     download a lot of tiles, that user is not seeing anyway */
+                );
+    }
+
+    private ThreadPoolExecutor downloadExecutor = DEFAULT_DOWNLOAD_JOB_DISPATCHER;
 
     /**
@@ -58 +86 @@
     @Override
     public TileJob createTileLoaderJob(Tile tile) {
-        return new TMSCachedTileLoaderJob(listener, tile, cache, connectTimeout, readTimeout, headers);
+        return new TMSCachedTileLoaderJob(listener, tile, cache,
+                connectTimeout, readTimeout, headers, downloadExecutor);
     }
 
@@ -86 +115 @@
     }
 
+    /**
+     * @return cache statistics as string
+     */
     public String getStats() {
         return cache.getStats();
     }
+
+    /**
+     * Sets the download executor for this tile loader factory. Enables to use different queuing method
+     * for this factory.
+     * @param downloadExecutor
+     */
+    public void setDownloadExecutor(ThreadPoolExecutor downloadExecutor) {
+        this.downloadExecutor = downloadExecutor;
+    }
+
+    /**
+     * @return Executor that handles the jobs for this tile loader
+     */
+    public ThreadPoolExecutor getDownloadExecutor() {
+        return downloadExecutor;
+    }
+
+    /**
+     * cancels all outstanding tasks in the queue. This rollbacks the state of the tiles in the queue
+     * to loading = false / loaded = false
+     */
+    public void cancelOutstandingTasks() {
+        for(Runnable elem: downloadExecutor.getQueue()) {
+            if (elem instanceof TMSCachedTileLoaderJob) {
+                TMSCachedTileLoaderJob loaderJob = (TMSCachedTileLoaderJob) elem;
+                if (downloadExecutor.remove(loaderJob)) {
+                    Tile t = loaderJob.getTile();
+                    t.finishLoading();
+                    t.setLoaded(false);
+                }
+            }
+        }
+    }
+
 }

trunk/src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java

@@ -15 +15 @@
 import java.util.concurrent.Executor;
 import java.util.concurrent.Semaphore;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -99 +97 @@
 
     /**
-     * overrides the THREAD_LIMIT in superclass, as we want to have separate limit and pool for TMS
-     */
-    public static final IntegerProperty THREAD_LIMIT = new IntegerProperty("imagery.tms.tmsloader.maxjobs", 25);
-
-    /**
-     * separate from JCS thread pool for TMS loader, so we can have different thread pools for default JCS
-     * and for TMS imagery
-     */
-    private static ThreadPoolExecutor DOWNLOAD_JOB_DISPATCHER = getThreadPoolExecutor();
-
-    private static ThreadPoolExecutor getThreadPoolExecutor() {
-        return new ThreadPoolExecutor(
-                THREAD_LIMIT.get().intValue(), // keep the thread number constant
-                THREAD_LIMIT.get().intValue(), // do not this number of threads
-                30, // keepalive for thread
-                TimeUnit.SECONDS,
-                // make queue of LIFO type - so recently requested tiles will be loaded first (assuming that these are which user is waiting to see)
-                new LIFOQueue(5)
-                    /* keep the queue size fairly small, we do not want to
-                     download a lot of tiles, that user is not seeing anyway */
-                );
-    }
-
-    /**
      * Reconfigures download dispatcher using current values of THREAD_LIMIT and HOST_LIMIT
      */
     public static final void reconfigureDownloadDispatcher() {
         HOST_LIMITS = new ConcurrentHashMap<>();
-        DOWNLOAD_JOB_DISPATCHER = getThreadPoolExecutor();
     }
 
@@ -139 +112 @@
      * @param headers to be sent together with request
      */
-    public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile, ICacheAccess<String, BufferedImageCacheEntry> cache, int connectTimeout, int readTimeout,
-            Map<String, String> headers) {
-        super(cache, connectTimeout, readTimeout, headers);
+    public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile,
+            ICacheAccess<String, BufferedImageCacheEntry> cache,
+            int connectTimeout, int readTimeout, Map<String, String> headers,
+            Executor downloadExecutor) {
+        super(cache, connectTimeout, readTimeout, headers, downloadExecutor);
         this.tile = tile;
         if (listener != null) {
@@ -230 +205 @@
         }
         return false;
-    }
-
-    @Override
-    protected Executor getDownloadExecutor() {
-        return DOWNLOAD_JOB_DISPATCHER;
     }
 

trunk/src/org/openstreetmap/josm/gui/layer/TMSLayer.java

@@ -70 +70 @@
 import org.openstreetmap.josm.gui.MapView;
 import org.openstreetmap.josm.gui.MapView.LayerChangeListener;
+import org.openstreetmap.josm.gui.NavigatableComponent.ZoomChangeListener;
 import org.openstreetmap.josm.gui.PleaseWaitRunnable;
 import org.openstreetmap.josm.gui.dialogs.LayerListDialog;
@@ -91 +92 @@
  *
  */
-public class TMSLayer extends ImageryLayer implements ImageObserver, TileLoaderListener {
+public class TMSLayer extends ImageryLayer implements ImageObserver, TileLoaderListener, ZoomChangeListener {
     public static final String PREFERENCE_PREFIX   = "imagery.tms";
 
@@ -104 +105 @@
     public static final IntegerProperty PROP_MIN_ZOOM_LVL = new IntegerProperty(PREFERENCE_PREFIX + ".min_zoom_lvl", DEFAULT_MIN_ZOOM);
     public static final IntegerProperty PROP_MAX_ZOOM_LVL = new IntegerProperty(PREFERENCE_PREFIX + ".max_zoom_lvl", DEFAULT_MAX_ZOOM);
-    //public static final BooleanProperty PROP_DRAW_DEBUG = new BooleanProperty(PREFERENCE_PREFIX + ".draw_debug", false);
-    public static final BooleanProperty PROP_ADD_TO_SLIPPYMAP_CHOOSER = new BooleanProperty(PREFERENCE_PREFIX + ".add_to_slippymap_chooser", true);
+    public static final BooleanProperty PROP_ADD_TO_SLIPPYMAP_CHOOSER = new BooleanProperty(PREFERENCE_PREFIX +
+            ".add_to_slippymap_chooser", true);
     public static final StringProperty PROP_TILECACHE_DIR;
 
@@ -118 +119 @@
     }
 
+    /**
+     * Interface for creating TileLoaders, ie. classes responsible for loading tiles on map
+     *
+     */
     public interface TileLoaderFactory {
+        /**
+         * @param listener object that will be notified, when tile has finished loading
+         * @return TileLoader that will notify the listener
+         */
         TileLoader makeTileLoader(TileLoaderListener listener);
+
+        /**
+         * @param listener object that will be notified, when tile has finished loading
+         * @param headers HTTP headers that should be sent by TileLoader to tile server
+         * @return TileLoader that will notify the listener
+         */
         TileLoader makeTileLoader(TileLoaderListener listener, Map<String, String> headers);
     }
@@ -189 +204 @@
      * method.
      *
-     * @param monitor
-     * @see OsmFileCacheTileLoader#clearCache(org.openstreetmap.gui.jmapviewer.interfaces.TileSource, org.openstreetmap.gui.jmapviewer.interfaces.TileClearController)
-     */
-    void clearTileCache(ProgressMonitor monitor) {
+     * @param monitor not used in this implementation - as cache clear is instaneus
+     */
+    public void clearTileCache(ProgressMonitor monitor) {
         tileCache.clear();
         if (tileLoader instanceof CachedTileLoader) {
             ((CachedTileLoader)tileLoader).clearCache(tileSource);
         }
+        redraw();
     }
 
@@ -225 +240 @@
      * @see MapFrame#repaint()
      */
-    void redraw() {
+    protected void redraw() {
         needRedraw = true;
         Main.map.repaint();
     }
 
-    static int checkMaxZoomLvl(int maxZoomLvl, TileSource ts) {
+    protected static int checkMaxZoomLvl(int maxZoomLvl, TileSource ts) {
         if(maxZoomLvl > MAX_ZOOM) {
             maxZoomLvl = MAX_ZOOM;
@@ -248 +263 @@
 
     public static void setMaxZoomLvl(int maxZoomLvl) {
-        maxZoomLvl = checkMaxZoomLvl(maxZoomLvl, null);
-        PROP_MAX_ZOOM_LVL.put(maxZoomLvl);
+        Integer newMaxZoom = Integer.valueOf(checkMaxZoomLvl(maxZoomLvl, null));
+        PROP_MAX_ZOOM_LVL.put(newMaxZoom);
     }
 
@@ -342 +357 @@
             return t;
         } else if (info.getImageryType() == ImageryType.BING) {
-            //return new CachedAttributionBingAerialTileSource(info.getId());
             return new CachedAttributionBingAerialTileSource(info);
         } else if (info.getImageryType() == ImageryType.SCANEX) {
-            //return new ScanexTileSource(info.getName(), info.getUrl(), info.getId(), info.getMaxZoom());
             return new ScanexTileSource(info);
         }
@@ -385 +398 @@
         }
 
-        // FIXME: tileCache = new MemoryTileCache();
         tileLoader = loaderFactory.makeTileLoader(this, headers);
         if (tileLoader instanceof TMSCachedTileLoader) {
@@ -464 +476 @@
             throw new IllegalStateException("Cannot create TMSLayer with non-TMS ImageryInfo");
         initTileSource(source);
+
+        MapView.addZoomChangeListener(this);
     }
 
@@ -538 +552 @@
         }));
 
-        /* FIXME
         tileOptionMenu.add(new JMenuItem(new AbstractAction(
                 tr("Request Update")) {
             public void actionPerformed(ActionEvent ae) {
                 if (clickedTile != null) {
-                    clickedTile.requestUpdate();
-                    redraw();
-                }
-            }
-        }));*/
+                    clickedTile.setLoaded(false);
+                    tileLoader.createTileLoaderJob(clickedTile).submit();
+                }
+            }
+        }));
 
         tileOptionMenu.add(new JMenuItem(new AbstractAction(
@@ -590 +603 @@
             @Override
             public void actionPerformed(ActionEvent ae) {
-                double new_factor = Math.sqrt(getScaleFactor(currentZoomLevel));
-                Main.map.mapView.zoomToFactor(new_factor);
+                double newFactor = Math.sqrt(getScaleFactor(currentZoomLevel));
+                Main.map.mapView.zoomToFactor(newFactor);
                 redraw();
             }
@@ -647 +660 @@
                 if (oldLayer == TMSLayer.this) {
                     Main.map.mapView.removeMouseListener(adapter);
+                    MapView.removeZoomChangeListener(TMSLayer.this);
                     MapView.removeLayerChangeListener(this);
                 }
@@ -653 +667 @@
     }
 
-    void zoomChanged() {
+    /**
+     * This fires every time the user changes the zoom, but also (due to ZoomChangeListener) - on all
+     * changes to visible map (panning/zooming)
+     */
+    @Override
+    public void zoomChanged() {
         if (Main.isDebugEnabled()) {
             Main.debug("zoomChanged(): " + currentZoomLevel);
         }
         needRedraw = true;
-    }
-
-    int getMaxZoomLvl() {
+        if (tileLoader instanceof TMSCachedTileLoader) {
+            ((TMSCachedTileLoader) tileLoader).cancelOutstandingTasks();
+        }
+    }
+
+    protected int getMaxZoomLvl() {
         if (info.getMaxZoom() != 0)
             return checkMaxZoomLvl(info.getMaxZoom(), tileSource);
@@ -667 +689 @@
     }
 
-    int getMinZoomLvl() {
+    protected int getMinZoomLvl() {
         return getMinZoomLvl(tileSource);
     }
@@ -674 +696 @@
      * Zoom in, go closer to map.
      *
-     * @return    true, if zoom increasing was successfull, false othervise
+     * @return    true, if zoom increasing was successful, false otherwise
      */
     public boolean zoomIncreaseAllowed() {
@@ -742 +764 @@
      * into the tileCache.
      */
-    Tile tempCornerTile(Tile t) {
+    private Tile tempCornerTile(Tile t) {
         int x = t.getXtile() + 1;
         int y = t.getYtile() + 1;
@@ -752 +774 @@
     }
 
-    Tile getOrCreateTile(int x, int y, int zoom) {
+    private Tile getOrCreateTile(int x, int y, int zoom) {
         Tile tile = getTile(x, y, zoom);
         if (tile == null) {
@@ -766 +788 @@
      * already in the cache.
      */
-    Tile getTile(int x, int y, int zoom) {
+    private Tile getTile(int x, int y, int zoom) {
         int max = 1 << zoom;
         if (x < 0 || x >= max || y < 0 || y >= max)
@@ -773 +795 @@
     }
 
-    boolean loadTile(Tile tile, boolean force) {
+    private boolean loadTile(Tile tile, boolean force) {
         if (tile == null)
             return false;
@@ -784 +806 @@
     }
 
-    void loadAllTiles(boolean force) {
+    private void loadAllTiles(boolean force) {
         MapView mv = Main.map.mapView;
         EastNorth topLeft = mv.getEastNorth(0, 0);
@@ -800 +822 @@
     }
 
-    void loadAllErrorTiles(boolean force) {
+    private void loadAllErrorTiles(boolean force) {
         MapView mv = Main.map.mapView;
         EastNorth topLeft = mv.getEastNorth(0, 0);
@@ -821 +843 @@
     }
 
-    boolean imageLoaded(Image i) {
+    private boolean imageLoaded(Image i) {
         if (i == null)
             return false;
@@ -837 +859 @@
      * @return  the image of the tile or null.
      */
-    Image getLoadedTileImage(Tile tile) {
+    private Image getLoadedTileImage(Tile tile) {
         if (!tile.isLoaded())
             return null;
@@ -846 +868 @@
     }
 
-    LatLon tileLatLon(Tile t) {
+    private LatLon tileLatLon(Tile t) {
         int zoom = t.getZoom();
         return new LatLon(tileSource.tileYToLat(t.getYtile(), zoom),
@@ -852 +874 @@
     }
 
-    Rectangle tileToRect(Tile t1) {
+    private Rectangle tileToRect(Tile t1) {
         /*
          * We need to get a box in which to draw, so advance by one tile in
@@ -870 +892 @@
     // 'border' is the screen cordinates that need to be drawn.
     //  We must not draw outside of it.
-    void drawImageInside(Graphics g, Image sourceImg, Rectangle source, Rectangle border) {
+    private void drawImageInside(Graphics g, Image sourceImg, Rectangle source, Rectangle border) {
         Rectangle target = source;
 
@@ -931 +953 @@
     // drawn currently.  If drawing the displayZoomLevel,
     // border is null and we draw the entire tile set.
-    List<Tile> paintTileImages(Graphics g, TileSet ts, int zoom, Tile border) {
+    private List<Tile> paintTileImages(Graphics g, TileSet ts, int zoom, Tile border) {
         if (zoom <= 0) return Collections.emptyList();
         Rectangle borderRect = null;
@@ -962 +984 @@
     }
 
-    void myDrawString(Graphics g, String text, int x, int y) {
+    private void myDrawString(Graphics g, String text, int x, int y) {
         Color oldColor = g.getColor();
         g.setColor(Color.black);
@@ -970 +992 @@
     }
 
-    void paintTileText(TileSet ts, Tile tile, Graphics g, MapView mv, int zoom, Tile t) {
+    private void paintTileText(TileSet ts, Tile tile, Graphics g, MapView mv, int zoom, Tile t) {
         int fontHeight = g.getFontMetrics().getHeight();
         if (tile == null)
@@ -1066 +1088 @@
          * Create a TileSet by EastNorth bbox taking a layer shift in account
          */
-        TileSet(EastNorth topLeft, EastNorth botRight, int zoom) {
+        private TileSet(EastNorth topLeft, EastNorth botRight, int zoom) {
             this(getShiftedLatLon(topLeft), getShiftedLatLon(botRight),zoom);
         }
@@ -1073 +1095 @@
          * Create a TileSet by known LatLon bbox without layer shift correction
          */
-        TileSet(LatLon topLeft, LatLon botRight, int zoom) {
+        private TileSet(LatLon topLeft, LatLon botRight, int zoom) {
             this.zoom = zoom;
             if (zoom == 0)
@@ -1107 +1129 @@
         }
 
-        boolean tooSmall() {
+        private boolean tooSmall() {
             return this.tilesSpanned() < 2.1;
         }
 
-        boolean tooLarge() {
+        private boolean tooLarge() {
             return this.tilesSpanned() > 10;
         }
 
-        boolean insane() {
+        private boolean insane() {
             return this.tilesSpanned() > 100;
         }
 
-        double tilesSpanned() {
+        private double tilesSpanned() {
             return Math.sqrt(1.0 * this.size());
         }
 
-        int size() {
+        private int size() {
             int x_span = x1 - x0 + 1;
             int y_span = y1 - y0 + 1;
@@ -1133 +1155 @@
          * already in the tileCache.
          */
-        List<Tile> allExistingTiles() {
+        private List<Tile> allExistingTiles() {
             return this.__allTiles(false);
         }
 
-        List<Tile> allTilesCreate() {
+        private List<Tile> allTilesCreate() {
             return this.__allTiles(true);
         }
@@ -1171 +1193 @@
         }
 
-        void loadAllTiles(boolean force) {
+        private void loadAllTiles(boolean force) {
             if (!autoLoad && !force)
                 return;
@@ -1179 +1201 @@
         }
 
-        void loadAllErrorTiles(boolean force) {
+        private void loadAllErrorTiles(boolean force) {
             if (!autoLoad && !force)
                 return;
@@ -1418 +1440 @@
      * user right-clicks on the map.
      */
-    Tile getTileForPixelpos(int px, int py) {
+    private Tile getTileForPixelpos(int px, int py) {
         if (Main.isDebugEnabled()) {
             Main.debug("getTileForPixelpos("+px+", "+py+")");