Changeset 14898 in josm

Timestamp:

2019-03-18T09:21:40+01:00 (6 years ago)

Author:

GerdP

Message:

fix #17291: improve handling of return code 404 in multifetch api

• improve performance when server returns error code 404 (redacted): split workload into two halves instead of requesting each single id
• reduce MAX_IDS_PER_REQUEST from 200 to 170 (this might decrease performace for large amounts of objects but should be safer)

File:

• trunk/src/org/openstreetmap/josm/io/MultiFetchServerObjectReader.java (modified) (4 diffs)

trunk/src/org/openstreetmap/josm/io/MultiFetchServerObjectReader.java

@@ -60 +60 @@
 public class MultiFetchServerObjectReader extends OsmServerReader {
     /**
-     * the max. number of primitives retrieved in one step. Assuming IDs with 7 digits,
-     * this leads to a max. request URL of ~ 1600 Bytes ((7 digits +  1 Separator) * 200),
+     * the max. number of primitives retrieved in one step. Assuming IDs with 10 digits,
+     * this leads to a max. request URL of ~ 1900 Bytes ((10 digits +  1 Separator) * 170),
      * which should be safe according to the
      * <a href="http://www.boutell.com/newfaq/misc/urllength.html">WWW FAQ</a>.
      */
-    private static final int MAX_IDS_PER_REQUEST = 200;
+    private static final int MAX_IDS_PER_REQUEST = 170;
 
     private final Set<Long> nodes;
@@ -335 +335 @@
             try {
                 FetchResult result = ecs.take().get();
+                if (result.rc404 != null) {
+                    List<Long> toSplit = new ArrayList<>(result.rc404);
+                    int n = toSplit.size() / 2;
+                    jobs.add(ecs.submit(new Fetcher(type, new HashSet<>(toSplit.subList(0, n)), progressMonitor)));
+                    jobs.add(ecs.submit(new Fetcher(type, new HashSet<>(toSplit.subList(n, toSplit.size())), progressMonitor)));
+                }
                 if (result.missingPrimitives != null) {
                     missingPrimitives.addAll(result.missingPrimitives);
@@ -428 +434 @@
         public final Set<PrimitiveId> missingPrimitives;
 
+        private Set<Long> rc404;
+
         /**
          * Constructs a {@code FetchResult}
@@ -487 +495 @@
             } catch (OsmApiException e) {
                 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
+                    if (pkg.size() > 4) {
+                        FetchResult res = new FetchResult(null, null);
+                        res.rc404 = pkg;
+                        return res;
+                    }
                     Logging.info(tr("Server replied with response code 404, retrying with an individual request for each object."));
                     return singleGetIdPackage(type, pkg, progressMonitor);