Changeset 33074 in osm for applications/editors/josm/plugins/wikipedia

Timestamp:

2016-11-18T20:00:42+01:00 (8 years ago)

Author:

simon04

Message:

JOSM/wikipedia: avoid HTTP 414

File:

• applications/editors/josm/plugins/wikipedia/src/org/wikipedia/WikipediaApp.java (modified) (2 diffs)

applications/editors/josm/plugins/wikipedia/src/org/wikipedia/WikipediaApp.java

@@ -18 +18 @@
 import java.util.Objects;
 import java.util.TreeMap;
-import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -194 +195 @@
      */
     public static Map<String, String> getWikidataForArticles(String wikipediaLang, List<String> articles) {
-        if (articles.size() > 50) {
-            final List<String> withoutDuplicates = new ArrayList<>(new TreeSet<>(articles));
-            return partitionList(withoutDuplicates, 50).stream()
-                    .flatMap(chunk -> getWikidataForArticles(wikipediaLang, chunk).entrySet().stream())
-                    .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
-        } else if (articles.isEmpty()) {
+        return articles.stream()
+                .distinct()
+                .collect(Collectors.groupingBy(new Function<String, Integer>() {
+                    final AtomicInteger group = new AtomicInteger();
+                    final AtomicInteger count = new AtomicInteger();
+                    final AtomicInteger length = new AtomicInteger();
+
+                    @Override
+                    public Integer apply(String o) {
+                        // max. 50 titles, max. 2048 of URL encoded title chars (to avoid HTTP 414)
+                        if (count.incrementAndGet() > 50 || length.addAndGet(Utils.encodeUrl(o).length()) > 2048) {
+                            count.set(0);
+                            length.set(0);
+                            return group.incrementAndGet();
+                        } else {
+                            return group.get();
+                        }
+                    }
+                }))
+                .values()
+                .stream()
+                .flatMap(chunk -> getWikidataForArticles0(wikipediaLang, chunk).entrySet().stream())
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+    }
+
+    private static Map<String, String> getWikidataForArticles0(String wikipediaLang, List<String> articles) {
+        if (articles.isEmpty()) {
             return Collections.emptyMap();
         }