Changeset 33074 in osm
- Timestamp:
- 2016-11-18T20:00:42+01:00 (8 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
applications/editors/josm/plugins/wikipedia/src/org/wikipedia/WikipediaApp.java
r33070 r33074 18 18 import java.util.Objects; 19 19 import java.util.TreeMap; 20 import java.util.TreeSet; 20 import java.util.concurrent.atomic.AtomicInteger; 21 import java.util.function.Function; 21 22 import java.util.regex.Pattern; 22 23 import java.util.stream.Collectors; … … 194 195 */ 195 196 public static Map<String, String> getWikidataForArticles(String wikipediaLang, List<String> articles) { 196 if (articles.size() > 50) { 197 final List<String> withoutDuplicates = new ArrayList<>(new TreeSet<>(articles)); 198 return partitionList(withoutDuplicates, 50).stream() 199 .flatMap(chunk -> getWikidataForArticles(wikipediaLang, chunk).entrySet().stream()) 200 .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); 201 } else if (articles.isEmpty()) { 197 return articles.stream() 198 .distinct() 199 .collect(Collectors.groupingBy(new Function<String, Integer>() { 200 final AtomicInteger group = new AtomicInteger(); 201 final AtomicInteger count = new AtomicInteger(); 202 final AtomicInteger length = new AtomicInteger(); 203 204 @Override 205 public Integer apply(String o) { 206 // max. 50 titles, max. 2048 of URL encoded title chars (to avoid HTTP 414) 207 if (count.incrementAndGet() > 50 || length.addAndGet(Utils.encodeUrl(o).length()) > 2048) { 208 count.set(0); 209 length.set(0); 210 return group.incrementAndGet(); 211 } else { 212 return group.get(); 213 } 214 } 215 })) 216 .values() 217 .stream() 218 .flatMap(chunk -> getWikidataForArticles0(wikipediaLang, chunk).entrySet().stream()) 219 .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); 220 } 221 222 private static Map<String, String> getWikidataForArticles0(String wikipediaLang, List<String> articles) { 223 if (articles.isEmpty()) { 202 224 return Collections.emptyMap(); 203 225 }
Note:
See TracChangeset
for help on using the changeset viewer.