Changeset 32626 in osm for applications

Timestamp:

2016-07-10T22:57:01+02:00 (8 years ago)

Author:

simon04

Message:

JOSM/wikipedia: query Wikidata description - #josm12039

Location:

applications/editors/josm/plugins/wikipedia

Files:

• src/org/wikipedia/WikidataItemSearchDialog.java (modified) (2 diffs)
• src/org/wikipedia/WikipediaApp.java (modified) (10 diffs)
• test/unit/org/wikipedia/WikipediaAppTest.java (modified) (2 diffs)

applications/editors/josm/plugins/wikipedia/src/org/wikipedia/WikidataItemSearchDialog.java

@@ -10 +10 @@
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
@@ -103 +104 @@
                     final List<WikipediaApp.WikidataEntry> entries = query == null || query.isEmpty()
                             ? Collections.<WikipediaApp.WikidataEntry>emptyList()
-                            : WikipediaApp.getWikidataEntriesForQuery(WikipediaToggleDialog.wikipediaLang.get(), query);
+                            : WikipediaApp.getWikidataEntriesForQuery(WikipediaToggleDialog.wikipediaLang.get(), query, Locale.getDefault());
                     GuiHelper.runInEDT(new Runnable() {
                         @Override

applications/editors/josm/plugins/wikipedia/src/org/wikipedia/WikipediaApp.java

@@ -27 +27 @@
 import javax.xml.xpath.XPathConstants;
 import javax.xml.xpath.XPathExpression;
+import javax.xml.xpath.XPathExpressionException;
 import javax.xml.xpath.XPathFactory;
 
@@ -86 +87 @@
                 final Document doc = DOCUMENT_BUILDER.parse(in);
                 final NodeList nodes = (NodeList) xpathPlacemark.evaluate(doc, XPathConstants.NODESET);
-                final List<String> names = new ArrayList<>(nodes.getLength());
                 final List<WikipediaEntry> entries = new ArrayList<>(nodes.getLength());
                 for (int i = 0; i < nodes.getLength(); i++) {
                     final Node node = nodes.item(i);
                     final String name = xpathName.evaluate(node);
-                    names.add(name);
                     final LatLon latLon = new LatLon((
                             (double) xpathLat.evaluate(node, XPathConstants.NUMBER)),
                             (double) xpathLon.evaluate(node, XPathConstants.NUMBER));
                     if ("wikidata".equals(wikipediaLang)) {
-                        entries.add(new WikidataEntry(name, null, latLon));
+                        entries.add(new WikidataEntry(name, null, latLon, null));
                     } else {
                         entries.add(new WikipediaEntry(wikipediaLang, name, name, latLon
@@ -103 +102 @@
                 }
                 if ("wikidata".equals(wikipediaLang)) {
-                    final Map<String, String> labels = new HashMap<>();
-                    for (final List<String> chunk : partitionList(names, 50)) {
-                        labels.putAll(getLabelForWikidata(chunk, Locale.getDefault()));
-                    }
                     final List<WikipediaEntry> entriesWithLabel = new ArrayList<>(nodes.getLength());
-                    for (WikipediaEntry entry : entries) {
-                        entriesWithLabel.add(new WikidataEntry(entry.wikipediaArticle, labels.get(entry.wikipediaArticle), entry.coordinate));
+                    for (final List<WikipediaEntry> chunk : partitionList(entries, 50)) {
+                        entriesWithLabel.addAll(getLabelForWikidata(chunk, Locale.getDefault()));
                     }
                     return entriesWithLabel;
@@ -121 +116 @@
     }
 
-    static List<WikidataEntry> getWikidataEntriesForQuery(final String language, final String query) {
+    static List<WikidataEntry> getWikidataEntriesForQuery(final String languageForQuery, final String query, final Locale localeForLabels) {
         try {
             final String url = "https://www.wikidata.org/w/api.php" +
                     "?action=wbsearchentities" +
-                    "&language=" + language +
+                    "&language=" + languageForQuery +
                     "&strictlanguage=false" +
                     "&search=" + Utils.encodeUrl(query) +
@@ -135 +130 @@
                 final NodeList nodes = (NodeList) X_PATH.compile("//entity").evaluate(xml, XPathConstants.NODESET);
                 final XPathExpression xpathId = X_PATH.compile("@id");
-                final XPathExpression xpathLabel = X_PATH.compile("@label");
                 for (int i = 0; i < nodes.getLength(); i++) {
                     final Node node = nodes.item(i);
                     final String id = (String) xpathId.evaluate(node, XPathConstants.STRING);
-                    final String label = (String) xpathLabel.evaluate(node, XPathConstants.STRING);
-                    r.add(new WikidataEntry(id, label, null));
-                }
-            }
-            return r;
+                    r.add(new WikidataEntry(id, null, null, null));
+                }
+            }
+            return getLabelForWikidata(r, localeForLabels);
         } catch (Exception ex) {
             throw new RuntimeException(ex);
@@ -277 +270 @@
 
     static String getLabelForWikidata(String wikidataId, Locale locale, String ... preferredLanguage) {
-        return getLabelForWikidata(Collections.singleton(wikidataId), locale, preferredLanguage).get(wikidataId);
-    }
-
-    static Map<String, String> getLabelForWikidata(Collection<String> wikidataIds, Locale locale, String ... preferredLanguage) {
-        try {
-            for (final String wikidataId : wikidataIds) {
-                ensureValidWikidataId(wikidataId);
-            }
+        try {
+            return getLabelForWikidata(Collections.singleton(new WikidataEntry(wikidataId, null, null, null)), locale, preferredLanguage).get(0).label;
+        } catch (IndexOutOfBoundsException ignore) {
+            return null;
+        }
+    }
+
+    static List<WikidataEntry> getLabelForWikidata(Collection<? extends WikipediaEntry> entries, Locale locale, String ... preferredLanguage) {
+        final Collection<String> wikidataIds = Utils.transform(entries, new Function<WikipediaEntry, String>() {
+            @Override
+            public String apply(WikipediaEntry x) {
+                return x.wikipediaArticle;
+            }
+        });
+        try {
             final String url = "https://www.wikidata.org/w/api.php" +
                     "?action=wbgetentities" +
-                    "&props=labels" +
+                    "&props=labels|descriptions" +
                     "&ids=" + Utils.join("|", wikidataIds) +
                     "&format=xml";
@@ -298 +298 @@
             languages.add("en");
             languages.add(null);
-            final Map<String, String> r = new HashMap<>();
+            final List<WikidataEntry> r = new ArrayList<>(entries.size());
             try (final InputStream in = HttpClient.create(new URL(url)).setReasonForRequest("Wikipedia").connect().getContent()) {
                 final Document xml = DOCUMENT_BUILDER.parse(in);
-                for (final String wikidataId : wikidataIds) {
-                    final Node entity = (Node) X_PATH.compile("//entity[@id='" + wikidataId + "']").evaluate(xml, XPathConstants.NODE);
-                    for (String language : languages) {
-                        final String label = (String) X_PATH.compile(language != null
-                                ? "./labels/label[@language='" + language + "']/@value"
-                                : "./labels/label/@value"
-                        ).evaluate(entity, XPathConstants.STRING);
-                        if (label != null && !label.isEmpty()) {
-                            r.put(wikidataId, label);
-                            break;
-                        }
-                    }
+                for (final WikipediaEntry entry : entries) {
+                    final Node entity = (Node) X_PATH.compile("//entity[@id='" + entry.wikipediaArticle + "']").evaluate(xml, XPathConstants.NODE);
+                    r.add(new WikidataEntry(
+                            entry.wikipediaArticle,
+                            getFirstField(languages, "label", entity),
+                            entry.coordinate,
+                            getFirstField(languages, "description", entity)
+                    ));
                 }
             }
@@ -319 +315 @@
             throw new RuntimeException(ex);
         }
+    }
+
+    private static String getFirstField(Iterable<String> languages, String field, Node entity) throws XPathExpressionException {
+        for (String language : languages) {
+            final String label = (String) X_PATH.compile(language != null
+                    ? ".//" + field + "[@language='" + language + "']/@value"
+                    : ".//" + field + "/@value"
+            ).evaluate(entity, XPathConstants.STRING);
+            if (label != null && !label.isEmpty()) {
+                return label;
+            }
+        }
+        return null;
     }
 
@@ -486 +495 @@
     static class WikidataEntry extends WikipediaEntry {
 
-        WikidataEntry(String id, String label, LatLon coordinate) {
+        final String description;
+
+        WikidataEntry(String id, String label, LatLon coordinate, String description) {
             super("wikidata", id, label, coordinate);
+            this.description = description;
             ensureValidWikidataId(id);
         }
@@ -498 +510 @@
         @Override
         public String getLabelText() {
-            return getLabelText(label, wikipediaArticle);
+            final String descriptionInParen = description == null ? "" : (" (" + description + ")");
+            return getLabelText(label, wikipediaArticle + descriptionInParen);
         }
 

applications/editors/josm/plugins/wikipedia/test/unit/org/wikipedia/WikipediaAppTest.java

@@ -151 +151 @@
     @Test
     public void testForQuery() throws Exception {
-        final List<WikipediaApp.WikidataEntry> entries = WikipediaApp.getWikidataEntriesForQuery("de", "Österreich");
-        assertThat(entries.get(0).wikipediaArticle, is("Q40"));
-        assertThat(entries.get(0).wikipediaLang, is("wikidata"));
-        // assertThat(entries.get(0).label, is("Österreich"));
+        final List<WikipediaApp.WikidataEntry> de = WikipediaApp.getWikidataEntriesForQuery("de", "Österreich", Locale.GERMAN);
+        final List<WikipediaApp.WikidataEntry> en = WikipediaApp.getWikidataEntriesForQuery("de", "Österreich", Locale.ENGLISH);
+        assertThat(de.get(0).wikipediaArticle, is("Q40"));
+        assertThat(de.get(0).wikipediaLang, is("wikidata"));
+        assertThat(de.get(0).label, is("Österreich"));
+        assertThat(de.get(0).description, is("Staat in Mitteleuropa"));
+        assertThat(en.get(0).label, is("Austria"));
+        assertThat(en.get(0).description, is("country in Central Europe"));
     }
 
@@ -188 +192 @@
         // not found -> null
         assertThat(WikipediaApp.getLabelForWikidata("Q" + Long.MAX_VALUE, Locale.ENGLISH), nullValue());
-        final Map<String, String> twoLabels = WikipediaApp.getLabelForWikidata(Arrays.asList("Q84", "Q1741"), Locale.GERMAN);
-        assertThat(twoLabels.get("Q84"), is("London"));
-        assertThat(twoLabels.get("Q1741"), is("Wien"));
+        final WikipediaApp.WikidataEntry q84 = new WikipediaApp.WikidataEntry("Q84", null, null, null);
+        final WikipediaApp.WikidataEntry q1741 = new WikipediaApp.WikidataEntry("Q1741", null, null, null);
+        final List<WikipediaApp.WikidataEntry> twoLabels = WikipediaApp.getLabelForWikidata(Arrays.asList(q84, q1741), Locale.GERMAN);
+        assertThat(twoLabels.get(0).label, is("London"));
+        assertThat(twoLabels.get(1).label, is("Wien"));
     }