Changeset 15470 in josm

Timestamp:

2019-10-22T23:54:14+02:00 (5 years ago)

Author:

Don-vip

Message:

fix #18249 - Allow unknown xml attributes to be added as tags (patch by taylor.smock)

Location:

trunk

Files:

• src/org/openstreetmap/josm/io/OsmReader.java (modified) (5 diffs)
• test/unit/org/openstreetmap/josm/io/OsmReaderTest.java (modified) (7 diffs)

trunk/src/org/openstreetmap/josm/io/OsmReader.java

@@ -7 +7 @@
 import java.util.Collection;
 import java.util.Objects;
+import java.util.Set;
+import java.util.TreeSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -40 +42 @@
     protected XMLStreamReader parser;
 
+    protected boolean convertUnknownToTags;
+
+    private static final Set<String> COMMON_XML_ATTRIBUTES = new TreeSet<>();
+
+    static {
+        COMMON_XML_ATTRIBUTES.add("id");
+        COMMON_XML_ATTRIBUTES.add("timestamp");
+        COMMON_XML_ATTRIBUTES.add("user");
+        COMMON_XML_ATTRIBUTES.add("uid");
+        COMMON_XML_ATTRIBUTES.add("visible");
+        COMMON_XML_ATTRIBUTES.add("version");
+        COMMON_XML_ATTRIBUTES.add("action");
+        COMMON_XML_ATTRIBUTES.add("changeset");
+        COMMON_XML_ATTRIBUTES.add("lat");
+        COMMON_XML_ATTRIBUTES.add("lon");
+    }
+
     /**
      * constructor (for private and subclasses use only)
@@ -46 +65 @@
      */
     protected OsmReader() {
+        this(false);
+    }
+
+    /**
+     * constructor (for private and subclasses use only)
+     * @param convertUnknownToTags if true, keep unknown xml attributes as tags
+     *
+     * @see #parseDataSet(InputStream, ProgressMonitor)
+     * @since 15470
+     */
+    protected OsmReader(boolean convertUnknownToTags) {
         // Restricts visibility
+        this.convertUnknownToTags = convertUnknownToTags;
     }
 
@@ -394 +425 @@
             parseAction(current, parser.getAttributeValue(null, "action"));
             parseChangeset(current, parser.getAttributeValue(null, "changeset"));
+
+            if (convertUnknownToTags) {
+                for (int i = 0; i < parser.getAttributeCount(); i++) {
+                    if (!COMMON_XML_ATTRIBUTES.contains(parser.getAttributeLocalName(i))) {
+                        parseTag(current, parser.getAttributeLocalName(i), parser.getAttributeValue(i));
+                    }
+                }
+            }
         } catch (UncheckedParseException | XMLStreamException e) {
             throw new IllegalDataException(e);
@@ -458 +497 @@
      */
     public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
-        return new OsmReader().doParseDataSet(source, progressMonitor);
+        return parseDataSet(source, progressMonitor, false);
+    }
+
+    /**
+     * Parse the given input source and return the dataset.
+     *
+     * @param source the source input stream. Must not be null.
+     * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
+     * @param convertUnknownToTags true if unknown xml attributes should be kept as tags
+     *
+     * @return the dataset with the parsed data
+     * @throws IllegalDataException if an error was found while parsing the data from the source
+     * @throws IllegalArgumentException if source is null
+     * @since 15470
+     */
+    public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor, boolean convertUnknownToTags)
+            throws IllegalDataException {
+        return new OsmReader(convertUnknownToTags).doParseDataSet(source, progressMonitor);
     }
 }

trunk/test/unit/org/openstreetmap/josm/io/OsmReaderTest.java

@@ -13 +13 @@
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.Arrays;
 
 import org.junit.Rule;
@@ -18 +19 @@
 import org.openstreetmap.josm.TestUtils;
 import org.openstreetmap.josm.data.osm.DataSet;
+import org.openstreetmap.josm.data.osm.Node;
 import org.openstreetmap.josm.data.osm.Way;
 import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
@@ -74 +76 @@
             assertTrue(OsmReader.parseDataSet(in, NullProgressMonitor.INSTANCE).allPrimitives().isEmpty());
         }
+        testUnknown(osm, true);
+        testUnknown(osm, false);
+    }
+
+    private static void testUnknown(String osm, boolean parseUnknownAttributes) throws Exception {
+        try (InputStream in = new ByteArrayInputStream(
+                ("<?xml version='1.0' encoding='UTF-8'?>" + osm).getBytes(StandardCharsets.UTF_8))) {
+            assertTrue(OsmReader.parseDataSet(in, NullProgressMonitor.INSTANCE, parseUnknownAttributes).allPrimitives()
+                    .isEmpty());
+        }
     }
 
@@ -127 +139 @@
 
     /**
+     * Test valid data.
+     * @param osm OSM data without XML prefix
+     * @param parseUnknownAttributes if true, attempt to parse unknown xml attributes
+     * @return parsed data set
+     * @throws Exception if any error occurs
+     */
+    private static DataSet testValidData(String osm, boolean parseUnknownAttributes) throws Exception {
+        try (InputStream in = new ByteArrayInputStream(
+                ("<?xml version='1.0' encoding='UTF-8'?>" + osm).getBytes(StandardCharsets.UTF_8))) {
+            return OsmReader.parseDataSet(in, NullProgressMonitor.INSTANCE, parseUnknownAttributes);
+        }
+    }
+
+    /**
      * Test invalid data.
      * @param osm OSM data without XML prefix
@@ -136 +162 @@
                 ("<?xml version='1.0' encoding='UTF-8'?>" + osm).getBytes(StandardCharsets.UTF_8))) {
             OsmReader.parseDataSet(in, NullProgressMonitor.INSTANCE);
+            fail("should throw exception");
+        } catch (IllegalDataException e) {
+            assertEquals(expectedError, e.getMessage());
+        }
+        testInvalidData(osm, expectedError, true);
+        testInvalidData(osm, expectedError, false);
+    }
+
+    /**
+     * Test invalid data.
+     *
+     * @param osm                    OSM data without XML prefix
+     * @param expectedError          expected error message
+     * @param parseUnknownAttributes if true, attempt to parse unknown xml
+     *                               attributes
+     * @throws Exception if any error occurs
+     */
+    private static void testInvalidData(String osm, String expectedError, boolean parseUnknownAttributes)
+            throws Exception {
+        try (InputStream in = new ByteArrayInputStream(
+                ("<?xml version='1.0' encoding='UTF-8'?>" + osm).getBytes(StandardCharsets.UTF_8))) {
+            OsmReader.parseDataSet(in, NullProgressMonitor.INSTANCE, parseUnknownAttributes);
             fail("should throw exception");
         } catch (IllegalDataException e) {
@@ -275 +323 @@
     @Test
     public void testGdprChangeset() throws Exception {
-        testValidData("<osm version='0.6'><node id='1' version='1' changeset='0'/></osm>");
+        String gdprChangeset = "<osm version='0.6'><node id='1' version='1' changeset='0'/></osm>";
+        testValidData(gdprChangeset);
+        testValidData(gdprChangeset, true);
+        testValidData(gdprChangeset, false);
     }
 
@@ -350 +401 @@
     @Test
     public void testRemark() throws Exception {
-        DataSet ds = testValidData(
-                "<osm version=\"0.6\" generator=\"Overpass API 0.7.55.4 3079d8ea\">\r\n" +
+        String query = "<osm version=\"0.6\" generator=\"Overpass API 0.7.55.4 3079d8ea\">\r\n" +
                 "<note>The data included in this document is from www.openstreetmap.org. The data is made available under ODbL.</note>\r\n" +
                 "<meta osm_base=\"2018-08-30T12:46:02Z\" areas=\"2018-08-30T12:40:02Z\"/>\r\n" +
                 "<remark>runtime error: Query ran out of memory in \"query\" at line 5.</remark>\r\n" +
-                "</osm>");
-        assertEquals("runtime error: Query ran out of memory in \"query\" at line 5.", ds.getRemark());
+                "</osm>";
+        for (DataSet ds : Arrays.asList(testValidData(query), testValidData(query, true), testValidData(query, false))) {
+            assertEquals("runtime error: Query ran out of memory in \"query\" at line 5.", ds.getRemark());
+        }
+    }
+
+    /**
+     * Test reading a file with unknown attributes in osm primitives
+     * @throws Exception if any error occurs
+     */
+    @Test
+    public void testUnknownAttributeTags() throws Exception {
+        String testData = "<osm version=\"0.6\" generator=\"fake generator\">"
+                + "<node id='1' version='1' visible='true' changeset='82' randomkey='randomvalue'></node>" + "</osm>";
+        DataSet ds = testValidData(testData);
+        assertEquals(0, ds.getNodes().iterator().next().getKeys().size());
+
+        ds = testValidData(testData, true);
+        Node firstNode = ds.getNodes().iterator().next();
+        assertEquals(1, firstNode.getKeys().size());
+        assertEquals("randomvalue", firstNode.get("randomkey"));
+
+        ds = testValidData(testData, false);
+        assertEquals(0, ds.getNodes().iterator().next().getKeys().size());
     }
 }