- Timestamp:
- 2023-03-20T21:52:18+01:00 (19 months ago)
- Location:
- trunk
- Files:
-
- 11 added
- 11 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/src/org/openstreetmap/josm/actions/ExtensionFileFilter.java
r18179 r18695 28 28 import org.openstreetmap.josm.gui.io.importexport.OsmChangeImporter; 29 29 import org.openstreetmap.josm.gui.io.importexport.OsmImporter; 30 import org.openstreetmap.josm.gui.io.importexport.OsmPbfImporter; 30 31 import org.openstreetmap.josm.gui.io.importexport.OziWptImporter; 31 32 import org.openstreetmap.josm.gui.io.importexport.RtkLibImporter; … … 66 67 OsmImporter.class, 67 68 OsmChangeImporter.class, 69 OsmPbfImporter.class, 68 70 GeoJSONImporter.class, 69 71 GpxImporter.class, -
trunk/src/org/openstreetmap/josm/data/Bounds.java
r17703 r18695 7 7 import java.text.DecimalFormat; 8 8 import java.text.MessageFormat; 9 import java.util. Objects;9 import java.util.Arrays; 10 10 11 11 import org.openstreetmap.josm.data.coor.ILatLon; … … 593 593 @Override 594 594 public int hashCode() { 595 return Objects.hash(minLat, minLon, maxLat, maxLon);595 return Arrays.hashCode(new double[] {minLat, minLon, maxLat, maxLon}); 596 596 } 597 597 -
trunk/src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/CommandInteger.java
r18431 r18695 51 51 52 52 /** 53 * Add a parameter 54 * @param parameterInteger The parameter to add (converted to {@code short}). 55 * @since xxx 56 */ 57 public void addParameter(long parameterInteger) { 58 this.parameters[added++] = (short) parameterInteger; 59 } 60 61 /** 53 62 * Get the operations for the command 54 63 * @return The operations -
trunk/src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Feature.java
r18485 r18695 28 28 /** 29 29 * The number format instance to use (using a static instance gets rid of quite o few allocations) 30 * Doing this reduced the allocations of {@link #parseTagValue(String, Layer, Number, List)} from 22.79% of parent to30 * Doing this reduced the allocations of {@link #parseTagValue(String, Layer, int, List)} from 22.79% of parent to 31 31 * 12.2% of parent. 32 32 */ … … 75 75 if (next.getField() == TAG_FIELD) { 76 76 // This is packed in v1 and v2 77 ProtobufPacked packed = new ProtobufPacked( byteArrayOutputStream,next.getBytes());77 ProtobufPacked packed = new ProtobufPacked(next.getBytes()); 78 78 if (tagList == null) { 79 79 tagList = new ArrayList<>(packed.getArray().length); … … 81 81 tagList.ensureCapacity(tagList.size() + packed.getArray().length); 82 82 } 83 for ( Numbernumber : packed.getArray()) {84 key = parseTagValue(key, layer, number, tagList);83 for (long number : packed.getArray()) { 84 key = parseTagValue(key, layer, (int) number, tagList); 85 85 } 86 86 } else if (next.getField() == GEOMETRY_FIELD) { 87 87 // This is packed in v1 and v2 88 ProtobufPacked packed = new ProtobufPacked( byteArrayOutputStream,next.getBytes());88 ProtobufPacked packed = new ProtobufPacked(next.getBytes()); 89 89 CommandInteger currentCommand = null; 90 for ( Numbernumber : packed.getArray()) {90 for (long number : packed.getArray()) { 91 91 if (currentCommand != null && currentCommand.hasAllExpectedParameters()) { 92 92 currentCommand = null; 93 93 } 94 94 if (currentCommand == null) { 95 currentCommand = new CommandInteger( number.intValue());95 currentCommand = new CommandInteger(Math.toIntExact(number)); 96 96 this.geometry.add(currentCommand); 97 97 } else { … … 128 128 * @return The new key (if {@code null}, then a value was parsed and added to tags) 129 129 */ 130 private static String parseTagValue(String key, Layer layer, Numbernumber, List<String> tagList) {130 private static String parseTagValue(String key, Layer layer, int number, List<String> tagList) { 131 131 if (key == null) { 132 key = layer.getKey(number .intValue());132 key = layer.getKey(number); 133 133 } else { 134 134 tagList.add(key); 135 Object value = layer.getValue(number .intValue());135 Object value = layer.getValue(number); 136 136 if (value instanceof Double || value instanceof Float) { 137 137 // reset grouping if the instance is a singleton -
trunk/src/org/openstreetmap/josm/data/protobuf/ProtobufPacked.java
r18473 r18695 3 3 4 4 import java.io.ByteArrayOutputStream; 5 import java.util.ArrayList; 6 import java.util.List; 5 import java.util.Arrays; 7 6 8 7 /** … … 13 12 */ 14 13 public class ProtobufPacked { 15 private static final Number[] NO_NUMBERS = new Number[0];16 14 private final byte[] bytes; 17 private final Number[] numbers;15 private final long[] numbers; 18 16 private int location; 19 17 … … 21 19 * Create a new ProtobufPacked object 22 20 * 23 * @param byteArrayOutputStream A reusable ByteArrayOutputStream (helps to reduce memory allocations)21 * @param ignored A reusable ByteArrayOutputStream (no longer used) 24 22 * @param bytes The packed bytes 23 * @deprecated since we aren't using the output stream anymore 25 24 */ 26 public ProtobufPacked(ByteArrayOutputStream byteArrayOutputStream, byte[] bytes) { 25 @Deprecated 26 public ProtobufPacked(ByteArrayOutputStream ignored, byte[] bytes) { 27 this(bytes); 28 } 29 30 /** 31 * Create a new ProtobufPacked object 32 * 33 * @param bytes The packed bytes 34 * @since 18695 35 */ 36 public ProtobufPacked(byte[] bytes) { 27 37 this.location = 0; 28 38 this.bytes = bytes; … … 31 41 // only adds 3.7 MB to the ArrayList#init calls. Note that the real-world test case (Mapillary vector tiles) 32 42 // primarily created Shorts. 33 List<Number> numbersT = new ArrayList<>(bytes.length); 43 long[] numbersT = new long[bytes.length]; 44 int index = 0; 34 45 // By reusing a ByteArrayOutputStream, we can reduce allocations in nextVarInt from 230 MB to 74 MB. 35 46 while (this.location < bytes.length) { 36 numbersT.add(ProtobufParser.convertByteArray(this.nextVarInt(byteArrayOutputStream), ProtobufParser.VAR_INT_BYTE_SIZE)); 47 int start = this.location; 48 numbersT[index] = ProtobufParser.convertByteArray(this.bytes, ProtobufParser.VAR_INT_BYTE_SIZE, 49 start, this.nextVarInt()); 50 index++; 37 51 } 38 52 39 this.numbers = numbersT.toArray(NO_NUMBERS); 53 if (numbersT.length == index) { 54 this.numbers = numbersT; 55 } else { 56 this.numbers = Arrays.copyOf(numbersT, index); 57 } 40 58 } 41 59 … … 45 63 * @return The number array 46 64 */ 47 public Number[] getArray() {65 public long[] getArray() { 48 66 return this.numbers; 49 67 } 50 68 51 private byte[] nextVarInt(final ByteArrayOutputStream byteArrayOutputStream) {52 // In a real world test, the largest List<Byte> seen had 3 elements. Use 4 to avoid most new array allocations.53 // Memory allocations went from 368 MB to 280 MB by using an initial array allocation. When using a54 // ByteArrayOutputStream, it went down to 230 MB. By further reusing the ByteArrayOutputStream between method55 // calls, it went down further to 73 MB.69 /** 70 * Gets the location where the next var int begins. Note: changes {@link ProtobufPacked#location}. 71 * @return The next varint location 72 */ 73 private int nextVarInt() { 56 74 while ((this.bytes[this.location] & ProtobufParser.MOST_SIGNIFICANT_BYTE) 57 75 == ProtobufParser.MOST_SIGNIFICANT_BYTE) { 58 76 // Get rid of the leading bit (shift left 1, then shift right 1 unsigned) 59 byteArrayOutputStream.write(this.bytes[this.location++] ^ ProtobufParser.MOST_SIGNIFICANT_BYTE); 77 this.bytes[this.location] = (byte) (this.bytes[this.location] ^ ProtobufParser.MOST_SIGNIFICANT_BYTE); 78 this.location++; 60 79 } 61 // The last byte doesn't drop the most significant bit 62 byteArrayOutputStream.write(this.bytes[this.location++]); 63 try { 64 return byteArrayOutputStream.toByteArray(); 65 } finally { 66 byteArrayOutputStream.reset(); 67 } 80 return ++this.location; 68 81 } 69 82 } -
trunk/src/org/openstreetmap/josm/data/protobuf/ProtobufParser.java
r18473 r18695 8 8 import java.io.InputStream; 9 9 import java.util.ArrayList; 10 import java.util.Arrays; 10 11 import java.util.Collection; 11 12 … … 31 32 */ 32 33 static final byte MOST_SIGNIFICANT_BYTE = (byte) (1 << 7); 34 private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; 35 33 36 /** 34 37 * Convert a byte array to a number (little endian) … … 39 42 */ 40 43 public static Number convertByteArray(byte[] bytes, byte byteSize) { 44 return convertLong(convertByteArray(bytes, byteSize, 0, bytes.length)); 45 } 46 47 /** 48 * Convert a byte array to a number (little endian) 49 * 50 * @param bytes The bytes to convert 51 * @param byteSize The size of the byte. For var ints, this is 7, for other ints, this is 8. 52 * @param start The start position in the byte array 53 * @param end The end position in the byte array (exclusive - [start, end) ) 54 * @return t 55 * he number from the byte array. Depending upon length of time the number will be stored, narrowing may be helpful. 56 * @since 18695 57 */ 58 public static long convertByteArray(byte[] bytes, byte byteSize, int start, int end) { 41 59 long number = 0; 42 for (int i = 0; i < bytes.length; i++) {60 for (int i = start; i < end; i++) { 43 61 // Need to convert to uint64 in order to avoid bit operation from filling in 1's and overflow issues 44 number += Byte.toUnsignedLong(bytes[i]) << (byteSize * i);45 } 46 return convertLong(number);62 number += Byte.toUnsignedLong(bytes[i]) << (byteSize * (i - start)); 63 } 64 return number; 47 65 } 48 66 … … 72 90 */ 73 91 public static Number decodeZigZag(Number signed) { 74 final long value = signed.longValue(); 75 return convertLong((value >> 1) ^ -(value & 1)); 92 return convertLong(decodeZigZag(signed.longValue())); 93 } 94 95 /** 96 * Decode a zig-zag encoded value 97 * 98 * @param signed The value to decode 99 * @return The decoded value 100 * @since 18695 101 */ 102 public static long decodeZigZag(long signed) { 103 return (signed >> 1) ^ -(signed & 1); 76 104 } 77 105 … … 204 232 */ 205 233 public byte[] nextLengthDelimited(ByteArrayOutputStream byteArrayOutputStream) throws IOException { 206 int length = convertByteArray(this.nextVarInt(byteArrayOutputStream), VAR_INT_BYTE_SIZE).intValue(); 234 final byte[] nextVarInt = this.nextVarInt(byteArrayOutputStream); 235 int length = (int) convertByteArray(nextVarInt, VAR_INT_BYTE_SIZE, 0, nextVarInt.length); 207 236 return readNextBytes(length); 208 237 } … … 237 266 * 238 267 * @param size The number of bytes to read 239 * @return a byte array of the specified size,filled with bytes read (unsigned)268 * @return a byte array filled with bytes read (unsigned) 240 269 * @throws IOException - if an IO error occurs 241 270 */ 242 271 private byte[] readNextBytes(int size) throws IOException { 243 272 byte[] bytesRead = new byte[size]; 244 for (int i = 0; i < bytesRead.length; i++) { 245 bytesRead[i] = (byte) this.nextByte(); 273 int read = this.inputStream.read(bytesRead); 274 if (read == -1) { 275 return EMPTY_BYTE_ARRAY; 276 } else if (read != size) { 277 return Arrays.copyOf(bytesRead, read); 246 278 } 247 279 return bytesRead; -
trunk/src/org/openstreetmap/josm/data/protobuf/ProtobufRecord.java
r18473 r18695 28 28 */ 29 29 public ProtobufRecord(ByteArrayOutputStream byteArrayOutputStream, ProtobufParser parser) throws IOException { 30 Number number = ProtobufParser.convertByteArray(parser.nextVarInt(byteArrayOutputStream), ProtobufParser.VAR_INT_BYTE_SIZE); 30 final byte[] varInt = parser.nextVarInt(byteArrayOutputStream); 31 long number = ProtobufParser.convertByteArray(varInt, ProtobufParser.VAR_INT_BYTE_SIZE, 0, varInt.length); 31 32 // I don't foresee having field numbers > {@code Integer#MAX_VALUE >> 3} 32 this.field = (int) number .longValue()>> 3;33 this.field = (int) number >> 3; 33 34 // 7 is 111 (so last three bits) 34 byte wireType = (byte) (number .longValue()& 7);35 byte wireType = (byte) (number & 7); 35 36 // By not using a stream, we reduce the number of allocations (for getting the WireType) from 257 MB to 40 MB. 36 37 // (The remaining 40 MB is from WireType#values). By using the cached getAllValues(), we drop the 40 MB. -
trunk/src/org/openstreetmap/josm/io/AbstractReader.java
r18208 r18695 274 274 protected abstract DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException; 275 275 276 /** 277 * An interface for reading binary data 278 * @since xxx 279 */ 280 @FunctionalInterface 281 protected interface BinaryParserWorker { 282 /** 283 * Effectively parses the file, depending on the binary format (PBF, etc.) 284 * @param ir input stream reader 285 * @throws IllegalDataException in case of invalid data 286 * @throws IOException in case of I/O error 287 */ 288 void accept(InputStream ir) throws IllegalDataException, IOException; 289 } 290 276 291 @FunctionalInterface 277 292 protected interface ParserWorker { … … 285 300 } 286 301 302 protected final DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor, BinaryParserWorker parserWorker) 303 throws IllegalDataException { 304 return this.doParseDataSet(source, progressMonitor, (Object) parserWorker); 305 } 306 287 307 protected final DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor, ParserWorker parserWorker) 308 throws IllegalDataException { 309 return this.doParseDataSet(source, progressMonitor, (Object) parserWorker); 310 } 311 312 private DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor, Object parserWorker) 288 313 throws IllegalDataException { 289 314 if (progressMonitor == null) { … … 297 322 progressMonitor.indeterminateSubTask(tr("Parsing OSM data...")); 298 323 299 try (InputStreamReader ir = UTFInputStreamReader.create(source)) { 300 parserWorker.accept(ir); 324 if (parserWorker instanceof ParserWorker) { 325 try (InputStreamReader ir = UTFInputStreamReader.create(source)) { 326 ((ParserWorker) parserWorker).accept(ir); 327 } 328 } else if (parserWorker instanceof BinaryParserWorker) { 329 ((BinaryParserWorker) parserWorker).accept(source); 330 } else { 331 throw new IllegalArgumentException("Unknown parser worker type: " + parserWorker.getClass()); 301 332 } 302 333 progressMonitor.worked(1); -
trunk/src/org/openstreetmap/josm/io/OsmJsonReader.java
r18658 r18695 183 183 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 184 184 try { 185 return doParseDataSet(source, progressMonitor, ir -> {185 return doParseDataSet(source, progressMonitor, (ParserWorker) ir -> { 186 186 setParser(Json.createParser(ir)); 187 187 parse(); -
trunk/src/org/openstreetmap/josm/io/OsmReader.java
r18541 r18695 502 502 @Override 503 503 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 504 return doParseDataSet(source, progressMonitor, ir -> {504 return doParseDataSet(source, progressMonitor, (ParserWorker) ir -> { 505 505 try { 506 506 setParser(XmlUtils.newSafeXMLInputFactory().createXMLStreamReader(ir)); -
trunk/test/unit/org/openstreetmap/josm/data/protobuf/ProtobufTest.java
r18473 r18695 43 43 * @since 17862 44 44 */ 45 class ProtobufTest {45 public class ProtobufTest { 46 46 /** 47 47 * Convert an int array into a byte array … … 49 49 * @return A byte array that can be used 50 50 */ 51 static byte[] toByteArray(int[] intArray) {51 public static byte[] toByteArray(int[] intArray) { 52 52 byte[] byteArray = new byte[intArray.length]; 53 53 for (int i = 0; i < intArray.length; i++) { … … 202 202 @Test 203 203 void testZigZag() { 204 assertEquals(0, ProtobufParser.decodeZigZag( 0).intValue());205 assertEquals(-1, ProtobufParser.decodeZigZag( 1).intValue());206 assertEquals(1, ProtobufParser.decodeZigZag( 2).intValue());207 assertEquals(-2, ProtobufParser.decodeZigZag( 3).intValue());204 assertEquals(0, ProtobufParser.decodeZigZag(Integer.valueOf(0)).intValue()); 205 assertEquals(-1, ProtobufParser.decodeZigZag(Integer.valueOf(1)).intValue()); 206 assertEquals(1, ProtobufParser.decodeZigZag(Long.valueOf(2)).intValue()); 207 assertEquals(-2, ProtobufParser.decodeZigZag(Long.valueOf(3)).intValue()); 208 208 } 209 209 }
Note:
See TracChangeset
for help on using the changeset viewer.