source: josm/trunk/src/org/openstreetmap/josm/io/OsmPbfReader.java@ 19038

Last change on this file since 19038 was 19038, checked in by stoecker, 8 weeks ago

checkstyle
File size: 46.6 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import java.io.BufferedInputStream;
5import java.io.ByteArrayInputStream;
6import java.io.ByteArrayOutputStream;
7import java.io.IOException;
8import java.io.InputStream;
9import java.util.ArrayList;
10import java.util.Arrays;
11import java.util.HashMap;
12import java.util.HashSet;
13import java.util.List;
14import java.util.Locale;
15import java.util.Map;
16import java.util.Set;
17
18import org.apache.commons.compress.utils.CountingInputStream;
19import org.openstreetmap.josm.data.Bounds;
20import org.openstreetmap.josm.data.DataSource;
21import org.openstreetmap.josm.data.coor.LatLon;
22import org.openstreetmap.josm.data.osm.BBox;
23import org.openstreetmap.josm.data.osm.DataSet;
24import org.openstreetmap.josm.data.osm.NodeData;
25import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
26import org.openstreetmap.josm.data.osm.PrimitiveData;
27import org.openstreetmap.josm.data.osm.RelationData;
28import org.openstreetmap.josm.data.osm.RelationMemberData;
29import org.openstreetmap.josm.data.osm.Tagged;
30import org.openstreetmap.josm.data.osm.UploadPolicy;
31import org.openstreetmap.josm.data.osm.User;
32import org.openstreetmap.josm.data.osm.WayData;
33import org.openstreetmap.josm.data.osm.pbf.Blob;
34import org.openstreetmap.josm.data.osm.pbf.BlobHeader;
35import org.openstreetmap.josm.data.osm.pbf.HeaderBlock;
36import org.openstreetmap.josm.data.osm.pbf.Info;
37import org.openstreetmap.josm.data.protobuf.ProtobufPacked;
38import org.openstreetmap.josm.data.protobuf.ProtobufParser;
39import org.openstreetmap.josm.data.protobuf.ProtobufRecord;
40import org.openstreetmap.josm.data.protobuf.WireType;
41import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
42import org.openstreetmap.josm.gui.progress.ProgressMonitor;
43import org.openstreetmap.josm.tools.Utils;
44
45import jakarta.annotation.Nonnull;
46import jakarta.annotation.Nullable;
47
48/**
49 * Read OSM data from an OSM PBF file
50 * @since 18695
51 */
52public final class OsmPbfReader extends AbstractReader {
53 private static final long[] EMPTY_LONG = new long[0];
54 /**
55 * Nano degrees
56 */
57 private static final double NANO_DEGREES = 1e-9;
58 /**
59 * The maximum BlobHeader size. BlobHeaders should (but not must) be less than half this
60 */
61 private static final int MAX_BLOBHEADER_SIZE = 64 * 1024;
62 /**
63 * The maximum Blob size. Blobs should (but not must) be less than half this
64 */
65 private static final int MAX_BLOB_SIZE = 32 * 1024 * 1024;
66
67 private OsmPbfReader() {
68 // Hide constructor
69 }
70
71 /**
72 * Parse the given input source and return the dataset.
73 *
74 * @param source the source input stream. Must not be null.
75 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
76 * @return the dataset with the parsed data
77 * @throws IllegalDataException if an error was found while parsing the data from the source
78 * @throws IllegalArgumentException if source is null
79 */
80 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
81 return new OsmPbfReader().doParseDataSet(source, progressMonitor);
82 }
83
84 @Override
85 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
86 return doParseDataSet(source, progressMonitor, this::parse);
87 }
88
89 private void parse(InputStream source) throws IllegalDataException, IOException {
90 final CountingInputStream inputStream;
91 if (source.markSupported()) {
92 inputStream = new CountingInputStream(source);
93 } else {
94 inputStream = new CountingInputStream(new BufferedInputStream(source));
95 }
96 try (ProtobufParser parser = new ProtobufParser(inputStream)) {
97 final ByteArrayOutputStream baos = new ByteArrayOutputStream();
98 HeaderBlock headerBlock = null;
99 BlobHeader blobHeader = null;
100 while (parser.hasNext() && !this.cancel) {
101 if (blobHeader == null) {
102 blobHeader = parseBlobHeader(inputStream, baos, parser);
103 } else if ("OSMHeader".equals(blobHeader.type())) {
104 if (headerBlock != null) {
105 throw new IllegalDataException("Too many header blocks in protobuf");
106 }
107 // OSM PBF is fun -- it has *nested* pbf data
108 Blob blob = parseBlob(blobHeader, inputStream, parser, baos);
109 headerBlock = parseHeaderBlock(blob, baos);
110 checkRequiredFeatures(headerBlock);
111 blobHeader = null;
112 } else if ("OSMData".equals(blobHeader.type())) {
113 if (headerBlock == null) {
114 throw new IllegalStateException("A header block must occur before the first data block");
115 }
116 Blob blob = parseBlob(blobHeader, inputStream, parser, baos);
117 parseDataBlock(baos, headerBlock, blob);
118 blobHeader = null;
119 } // Other software *may* extend the FileBlocks (from just "OSMHeader" and "OSMData"), so don't throw an error.
120 }
121 }
122 }
123
124 /**
125 * Parse a blob header
126 *
127 * @param cis A counting stream to ensure we don't read too much data
128 * @param baos A reusable stream
129 * @param parser The parser to read from
130 * @return The BlobHeader message
131 * @throws IOException if one of the streams has an issue
132 * @throws IllegalDataException If the OSM PBF is (probably) corrupted
133 */
134 @Nonnull
135 private static BlobHeader parseBlobHeader(CountingInputStream cis, ByteArrayOutputStream baos, ProtobufParser parser)
136 throws IOException, IllegalDataException {
137 String type = null;
138 byte[] indexData = null;
139 int datasize = Integer.MIN_VALUE;
140 int length = 0;
141 long start = cis.getBytesRead();
142 while (parser.hasNext() && (length == 0 || cis.getBytesRead() - start < length)) {
143 final ProtobufRecord current = new ProtobufRecord(baos, parser);
144 switch (current.getField()) {
145 case 1:
146 type = current.asString();
147 break;
148 case 2:
149 indexData = current.getBytes();
150 break;
151 case 3:
152 datasize = current.asUnsignedVarInt().intValue();
153 break;
154 default:
155 start = cis.getBytesRead();
156 length += current.asUnsignedVarInt().intValue();
157 if (length > MAX_BLOBHEADER_SIZE) { // There is a hard limit of 64 KiB for the BlobHeader. It *should* be less than 32 KiB.
158 throw new IllegalDataException("OSM PBF BlobHeader is too large. PBF is probably corrupted. (" +
159 Utils.getSizeString(MAX_BLOBHEADER_SIZE, Locale.ENGLISH) + " < " + Utils.getSizeString(length, Locale.ENGLISH));
160 }
161 }
162 }
163 if (type == null || Integer.MIN_VALUE == datasize) {
164 throw new IllegalDataException("OSM PBF BlobHeader could not be read. PBF is probably corrupted.");
165 } else if (datasize > MAX_BLOB_SIZE) { // There is a hard limit of 32 MiB for the blob size. It *should* be less than 16 MiB.
166 throw new IllegalDataException("OSM PBF Blob size is too large. PBF is probably corrupted. ("
167 + Utils.getSizeString(MAX_BLOB_SIZE, Locale.ENGLISH) + " < " + Utils.getSizeString(datasize, Locale.ENGLISH));
168 }
169 return new BlobHeader(type, indexData, datasize);
170 }
171
172 /**
173 * Parse a blob from the PBF file
174 *
175 * @param header The header with the blob information (most critically, the length of the blob)
176 * @param cis Used to ensure we don't read too much data
177 * @param parser The parser to read records from
178 * @param baos The reusable output stream
179 * @return The blob to use elsewhere
180 * @throws IOException If one of the streams has an issue
181 */
182 @Nonnull
183 private static Blob parseBlob(BlobHeader header, CountingInputStream cis, ProtobufParser parser, ByteArrayOutputStream baos)
184 throws IOException {
185 long start = cis.getBytesRead();
186 int size = Integer.MIN_VALUE;
187 Blob.CompressionType type = null;
188 ProtobufRecord current;
189 // Needed since size and compression type + compression data may be in a different order
190 byte[] bytes = null;
191 while (parser.hasNext() && cis.getBytesRead() - start < header.dataSize()) {
192 current = new ProtobufRecord(baos, parser);
193 switch (current.getField()) {
194 case 1:
195 type = Blob.CompressionType.raw;
196 bytes = current.getBytes();
197 break;
198 case 2:
199 size = current.asUnsignedVarInt().intValue();
200 break;
201 case 3:
202 type = Blob.CompressionType.zlib;
203 bytes = current.getBytes();
204 break;
205 case 4:
206 type = Blob.CompressionType.lzma;
207 bytes = current.getBytes();
208 break;
209 case 5:
210 type = Blob.CompressionType.bzip2;
211 bytes = current.getBytes();
212 break;
213 case 6:
214 type = Blob.CompressionType.lz4;
215 bytes = current.getBytes();
216 break;
217 case 7:
218 type = Blob.CompressionType.zstd;
219 bytes = current.getBytes();
220 break;
221 default:
222 throw new IllegalStateException("Unknown compression type: " + current.getField());
223 }
224 }
225 if (type == null) {
226 throw new IllegalStateException("Compression type not found, pbf may be malformed");
227 }
228 return new Blob(size, type, bytes);
229 }
230
231 /**
232 * Parse a header block. This assumes that the parser has hit a string with the text "OSMHeader".
233 *
234 * @param blob The blob with the header block data
235 * @param baos The reusable output stream to use
236 * @return The parsed HeaderBlock
237 * @throws IOException if one of the {@link InputStream}s has a problem
238 */
239 @Nonnull
240 private static HeaderBlock parseHeaderBlock(Blob blob, ByteArrayOutputStream baos) throws IOException {
241 try (InputStream blobInput = blob.inputStream();
242 ProtobufParser parser = new ProtobufParser(blobInput)) {
243 BBox bbox = null;
244 List<String> required = new ArrayList<>();
245 List<String> optional = new ArrayList<>();
246 String program = null;
247 String source = null;
248 Long osmosisReplicationTimestamp = null;
249 Long osmosisReplicationSequenceNumber = null;
250 String osmosisReplicationBaseUrl = null;
251 while (parser.hasNext()) {
252 final ProtobufRecord current = new ProtobufRecord(baos, parser);
253 switch (current.getField()) {
254 case 1: // bbox
255 bbox = parseBBox(baos, current);
256 break;
257 case 4: // repeated required features
258 required.add(current.asString());
259 break;
260 case 5: // repeated optional features
261 optional.add(current.asString());
262 break;
263 case 16: // writing program
264 program = current.asString();
265 break;
266 case 17: // source
267 source = current.asString();
268 break;
269 case 32: // osmosis replication timestamp
270 osmosisReplicationTimestamp = current.asSignedVarInt().longValue();
271 break;
272 case 33: // osmosis replication sequence number
273 osmosisReplicationSequenceNumber = current.asSignedVarInt().longValue();
274 break;
275 case 34: // osmosis replication base url
276 osmosisReplicationBaseUrl = current.asString();
277 break;
278 default: // fall through -- unknown header block field
279 }
280 }
281 return new HeaderBlock(bbox, required.toArray(new String[0]), optional.toArray(new String[0]), program,
282 source, osmosisReplicationTimestamp, osmosisReplicationSequenceNumber, osmosisReplicationBaseUrl);
283 }
284 }
285
286 /**
287 * Ensure that we support all the required features in the PBF
288 *
289 * @param headerBlock The HeaderBlock to check
290 * @throws IllegalDataException If there exists at least one feature that we do not support
291 */
292 private static void checkRequiredFeatures(HeaderBlock headerBlock) throws IllegalDataException {
293 Set<String> supportedFeatures = new HashSet<>(Arrays.asList("OsmSchema-V0.6", "DenseNodes", "HistoricalInformation"));
294 for (String requiredFeature : headerBlock.requiredFeatures()) {
295 if (!supportedFeatures.contains(requiredFeature)) {
296 throw new IllegalDataException("PBF Parser: Unknown required feature " + requiredFeature);
297 }
298 }
299 }
300
301 /**
302 * Parse a data blob (should be "OSMData")
303 *
304 * @param baos The reusable stream
305 * @param headerBlock The header block with data source information
306 * @param blob The blob to read OSM data from
307 * @throws IOException if we don't support the compression type
308 * @throws IllegalDataException If an invalid OSM primitive was read
309 */
310 private void parseDataBlock(ByteArrayOutputStream baos, HeaderBlock headerBlock, Blob blob) throws IOException, IllegalDataException {
311 String[] stringTable = null; // field 1, note that stringTable[0] is a delimiter, so it is always blank and unused
312 // field 2 -- we cannot parse these live just in case the following fields come later
313 List<ProtobufRecord> primitiveGroups = new ArrayList<>();
314 int granularity = 100; // field 17
315 long latOffset = 0; // field 19
316 long lonOffset = 0; // field 20
317 int dateGranularity = 1000; // field 18, default is milliseconds since the 1970 epoch
318 try (InputStream inputStream = blob.inputStream();
319 ProtobufParser parser = new ProtobufParser(inputStream)) {
320 while (parser.hasNext()) {
321 ProtobufRecord protobufRecord = new ProtobufRecord(baos, parser);
322 switch (protobufRecord.getField()) {
323 case 1:
324 stringTable = parseStringTable(baos, protobufRecord.getBytes());
325 break;
326 case 2:
327 primitiveGroups.add(protobufRecord);
328 break;
329 case 17:
330 granularity = protobufRecord.asUnsignedVarInt().intValue();
331 break;
332 case 18:
333 dateGranularity = protobufRecord.asUnsignedVarInt().intValue();
334 break;
335 case 19:
336 latOffset = protobufRecord.asUnsignedVarInt().longValue();
337 break;
338 case 20:
339 lonOffset = protobufRecord.asUnsignedVarInt().longValue();
340 break;
341 default: // Pass, since someone might have extended the format
342 }
343 }
344 }
345 final PrimitiveBlockRecord primitiveBlockRecord = new PrimitiveBlockRecord(stringTable, granularity, latOffset, lonOffset,
346 dateGranularity);
347 final DataSet ds = getDataSet();
348 if (!primitiveGroups.isEmpty() && headerBlock.bbox() != null) {
349 try {
350 ds.beginUpdate();
351 ds.addDataSource(new DataSource(new Bounds((LatLon) headerBlock.bbox().getMin(), (LatLon) headerBlock.bbox().getMax()),
352 headerBlock.source()));
353 } finally {
354 ds.endUpdate();
355 }
356 }
357 for (ProtobufRecord primitiveGroup : primitiveGroups) {
358 try {
359 ds.beginUpdate();
360 parsePrimitiveGroup(baos, primitiveGroup.getBytes(), primitiveBlockRecord);
361 } finally {
362 ds.endUpdate();
363 }
364 }
365 }
366
367 /**
368 * This parses a bbox from a record (HeaderBBox message)
369 *
370 * @param baos The reusable {@link ByteArrayOutputStream} to avoid unnecessary allocations
371 * @param current The current record
372 * @return The <i>immutable</i> bbox, or {@code null}
373 * @throws IOException If something happens with the {@link InputStream}s (probably won't happen)
374 */
375 @Nullable
376 private static BBox parseBBox(ByteArrayOutputStream baos, ProtobufRecord current) throws IOException {
377 try (ByteArrayInputStream bboxInputStream = new ByteArrayInputStream(current.getBytes());
378 ProtobufParser bboxParser = new ProtobufParser(bboxInputStream)) {
379 double left = Double.NaN;
380 double right = Double.NaN;
381 double top = Double.NaN;
382 double bottom = Double.NaN;
383 while (bboxParser.hasNext()) {
384 ProtobufRecord protobufRecord = new ProtobufRecord(baos, bboxParser);
385 if (protobufRecord.getType() == WireType.VARINT) {
386 double value = protobufRecord.asSignedVarInt().longValue() * NANO_DEGREES;
387 switch (protobufRecord.getField()) {
388 case 1:
389 left = value;
390 break;
391 case 2:
392 right = value;
393 break;
394 case 3:
395 top = value;
396 break;
397 case 4:
398 bottom = value;
399 break;
400 default: // Fall through -- someone might have extended the format
401 }
402 }
403 }
404 if (!Double.isNaN(left) && !Double.isNaN(top) && !Double.isNaN(right) && !Double.isNaN(bottom)) {
405 return new BBox(left, top, right, bottom).toImmutable();
406 }
407 }
408 return null;
409 }
410
411 /**
412 * Parse the string table
413 *
414 * @param baos The reusable stream
415 * @param bytes The message bytes
416 * @return The parsed table (reminder: index 0 is empty, note that all strings are already interned by {@link String#intern()})
417 * @throws IOException if something happened while reading a {@link ByteArrayInputStream}
418 */
419 @Nonnull
420 private static String[] parseStringTable(ByteArrayOutputStream baos, byte[] bytes) throws IOException {
421 try (ByteArrayInputStream is = new ByteArrayInputStream(bytes);
422 ProtobufParser parser = new ProtobufParser(is)) {
423 List<String> list = new ArrayList<>();
424 while (parser.hasNext()) {
425 ProtobufRecord protobufRecord = new ProtobufRecord(baos, parser);
426 if (protobufRecord.getField() == 1) {
427 list.add(protobufRecord.asString().intern()); // field is technically repeated bytes
428 }
429 }
430 return list.toArray(new String[0]);
431 }
432 }
433
434 /**
435 * Parse a PrimitiveGroup. Note: this parsing implementation doesn't check and make certain that all primitives in the group are the same
436 * type.
437 *
438 * @param baos The reusable stream
439 * @param bytes The bytes to decode
440 * @param primitiveBlockRecord The record to use for creating the primitives
441 * @throws IllegalDataException if one of the primitive records was invalid
442 * @throws IOException if something happened while reading a {@link ByteArrayInputStream}
443 */
444 private void parsePrimitiveGroup(ByteArrayOutputStream baos, byte[] bytes, PrimitiveBlockRecord primitiveBlockRecord)
445 throws IllegalDataException, IOException {
446 try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
447 ProtobufParser parser = new ProtobufParser(bais)) {
448 while (parser.hasNext()) {
449 ProtobufRecord protobufRecord = new ProtobufRecord(baos, parser);
450 switch (protobufRecord.getField()) {
451 case 1: // Nodes, repeated
452 parseNode(baos, protobufRecord.getBytes(), primitiveBlockRecord);
453 break;
454 case 2: // Dense nodes, not repeated
455 parseDenseNodes(baos, protobufRecord.getBytes(), primitiveBlockRecord);
456 break;
457 case 3: // Ways, repeated
458 parseWay(baos, protobufRecord.getBytes(), primitiveBlockRecord);
459 break;
460 case 4: // relations, repeated
461 parseRelation(baos, protobufRecord.getBytes(), primitiveBlockRecord);
462 break;
463 case 5: // Changesets, repeated
464 // Skip -- we don't have a good way to store changeset information in JOSM
465 default: // OSM PBF could be extended
466 }
467 }
468 }
469 }
470
471 /**
472 * Parse a singular node
473 *
474 * @param baos The reusable stream
475 * @param bytes The bytes to decode
476 * @param primitiveBlockRecord The record to use (mostly for tags and lat/lon calculations)
477 * @throws IllegalDataException if the PBF did not provide all the data necessary for node creation
478 * @throws IOException if something happened while reading a {@link ByteArrayInputStream}
479 */
480 private void parseNode(ByteArrayOutputStream baos, byte[] bytes, PrimitiveBlockRecord primitiveBlockRecord)
481 throws IllegalDataException, IOException {
482 try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
483 ProtobufParser parser = new ProtobufParser(bais)) {
484 long id = Long.MIN_VALUE;
485 List<String> keys = new ArrayList<>();
486 List<String> values = new ArrayList<>();
487 Info info = null;
488 long lat = Long.MIN_VALUE;
489 long lon = Long.MIN_VALUE;
490 while (parser.hasNext()) {
491 ProtobufRecord protobufRecord = new ProtobufRecord(baos, parser);
492 switch (protobufRecord.getField()) {
493 case 1:
494 id = protobufRecord.asSignedVarInt().intValue();
495 break;
496 case 2:
497 for (long number : new ProtobufPacked(protobufRecord.getBytes()).getArray()) {
498 keys.add(primitiveBlockRecord.stringTable[(int) number]);
499 }
500 break;
501 case 3:
502 for (long number : new ProtobufPacked(protobufRecord.getBytes()).getArray()) {
503 values.add(primitiveBlockRecord.stringTable[(int) number]);
504 }
505 break;
506 case 4:
507 info = parseInfo(baos, protobufRecord.getBytes());
508 break;
509 case 8:
510 lat = protobufRecord.asSignedVarInt().longValue();
511 break;
512 case 9:
513 lon = protobufRecord.asSignedVarInt().longValue();
514 break;
515 default: // Fall through -- PBF could be extended (unlikely)
516 }
517 }
518 if (id == Long.MIN_VALUE || lat == Long.MIN_VALUE || lon == Long.MIN_VALUE) {
519 throw new IllegalDataException("OSM PBF did not provide all the required node information");
520 }
521 NodeData node = new NodeData(id);
522 node.setCoor(calculateLatLon(primitiveBlockRecord, lat, lon));
523 addTags(node, keys, values);
524 if (info != null) {
525 setOsmPrimitiveData(primitiveBlockRecord, node, info);
526 } else {
527 ds.setUploadPolicy(UploadPolicy.DISCOURAGED);
528 }
529 buildPrimitive(node);
530 }
531 }
532
533 /**
534 * Parse dense nodes from a record
535 *
536 * @param baos The reusable output stream
537 * @param bytes The bytes for the dense node
538 * @param primitiveBlockRecord Used for data that is common between several different objects.
539 * @throws IllegalDataException if the nodes could not be parsed, or one of the nodes would be malformed
540 * @throws IOException if something happened while reading a {@link ByteArrayInputStream}
541 */
542 private void parseDenseNodes(ByteArrayOutputStream baos, byte[] bytes, PrimitiveBlockRecord primitiveBlockRecord)
543 throws IllegalDataException, IOException {
544 long[] ids = EMPTY_LONG;
545 long[] lats = EMPTY_LONG;
546 long[] lons = EMPTY_LONG;
547 long[] keyVals = EMPTY_LONG; // technically can be int
548 Info[] denseInfo = null;
549 try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
550 ProtobufParser parser = new ProtobufParser(bais)) {
551 while (parser.hasNext()) {
552 ProtobufRecord protobufRecord = new ProtobufRecord(baos, parser);
553 switch (protobufRecord.getField()) {
554 case 1: // packed node ids, DELTA encoded
555 long[] tids = decodePackedSInt64(new ProtobufPacked(protobufRecord.getBytes()).getArray());
556 ids = joinArrays(ids, tids);
557 break;
558 case 5: // DenseInfo
559 denseInfo = parseDenseInfo(baos, protobufRecord.getBytes()); // not repeated or packed
560 break;
561 case 8: // packed lat, DELTA encoded
562 long[] tlats = decodePackedSInt64(new ProtobufPacked(protobufRecord.getBytes()).getArray());
563 lats = joinArrays(lats, tlats);
564 break;
565 case 9: // packed lon, DELTA encoded
566 long[] tlons = decodePackedSInt64(new ProtobufPacked(protobufRecord.getBytes()).getArray());
567 lons = joinArrays(lons, tlons);
568 break;
569 case 10: // key_val mappings, packed. '0' used as separator between nodes
570 long[] tkeyVal = new ProtobufPacked(protobufRecord.getBytes()).getArray();
571 keyVals = joinArrays(keyVals, tkeyVal);
572 break;
573 default: // Someone might have extended the PBF format
574 }
575 }
576 }
577
578 int keyValIndex = 0; // This index must not reset between nodes, and must always increment
579 if (ids.length == lats.length && lats.length == lons.length && (denseInfo == null || denseInfo.length == lons.length)) {
580 long id = 0;
581 long lat = 0;
582 long lon = 0;
583 for (int i = 0; i < ids.length; i++) {
584 final NodeData node;
585 id += ids[i];
586 node = new NodeData(id);
587 if (denseInfo != null) {
588 Info info = denseInfo[i];
589 setOsmPrimitiveData(primitiveBlockRecord, node, info);
590 } else {
591 ds.setUploadPolicy(UploadPolicy.DISCOURAGED);
592 }
593 lat += lats[i];
594 lon += lons[i];
595 // Not very efficient when Node doesn't store the LatLon. Hopefully not too much of an issue
596 node.setCoor(calculateLatLon(primitiveBlockRecord, lat, lon));
597 String key = null;
598 while (keyValIndex < keyVals.length) {
599 int stringIndex = (int) keyVals[keyValIndex];
600 // StringTable[0] is always an empty string, and acts as a separator between the tags of different nodes here
601 if (stringIndex != 0) {
602 if (key == null) {
603 key = primitiveBlockRecord.stringTable[stringIndex];
604 } else {
605 node.put(key, primitiveBlockRecord.stringTable[stringIndex]);
606 key = null;
607 }
608 keyValIndex++;
609 } else {
610 keyValIndex++;
611 break;
612 }
613 }
614 // Just add the nodes as we make them -- avoid creating another list that expands every time we parse a node
615 buildPrimitive(node);
616 }
617 } else {
618 throw new IllegalDataException("OSM PBF has mismatched DenseNode lengths");
619 }
620 }
621
622 /**
623 * Parse a way from the PBF
624 *
625 * @param baos The reusable stream
626 * @param bytes The bytes for the way
627 * @param primitiveBlockRecord Used for common information, like tags
628 * @throws IllegalDataException if an invalid way could have been created
629 * @throws IOException if something happened while reading a {@link ByteArrayInputStream}
630 */
631 private void parseWay(ByteArrayOutputStream baos, byte[] bytes, PrimitiveBlockRecord primitiveBlockRecord)
632 throws IllegalDataException, IOException {
633 long id = Long.MIN_VALUE;
634 List<String> keys = new ArrayList<>();
635 List<String> values = new ArrayList<>();
636 Info info = null;
637 long[] refs = EMPTY_LONG; // DELTA encoded
638 // We don't do live drawing, so we don't care about lats and lons (we essentially throw them away with the current parser)
639 // This is for the optional feature "LocationsOnWays"
640 try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
641 ProtobufParser parser = new ProtobufParser(bais)) {
642 while (parser.hasNext()) {
643 ProtobufRecord protobufRecord = new ProtobufRecord(baos, parser);
644 switch (protobufRecord.getField()) {
645 case 1:
646 id = protobufRecord.asUnsignedVarInt().longValue();
647 break;
648 case 2:
649 for (long number : new ProtobufPacked(protobufRecord.getBytes()).getArray()) {
650 keys.add(primitiveBlockRecord.stringTable[(int) number]);
651 }
652 break;
653 case 3:
654 for (long number : new ProtobufPacked(protobufRecord.getBytes()).getArray()) {
655 values.add(primitiveBlockRecord.stringTable[(int) number]);
656 }
657 break;
658 case 4:
659 info = parseInfo(baos, protobufRecord.getBytes());
660 break;
661 case 8:
662 long[] tRefs = decodePackedSInt64(new ProtobufPacked(protobufRecord.getBytes()).getArray());
663 refs = joinArrays(refs, tRefs);
664 break;
665 // case 9 and 10 are for "LocationsOnWays" -- this is only usable if we can create the way geometry directly
666 // if this is ever supported, lats = joinArrays(lats, decodePackedSInt64(...))
667 default: // PBF could be expanded by other people
668 }
669 }
670 }
671 if (refs.length == 0 || id == Long.MIN_VALUE) {
672 throw new IllegalDataException("A way with either no id or no nodes was found");
673 }
674 WayData wayData = new WayData(id);
675 List<Long> nodeIds = new ArrayList<>(refs.length);
676 long ref = 0;
677 for (long tRef : refs) {
678 ref += tRef;
679 nodeIds.add(ref);
680 }
681 this.ways.put(wayData.getUniqueId(), nodeIds);
682 addTags(wayData, keys, values);
683 if (info != null) {
684 setOsmPrimitiveData(primitiveBlockRecord, wayData, info);
685 } else {
686 ds.setUploadPolicy(UploadPolicy.DISCOURAGED);
687 }
688 buildPrimitive(wayData);
689 }
690
691 /**
692 * Parse a relation from a PBF
693 *
694 * @param baos The reusable stream
695 * @param bytes The bytes to use
696 * @param primitiveBlockRecord Mostly used for tags
697 * @throws IllegalDataException if the PBF had a bad relation definition
698 * @throws IOException if something happened while reading a {@link ByteArrayInputStream}
699 */
700 private void parseRelation(ByteArrayOutputStream baos, byte[] bytes, PrimitiveBlockRecord primitiveBlockRecord)
701 throws IllegalDataException, IOException {
702 long id = Long.MIN_VALUE;
703 List<String> keys = new ArrayList<>();
704 List<String> values = new ArrayList<>();
705 Info info = null;
706 long[] rolesStringId = EMPTY_LONG; // Technically int
707 long[] memids = EMPTY_LONG;
708 long[] types = EMPTY_LONG; // Technically an enum
709 try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
710 ProtobufParser parser = new ProtobufParser(bais)) {
711 while (parser.hasNext()) {
712 ProtobufRecord protobufRecord = new ProtobufRecord(baos, parser);
713 switch (protobufRecord.getField()) {
714 case 1:
715 id = protobufRecord.asUnsignedVarInt().longValue();
716 break;
717 case 2:
718 for (long number : new ProtobufPacked(protobufRecord.getBytes()).getArray()) {
719 keys.add(primitiveBlockRecord.stringTable[(int) number]);
720 }
721 break;
722 case 3:
723 for (long number : new ProtobufPacked(protobufRecord.getBytes()).getArray()) {
724 values.add(primitiveBlockRecord.stringTable[(int) number]);
725 }
726 break;
727 case 4:
728 info = parseInfo(baos, protobufRecord.getBytes());
729 break;
730 case 8:
731 long[] tRoles = new ProtobufPacked(protobufRecord.getBytes()).getArray();
732 rolesStringId = joinArrays(rolesStringId, tRoles);
733 break;
734 case 9:
735 long[] tMemids = decodePackedSInt64(new ProtobufPacked(protobufRecord.getBytes()).getArray());
736 memids = joinArrays(memids, tMemids);
737 break;
738 case 10:
739 long[] tTypes = new ProtobufPacked(protobufRecord.getBytes()).getArray();
740 types = joinArrays(types, tTypes);
741 break;
742 default: // Fall through for PBF extensions
743 }
744 }
745 }
746 if (keys.size() != values.size() || rolesStringId.length != memids.length || memids.length != types.length || id == Long.MIN_VALUE) {
747 throw new IllegalDataException("OSM PBF contains a bad relation definition");
748 }
749 RelationData data = new RelationData(id);
750 if (info != null) {
751 setOsmPrimitiveData(primitiveBlockRecord, data, info);
752 } else {
753 ds.setUploadPolicy(UploadPolicy.DISCOURAGED);
754 }
755 addTags(data, keys, values);
756 OsmPrimitiveType[] valueTypes = OsmPrimitiveType.values();
757 List<RelationMemberData> members = new ArrayList<>(rolesStringId.length);
758 long memberId = 0;
759 for (int i = 0; i < rolesStringId.length; i++) {
760 String role = primitiveBlockRecord.stringTable[(int) rolesStringId[i]];
761 memberId += memids[i];
762 OsmPrimitiveType type = valueTypes[(int) types[i]];
763 members.add(new RelationMemberData(role, type, memberId));
764 }
765 this.relations.put(data.getUniqueId(), members);
766 buildPrimitive(data);
767 }
768
769 /**
770 * Parse info for an object
771 *
772 * @param baos The reusable stream to use
773 * @param bytes The bytes to decode
774 * @return The info for an object
775 * @throws IOException if something happened while reading a {@link ByteArrayInputStream}
776 */
777 @Nonnull
778 private static Info parseInfo(ByteArrayOutputStream baos, byte[] bytes) throws IOException {
779 try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
780 ProtobufParser parser = new ProtobufParser(bais)) {
781 int version = -1;
782 Long timestamp = null;
783 Long changeset = null;
784 Integer uid = null;
785 Integer userSid = null;
786 boolean visible = true;
787 while (parser.hasNext()) {
788 ProtobufRecord protobufRecord = new ProtobufRecord(baos, parser);
789 switch (protobufRecord.getField()) {
790 case 1:
791 version = protobufRecord.asUnsignedVarInt().intValue();
792 break;
793 case 2:
794 timestamp = protobufRecord.asUnsignedVarInt().longValue();
795 break;
796 case 3:
797 changeset = protobufRecord.asUnsignedVarInt().longValue();
798 break;
799 case 4:
800 uid = protobufRecord.asUnsignedVarInt().intValue();
801 break;
802 case 5:
803 userSid = protobufRecord.asUnsignedVarInt().intValue();
804 break;
805 case 6:
806 visible = protobufRecord.asUnsignedVarInt().byteValue() == 1;
807 break;
808 default: // Fall through, since the PBF format could be extended
809 }
810 }
811 return new Info(version, timestamp, changeset, uid, userSid, visible);
812 }
813 }
814
815 /**
816 * Calculate the actual lat lon
817 *
818 * @param primitiveBlockRecord The record with offset and granularity data
819 * @param lat The latitude from the PBF
820 * @param lon The longitude from the PBF
821 * @return The actual {@link LatLon}, accounting for PBF offset and granularity changes
822 */
823 @Nonnull
824 private static LatLon calculateLatLon(PrimitiveBlockRecord primitiveBlockRecord, long lat, long lon) {
825 return new LatLon(NANO_DEGREES * (primitiveBlockRecord.latOffset + (primitiveBlockRecord.granularity * lat)),
826 NANO_DEGREES * (primitiveBlockRecord.lonOffset + (primitiveBlockRecord.granularity * lon)));
827 }
828
829 /**
830 * Add a set of tags to a primitive
831 *
832 * @param primitive The primitive to add tags to
833 * @param keys The keys (must match the size of the values)
834 * @param values The values (must match the size of the keys)
835 */
836 private static void addTags(Tagged primitive, List<String> keys, List<String> values) {
837 if (keys.isEmpty()) {
838 return;
839 }
840 Map<String, String> tagMap = new HashMap<>(keys.size());
841 for (int i = 0; i < keys.size(); i++) {
842 tagMap.put(keys.get(i), values.get(i));
843 }
844 primitive.putAll(tagMap);
845 }
846
847 /**
848 * Set the primitive data for an object
849 *
850 * @param primitiveBlockRecord The record with data for the current primitive (currently uses {@link PrimitiveBlockRecord#stringTable} and
851 * {@link PrimitiveBlockRecord#dateGranularity}).
852 * @param primitive The primitive to add the information to
853 * @param info The specific info for the primitive
854 */
855 private static void setOsmPrimitiveData(PrimitiveBlockRecord primitiveBlockRecord, PrimitiveData primitive, Info info) {
856 primitive.setVisible(info.isVisible());
857 if (info.timestamp() != null) {
858 primitive.setRawTimestamp(Math.toIntExact(info.timestamp() * primitiveBlockRecord.dateGranularity / 1000));
859 }
860 if (info.uid() != null && info.userSid() != null) {
861 primitive.setUser(User.createOsmUser(info.uid(), primitiveBlockRecord.stringTable[info.userSid()]));
862 } else if (info.uid() != null) {
863 primitive.setUser(User.getById(info.uid()));
864 }
865 if (info.version() > 0) {
866 primitive.setVersion(info.version());
867 }
868 if (info.changeset() != null) {
869 primitive.setChangesetId(Math.toIntExact(info.changeset()));
870 }
871 }
872
873 /**
874 * Convert an array of numbers to an array of longs, decoded from uint (zig zag decoded)
875 *
876 * @param numbers The numbers to convert
877 * @return The long array (the same array that was passed in)
878 */
879 @Nonnull
880 private static long[] decodePackedSInt64(long[] numbers) {
881 for (int i = 0; i < numbers.length; i++) {
882 numbers[i] = ProtobufParser.decodeZigZag(numbers[i]);
883 }
884 return numbers;
885 }
886
887 /**
888 * Join two different arrays
889 *
890 * @param array1 The first array
891 * @param array2 The second array
892 * @return The joined arrays -- may return one of the original arrays, if the other is empty
893 */
894 @Nonnull
895 private static long[] joinArrays(long[] array1, long[] array2) {
896 if (array1.length == 0) {
897 return array2;
898 }
899 if (array2.length == 0) {
900 return array1;
901 }
902 long[] result = Arrays.copyOf(array1, array1.length + array2.length);
903 System.arraycopy(array2, 0, result, array1.length, array2.length);
904 return result;
905 }
906
907 /**
908 * Parse dense info
909 *
910 * @param baos The reusable stream
911 * @param bytes The bytes to decode
912 * @return The dense info array
913 * @throws IllegalDataException If the data has mismatched array lengths
914 * @throws IOException if something happened while reading a {@link ByteArrayInputStream}
915 */
916 @Nonnull
917 private static Info[] parseDenseInfo(ByteArrayOutputStream baos, byte[] bytes) throws IllegalDataException, IOException {
918 long[] version = EMPTY_LONG; // technically ints
919 long[] timestamp = EMPTY_LONG;
920 long[] changeset = EMPTY_LONG;
921 long[] uid = EMPTY_LONG; // technically int
922 long[] userSid = EMPTY_LONG; // technically int
923 long[] visible = EMPTY_LONG; // optional, true if not set, technically booleans
924 try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
925 ProtobufParser parser = new ProtobufParser(bais)) {
926 while (parser.hasNext()) {
927 ProtobufRecord protobufRecord = new ProtobufRecord(baos, parser);
928 switch (protobufRecord.getField()) {
929 case 1:
930 long[] tVersion = new ProtobufPacked(protobufRecord.getBytes()).getArray();
931 version = joinArrays(version, tVersion);
932 break;
933 case 2:
934 long[] tTimestamp = decodePackedSInt64(new ProtobufPacked(protobufRecord.getBytes()).getArray());
935 timestamp = joinArrays(timestamp, tTimestamp);
936 break;
937 case 3:
938 long[] tChangeset = decodePackedSInt64(new ProtobufPacked(protobufRecord.getBytes()).getArray());
939 changeset = joinArrays(changeset, tChangeset);
940 break;
941 case 4:
942 long[] tUid = decodePackedSInt64(new ProtobufPacked(protobufRecord.getBytes()).getArray());
943 uid = joinArrays(uid, tUid);
944 break;
945 case 5:
946 long[] tUserSid = decodePackedSInt64(new ProtobufPacked(protobufRecord.getBytes()).getArray());
947 userSid = joinArrays(userSid, tUserSid);
948 break;
949 case 6:
950 long[] tVisible = new ProtobufPacked(protobufRecord.getBytes()).getArray();
951 visible = joinArrays(visible, tVisible);
952 break;
953 default: // Fall through
954 }
955 }
956 }
957 if (version.length > 0) {
958 Info[] infos = new Info[version.length];
959 long lastTimestamp = 0; // delta encoded
960 long lastChangeset = 0; // delta encoded
961 long lastUid = 0; // delta encoded,
962 long lastUserSid = 0; // delta encoded, string id for username
963 for (int i = 0; i < version.length; i++) {
964 if (timestamp.length > i)
965 lastTimestamp += timestamp[i];
966 if (changeset.length > i)
967 lastChangeset += changeset[i];
968 if (uid.length > i && userSid.length > i) {
969 lastUid += uid[i];
970 lastUserSid += userSid[i];
971 }
972 infos[i] = new Info((int) version[i], lastTimestamp, lastChangeset, (int) lastUid, (int) lastUserSid,
973 visible == EMPTY_LONG || visible[i] == 1);
974 }
975 return infos;
976 }
977 throw new IllegalDataException("OSM PBF has mismatched DenseInfo lengths");
978 }
979
980 /**
981 * A record class for passing PrimitiveBlock information to the PrimitiveGroup parser
982 */
983 private static final class PrimitiveBlockRecord {
984 private final String[] stringTable;
985 private final int granularity;
986 private final long latOffset;
987 private final long lonOffset;
988 private final int dateGranularity;
989
990 /**
991 * Create a new record
992 *
993 * @param stringTable The string table (reminder: 0 index is empty, as it is used by DenseNode to separate node tags)
994 * @param granularity units of nanodegrees, used to store coordinates
995 * @param latOffset offset value between the output coordinates and the granularity grid in units of nanodegrees
996 * @param lonOffset offset value between the output coordinates and the granularity grid in units of nanodegrees
997 * @param dateGranularity Granularity of dates, normally represented in units of milliseconds since the 1970 epoch
998 */
999 PrimitiveBlockRecord(String[] stringTable, int granularity, long latOffset, long lonOffset,
1000 int dateGranularity) {
1001 this.stringTable = stringTable;
1002 this.granularity = granularity;
1003 this.latOffset = latOffset;
1004 this.lonOffset = lonOffset;
1005 this.dateGranularity = dateGranularity;
1006 }
1007
1008 }
1009}
Note: See TracBrowser for help on using the repository browser.