Ignore:
Timestamp:
2023-10-05T15:11:37+02:00 (15 months ago)
Author:
taylor.smock
Message:

See #8647: FIT file support

This fixes an issue where there are apparent lines to null island. There were two
reasons for this:

  • A point around -1, -1 at a far-distant time point. There are two unknown fields with -1; the "good" data had values >= 0 for both fields (field 13 and field 107).
  • Invalid values which were mapped to 180, 180.
Location:
applications/editors/josm/plugins/FIT/src/main/java/org/openstreetmap/josm/plugins/fit
Files:
2 added
7 edited

Legend:

Unmodified
Added
Removed

  • applications/editors/josm/plugins/FIT/src/main/java/org/openstreetmap/josm/plugins/fit/FitPlugin.java

    r36151 r36158  
    66import java.io.File;
    77import java.io.IOException;
    8 import java.io.InputStream;
     8import java.lang.reflect.RecordComponent;
    99import java.nio.file.Files;
    10 import java.time.Instant;
    1110import java.util.ArrayList;
     11import java.util.Arrays;
    1212import java.util.Collections;
    13 import java.util.Map;
     13import java.util.TreeMap;
     14import java.util.concurrent.TimeUnit;
    1415
    1516import org.openstreetmap.josm.actions.ExtensionFileFilter;
    … …  
    2627import org.openstreetmap.josm.plugins.fit.lib.FitReader;
    2728import org.openstreetmap.josm.plugins.fit.lib.FitReaderOptions;
     29import org.openstreetmap.josm.plugins.fit.lib.global.FitEvent;
    2830import org.openstreetmap.josm.plugins.fit.lib.global.HeartRateCadenceDistanceSpeed;
    2931
    … …  
    5355        @Override
    5456        public void importData(File file, ProgressMonitor progressMonitor) throws IOException {
    55             try (InputStream inputStream = Files.newInputStream(file.toPath())) {
     57            try (var inputStream = Files.newInputStream(file.toPath())) {
    5658                final var records = FitReader.read(inputStream, FitReaderOptions.TRY_TO_FINISH);
    5759                final var gpxData = new GpxData(true);
    5860                progressMonitor.beginTask(tr("Processing FIT records"), records.length);
    5961                final var waypoints = new ArrayList<WayPoint>(records.length % 1000);
    60                 for (int i = 0; i < records.length; i++) {
     62                for (var i = 0; i < records.length; i++) {
    6163                    var r = records[i];
    6264                    if (i % 1000 == 0) {
    6365                        progressMonitor.worked(1);
    6466                    }
    65                     if (r instanceof HeartRateCadenceDistanceSpeed(
    66                             Instant timestamp, double lat, double lon, double ele, short heartRate, short cadence,
    67                             int distance, int speed, long[][] unknown,
    68                             org.openstreetmap.josm.plugins.fit.lib.global.FitDevDataRecord devData
    69                     )) {
    70                         final var waypoint = new WayPoint(new LatLon(lat, lon));
    71                         waypoint.setInstant(timestamp);
    72                         waypoint.attr.putAll(Map.of("ele", ele, "heart_rate", heartRate,
    73                                 "cadence", cadence, "distance", distance, "speed", speed,
    74                                 "unknown", unknown, "devData", devData));
    75                         waypoints.add(waypoint);
     67                    if (r instanceof HeartRateCadenceDistanceSpeed heartRateCadenceDistanceSpeed) {
     68                        final var lat = heartRateCadenceDistanceSpeed.lat();
     69                        final var lon = heartRateCadenceDistanceSpeed.lon();
     70                        if (!Double.isNaN(lat) && !Double.isNaN(lon)) {
     71                            final var waypoint = new WayPoint(new LatLon(lat, lon));
     72                            waypoint.setInstant(heartRateCadenceDistanceSpeed.timestamp());
     73                            // Use a sorted map for consistency
     74                            final var map = new TreeMap<String, Object>();
     75                            for (RecordComponent component : HeartRateCadenceDistanceSpeed.class
     76                                    .getRecordComponents()) {
     77                                if (Arrays.asList("lat", "lon", "timestamp", "unknown").contains(component.getName())) {
     78                                    continue; // skip information that has specific fields
     79                                }
     80                                try {
     81                                    map.put(component.getName(),
     82                                            component.getAccessor().invoke(heartRateCadenceDistanceSpeed));
     83                                } catch (ReflectiveOperationException e) {
     84                                    // This should never happen; the component accessors should _always_ be public.
     85                                    throw new IOException(e);
     86                                }
     87                            }
     88                            map.put("unknown", Arrays.deepToString(heartRateCadenceDistanceSpeed.unknown()));
     89                            waypoint.attr.putAll(map);
     90                            if (!waypoints.isEmpty() && Math.abs(waypoints.getLast().getInstant().getEpochSecond()
     91                                    - waypoint.getInstant().getEpochSecond()) > TimeUnit.DAYS.toDays(365)) {
     92                                createTrack(gpxData, new ArrayList<>(waypoints));
     93                                waypoints.clear();
     94                            } else {
     95                                waypoints.add(waypoint);
     96                            }
     97                        }
     98                    } else if (r instanceof FitEvent) {
     99                        // break up the events. It would be better to only do this on lap events.
     100                        gpxData.addTrack(new GpxTrack(Collections.singleton(new ArrayList<>(waypoints)),
     101                                Collections.emptyMap()));
     102                        waypoints.clear();
    76103                    }
    77104                }
    78105                waypoints.trimToSize();
    79                 gpxData.addTrack(new GpxTrack(Collections.singleton(waypoints), Collections.emptyMap()));
     106                createTrack(gpxData, waypoints);
    80107                gpxData.endUpdate();
    81108                MainApplication.getLayerManager().addLayer(new GpxLayer(gpxData, file.getName(), true));
    82109            }
    83110        }
     111
     112        private static void createTrack(GpxData gpxData, ArrayList<WayPoint> waypoints) {
     113            if (waypoints.size() > 1) {
     114                gpxData.addTrack(new GpxTrack(Collections.singleton(waypoints), Collections.emptyMap()));
     115            } else if (!waypoints.isEmpty()) {
     116                gpxData.addWaypoint(waypoints.get(0));
     117            }
     118        }
    84119    }
    85120}

  • applications/editors/josm/plugins/FIT/src/main/java/org/openstreetmap/josm/plugins/fit/lib/global/FitData.java

    r36151 r36158  
    77public sealed
    88interface FitData
    9 permits FitDeveloperDataIdMessage, FitDevice, FitUnknownRecord, HeartRateCadenceDistanceSpeed
     9permits FileCreator, FitDeveloperDataIdMessage, FitDevice, FitEvent, FitUnknownRecord, HeartRateCadenceDistanceSpeed
    1010{
    1111

  • applications/editors/josm/plugins/FIT/src/main/java/org/openstreetmap/josm/plugins/fit/lib/global/FitUnknownRecord.java

    r36151 r36158  
    22package org.openstreetmap.josm.plugins.fit.lib.global;
    33
     4import org.openstreetmap.josm.plugins.fit.lib.records.IFitDevData;
    45import org.openstreetmap.josm.plugins.fit.lib.records.internal.IField;
    56
    6 public record FitUnknownRecord(FieldData[] data, FitDevDataRecord devData) implements FitData {
     7public record FitUnknownRecord(int globalMessageNumber, FieldData[] data, FitDevDataRecord devData) implements FitData {
    78
    8     public record FieldData(IField field, byte[] data) {}
     9    public record FieldData(IField field, IFitDevData<?> data) {}
    910}

  • applications/editors/josm/plugins/FIT/src/main/java/org/openstreetmap/josm/plugins/fit/lib/global/Global.java

    r36151 r36158  
    22package org.openstreetmap.josm.plugins.fit.lib.global;
    33
     4import java.io.ByteArrayInputStream;
    45import java.io.IOException;
    56import java.io.InputStream;
    67import java.util.List;
    78
     9import org.openstreetmap.josm.plugins.fit.lib.FitBaseType;
     10import org.openstreetmap.josm.plugins.fit.lib.records.FitDevStringData;
    811import org.openstreetmap.josm.plugins.fit.lib.records.internal.FitDefinitionMessage;
    912import org.openstreetmap.josm.plugins.fit.lib.records.internal.FitDeveloperField;
    … …  
    1114import org.openstreetmap.josm.plugins.fit.lib.records.internal.FitField;
    1215import org.openstreetmap.josm.plugins.fit.lib.utils.DevDataUtils;
     16import org.openstreetmap.josm.plugins.fit.lib.utils.StringUtils;
    1317
    1418/**
    … …  
    4145            case 20 ->
    4246                    HeartRateCadenceDistanceSpeed.parse(littleEndian, fieldList, developerFieldList, developerFields, inputStream);
     47            case 21 -> FitEvent.parse(littleEndian, fieldList, developerFieldList, developerFields, inputStream);
    4348            case MESSAGE_NUMBER_DEV_FIELD_DESCRIPTION -> FitDeveloperFieldDescriptionMessage
    4449                    .parse(littleEndian, fieldList, developerFieldList, developerFields, inputStream);
    4550            case MESSAGE_NUMBER_APP_ID ->
    4651                    FitDeveloperDataIdMessage.parse(littleEndian, fieldList, developerFieldList, developerFields, inputStream);
     52            case 49 -> FileCreator.parse(littleEndian, fieldList, developerFieldList, developerFields, inputStream);
    4753            default -> {
    4854                final var fieldData = new FitUnknownRecord.FieldData[fieldList.size()];
    4955                var index = 0;
    5056                for (FitField field : fieldList) {
    51                     fieldData[index++] = new FitUnknownRecord.FieldData(field, inputStream.readNBytes(field.size()));
     57                    if (field.fitBaseType() == FitBaseType.string) {
     58                        fieldData[index++] = new FitUnknownRecord.FieldData(field,
     59                                new FitDevStringData("", "",
     60                                        StringUtils.decodeString(new ByteArrayInputStream(inputStream.readNBytes(field.size())))));
     61                    } else {
     62                        fieldData[index++] = new FitUnknownRecord.FieldData(field, DevDataUtils.getData(field.fitBaseType(),
     63                                "", "", field.size(), littleEndian, inputStream));
     64                    }
    5265                }
    53                 yield new FitUnknownRecord(fieldData,
     66                yield new FitUnknownRecord(globalMessageNumber, fieldData,
    5467                        DevDataUtils.parseDevFields(littleEndian, developerFieldList, developerFields, inputStream));
    5568            }

  • applications/editors/josm/plugins/FIT/src/main/java/org/openstreetmap/josm/plugins/fit/lib/global/HeartRateCadenceDistanceSpeed.java

    r36151 r36158  
    2020                                            int distance, int speed, long[][] unknown,
    2121                                            FitDevDataRecord devData) implements FitData, IFitTimestamp<HeartRateCadenceDistanceSpeed> {
    22     private static final long[][] NO_UNKNOWNS = new long[0][];
     22    static final long[][] NO_UNKNOWNS = new long[0][];
    2323
    2424    // Using the 2023-09-09-12-0016.fit.gpx file provided by richlv:
    … …  
    6565                case 5 -> distance = NumberUtils.decodeInt(size, littleEndian, inputStream);
    6666                case 6 -> speed = NumberUtils.decodeInt(size, littleEndian, inputStream);
    67                 case 253 -> timestamp = Instant.ofEpochSecond(EPOCH_DIFFERENCE + NumberUtils.decodeLong(size, littleEndian, inputStream));
     67                // 13 seems to be either -1 (invalid entry?), or 23-27
     68                // 107 seems to always be -1 (invalid entry?), 0, and 1.
     69                case 253 -> timestamp = decodeInstant(size, littleEndian, inputStream);
    6870                default -> {
    6971                    unknowns = Arrays.copyOf(unknowns, unknowns.length + 1);
    … …  
    7779    }
    7880
     81    static Instant decodeInstant(short size, boolean littleEndian, InputStream inputStream) throws IOException {
     82        final var timestamp = NumberUtils.decodeLong(size, littleEndian, inputStream);
     83        return Instant.ofEpochSecond(EPOCH_DIFFERENCE + timestamp);
     84    }
     85
    7986    private static double decodeDegrees(long original) {
     87        if (original == FitBaseType.sint32.invalidValue()) {
     88            return Double.NaN;
     89        }
    8090        // signed ints, no need to look into zigzag decoding
    8191        // 0 -> 0 (assumed)

  • applications/editors/josm/plugins/FIT/src/main/java/org/openstreetmap/josm/plugins/fit/lib/global/IFitTimestamp.java

    r36151 r36158  
    77
    88interface IFitTimestamp<T>
    9 permits HeartRateCadenceDistanceSpeed
     9permits FitEvent, HeartRateCadenceDistanceSpeed
    1010{
    1111

  • applications/editors/josm/plugins/FIT/src/main/java/org/openstreetmap/josm/plugins/fit/lib/utils/DevDataUtils.java

    r36151 r36158  
    3838            final String units = devField.units();
    3939            final short fieldSize = fitField.size();
    40             arrayData[index++] = switch (FitBaseType.fromBaseTypeField(devField.fitBaseTypeId())) {
    41             case float64 -> new FitDevDoubleData(fieldName, units,
    42                     NumberUtils.decodeDouble(fieldSize, littleEndian, inputStream));
    43             case float32 -> new FitDevFloatData(fieldName, units,
    44                     NumberUtils.decodeFloat(fieldSize, littleEndian, inputStream));
    45             case enum_, sint8, uint8, uint8z, sint16, uint16, uint16z, sint32 -> new FitDevIntData(fieldName, units,
    46                     NumberUtils.decodeInt(fieldSize, littleEndian, inputStream));
    47             case uint32, uint32z, sint64, uint64, uint64z -> new FitDevLongData(fieldName, units,
    48                     NumberUtils.decodeLong(fieldSize, littleEndian, inputStream));
    49             case string -> new FitDevStringData(fieldName, units, StringUtils.decodeString(inputStream));
    50             case byte_, UNKNOWN -> new FitDevUnknown(fieldName, units, inputStream.readNBytes(fieldSize));
    51             };
     40            arrayData[index++] = getData(FitBaseType.fromBaseTypeField(devField.fitBaseTypeId()), fieldName, units,
     41                    fieldSize, littleEndian, inputStream);
    5242        }
    5343        return new FitDevDataRecord(arrayData);
    5444    }
     45
     46    public static IFitDevData<?> getData(FitBaseType type, String fieldName, String units, short fieldSize,
     47            boolean littleEndian, InputStream inputStream) throws IOException {
     48        return switch (type) {
     49        case float64 -> new FitDevDoubleData(fieldName, units,
     50                NumberUtils.decodeDouble(fieldSize, littleEndian, inputStream));
     51        case float32 -> new FitDevFloatData(fieldName, units,
     52                NumberUtils.decodeFloat(fieldSize, littleEndian, inputStream));
     53        case enum_, sint8, uint8, uint8z, sint16, uint16, uint16z, sint32 -> new FitDevIntData(fieldName, units,
     54                NumberUtils.decodeInt(fieldSize, littleEndian, inputStream));
     55        case uint32, uint32z, sint64, uint64, uint64z -> new FitDevLongData(fieldName, units,
     56                NumberUtils.decodeLong(fieldSize, littleEndian, inputStream));
     57        case string -> new FitDevStringData(fieldName, units, StringUtils.decodeString(inputStream));
     58        case byte_, UNKNOWN -> new FitDevUnknown(fieldName, units, inputStream.readNBytes(fieldSize));
     59        };
     60    }
    5561}
Note: See TracChangeset for help on using the changeset viewer.