source: josm/trunk/src/org/openstreetmap/josm/actions/downloadtasks/DownloadOsmTask.java

Last change on this file was 19080, checked in by taylor.smock, 6 weeks ago

See #23671: Deprecate Utils#isBlank and replace instances of it with Utils#isStripEmpty

As noted in r19079, the two functions were identical in behavior.
  • Property svn:eol-style set to native
File size: 22.9 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.actions.downloadtasks;
3
4import static java.util.function.Predicate.not;
5import static org.openstreetmap.josm.tools.I18n.tr;
6
7import java.io.IOException;
8import java.net.MalformedURLException;
9import java.net.URL;
10import java.util.ArrayList;
11import java.util.Arrays;
12import java.util.Collection;
13import java.util.Collections;
14import java.util.HashSet;
15import java.util.Objects;
16import java.util.Optional;
17import java.util.Set;
18import java.util.concurrent.Future;
19import java.util.regex.Matcher;
20import java.util.regex.Pattern;
21import java.util.stream.Stream;
22
23import org.openstreetmap.josm.data.Bounds;
24import org.openstreetmap.josm.data.DataSource;
25import org.openstreetmap.josm.data.ProjectionBounds;
26import org.openstreetmap.josm.data.ViewportData;
27import org.openstreetmap.josm.data.coor.LatLon;
28import org.openstreetmap.josm.data.osm.DataSet;
29import org.openstreetmap.josm.data.osm.OsmPrimitive;
30import org.openstreetmap.josm.data.osm.Relation;
31import org.openstreetmap.josm.data.osm.Way;
32import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor;
33import org.openstreetmap.josm.gui.MainApplication;
34import org.openstreetmap.josm.gui.MapFrame;
35import org.openstreetmap.josm.gui.PleaseWaitRunnable;
36import org.openstreetmap.josm.gui.io.UpdatePrimitivesTask;
37import org.openstreetmap.josm.gui.layer.OsmDataLayer;
38import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
39import org.openstreetmap.josm.gui.progress.ProgressMonitor;
40import org.openstreetmap.josm.io.BoundingBoxDownloader;
41import org.openstreetmap.josm.io.Compression;
42import org.openstreetmap.josm.io.OsmServerLocationReader;
43import org.openstreetmap.josm.io.OsmServerReader;
44import org.openstreetmap.josm.io.OsmTransferCanceledException;
45import org.openstreetmap.josm.io.OsmTransferException;
46import org.openstreetmap.josm.io.OverpassDownloadReader;
47import org.openstreetmap.josm.io.UrlPatterns.OsmUrlPattern;
48import org.openstreetmap.josm.spi.preferences.Config;
49import org.openstreetmap.josm.tools.Logging;
50import org.openstreetmap.josm.tools.Utils;
51import org.xml.sax.SAXException;
52
53/**
54 * Open the download dialog and download the data.
55 * Run in the worker thread.
56 */
57public class DownloadOsmTask extends AbstractDownloadTask<DataSet> {
58
59 protected Bounds currentBounds;
60 protected DownloadTask downloadTask;
61
62 protected String newLayerName;
63
64 /** This allows subclasses to ignore this warning */
65 protected boolean warnAboutEmptyArea = true;
66
67 protected static final String OVERPASS_INTERPRETER_DATA = "interpreter?data=";
68
69 private static final String NO_DATA_FOUND = tr("No data found in this area.");
70 static {
71 PostDownloadHandler.addNoDataErrorMessage(NO_DATA_FOUND);
72 }
73
74 @Override
75 public String[] getPatterns() {
76 if (this.getClass() == DownloadOsmTask.class) {
77 return patterns(OsmUrlPattern.class);
78 } else {
79 return super.getPatterns();
80 }
81 }
82
83 @Override
84 public String getTitle() {
85 if (this.getClass() == DownloadOsmTask.class) {
86 return tr("Download OSM");
87 } else {
88 return super.getTitle();
89 }
90 }
91
92 @Override
93 public Future<?> download(DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) {
94 return download(new BoundingBoxDownloader(downloadArea), settings, downloadArea, progressMonitor);
95 }
96
97 /**
98 * Asynchronously launches the download task for a given bounding box.
99 *
100 * Set <code>progressMonitor</code> to null, if the task should create, open, and close a progress monitor.
101 * Set progressMonitor to {@link NullProgressMonitor#INSTANCE} if progress information is to
102 * be discarded.
103 *
104 * You can wait for the asynchronous download task to finish by synchronizing on the returned
105 * {@link Future}, but make sure not to freeze up JOSM. Example:
106 * <pre>
107 * Future&lt;?&gt; future = task.download(...);
108 * // DON'T run this on the Swing EDT or JOSM will freeze
109 * future.get(); // waits for the download task to complete
110 * </pre>
111 *
112 * The following example uses a pattern which is better suited if a task is launched from
113 * the Swing EDT:
114 * <pre>
115 * final Future&lt;?&gt; future = task.download(...);
116 * Runnable runAfterTask = new Runnable() {
117 * public void run() {
118 * // this is not strictly necessary because of the type of executor service
119 * // Main.worker is initialized with, but it doesn't harm either
120 * //
121 * future.get(); // wait for the download task to complete
122 * doSomethingAfterTheTaskCompleted();
123 * }
124 * }
125 * MainApplication.worker.submit(runAfterTask);
126 * </pre>
127 * @param reader the reader used to parse OSM data (see {@link OsmServerReader#parseOsm})
128 * @param settings download settings
129 * @param downloadArea the area to download
130 * @param progressMonitor the progressMonitor
131 * @return the future representing the asynchronous task
132 * @since 13927
133 */
134 public Future<?> download(OsmServerReader reader, DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) {
135 return download(new DownloadTask(settings, reader, progressMonitor, zoomAfterDownload), downloadArea);
136 }
137
138 protected Future<?> download(DownloadTask downloadTask, Bounds downloadArea) {
139 this.downloadTask = downloadTask;
140 this.currentBounds = new Bounds(downloadArea);
141 // We need submit instead of execute so we can wait for it to finish and get the error
142 // message if necessary. If no one calls getErrorMessage() it just behaves like execute.
143 return MainApplication.worker.submit(downloadTask);
144 }
145
146 /**
147 * This allows subclasses to perform operations on the URL before {@link #loadUrl} is performed.
148 * @param url the original URL
149 * @return the modified URL
150 */
151 protected String modifyUrlBeforeLoad(String url) {
152 return url;
153 }
154
155 /**
156 * Loads a given URL from the OSM Server
157 * @param settings download settings
158 * @param url The URL as String
159 */
160 @Override
161 public Future<?> loadUrl(DownloadParams settings, String url, ProgressMonitor progressMonitor) {
162 String newUrl = modifyUrlBeforeLoad(url);
163 Optional<OsmUrlPattern> urlPattern = Arrays.stream(OsmUrlPattern.values()).filter(p -> p.matches(newUrl)).findFirst();
164 downloadTask = new DownloadTask(settings, getOsmServerReader(newUrl), progressMonitor, true, Compression.byExtension(newUrl));
165 currentBounds = null;
166 // Extract .osm filename from URL to set the new layer name
167 extractOsmFilename(settings, urlPattern.orElse(OsmUrlPattern.EXTERNAL_OSM_FILE).pattern(), newUrl);
168 return MainApplication.worker.submit(downloadTask);
169 }
170
171 protected OsmServerReader getOsmServerReader(String url) {
172 try {
173 String host = new URL(url).getHost();
174 for (String knownOverpassServer : OverpassDownloadReader.OVERPASS_SERVER_HISTORY.get()) {
175 if (host.equals(new URL(knownOverpassServer).getHost())) {
176 int index = url.indexOf(OVERPASS_INTERPRETER_DATA);
177 if (index > 0) {
178 return new OverpassDownloadReader(new Bounds(LatLon.ZERO), knownOverpassServer,
179 Utils.decodeUrl(url.substring(index + OVERPASS_INTERPRETER_DATA.length())));
180 }
181 }
182 }
183 } catch (MalformedURLException e) {
184 Logging.error(e);
185 }
186 return new OsmServerLocationReader(url);
187 }
188
189 protected final void extractOsmFilename(DownloadParams settings, String pattern, String url) {
190 newLayerName = settings.getLayerName();
191 if (Utils.isEmpty(newLayerName)) {
192 Matcher matcher = Pattern.compile(pattern).matcher(url);
193 newLayerName = matcher.matches() && matcher.groupCount() > 0 ? Utils.decodeUrl(matcher.group(1)) : null;
194 }
195 }
196
197 @Override
198 public void cancel() {
199 if (downloadTask != null) {
200 downloadTask.cancel();
201 }
202 }
203
204 @Override
205 public boolean isSafeForRemotecontrolRequests() {
206 return true;
207 }
208
209 @Override
210 public ProjectionBounds getDownloadProjectionBounds() {
211 return downloadTask != null ? downloadTask.computeBbox(currentBounds).orElse(null) : null;
212 }
213
214 protected Collection<OsmPrimitive> searchPotentiallyDeletedPrimitives(DataSet ds) {
215 return downloadTask.searchPrimitivesToUpdate(currentBounds, ds);
216 }
217
218 protected final void rememberDownloadedBounds(Bounds bounds) {
219 if (bounds != null) {
220 Config.getPref().put("osm-download.bounds", bounds.encodeAsString(";"));
221 }
222 }
223
224 /**
225 * Superclass of internal download task.
226 * @since 7636
227 */
228 public abstract static class AbstractInternalTask extends PleaseWaitRunnable {
229
230 protected final DownloadParams settings;
231 protected final boolean zoomAfterDownload;
232 protected DataSet dataSet;
233
234 /**
235 * Constructs a new {@code AbstractInternalTask}.
236 * @param settings download settings
237 * @param title message for the user
238 * @param ignoreException If true, exception will be propagated to calling code. If false then
239 * exception will be thrown directly in EDT. When this runnable is executed using executor framework
240 * then use false unless you read result of task (because exception will get lost if you don't)
241 * @param zoomAfterDownload If true, the map view will zoom to download area after download
242 */
243 protected AbstractInternalTask(DownloadParams settings, String title, boolean ignoreException, boolean zoomAfterDownload) {
244 super(title, ignoreException);
245 this.settings = Objects.requireNonNull(settings);
246 this.zoomAfterDownload = zoomAfterDownload;
247 }
248
249 /**
250 * Constructs a new {@code AbstractInternalTask}.
251 * @param settings download settings
252 * @param title message for the user
253 * @param progressMonitor progress monitor
254 * @param ignoreException If true, exception will be propagated to calling code. If false then
255 * exception will be thrown directly in EDT. When this runnable is executed using executor framework
256 * then use false unless you read result of task (because exception will get lost if you don't)
257 * @param zoomAfterDownload If true, the map view will zoom to download area after download
258 */
259 protected AbstractInternalTask(DownloadParams settings, String title, ProgressMonitor progressMonitor, boolean ignoreException,
260 boolean zoomAfterDownload) {
261 super(title, progressMonitor, ignoreException);
262 this.settings = Objects.requireNonNull(settings);
263 this.zoomAfterDownload = zoomAfterDownload;
264 }
265
266 protected OsmDataLayer getEditLayer() {
267 return MainApplication.getLayerManager().getEditLayer();
268 }
269
270 private static Stream<OsmDataLayer> getModifiableDataLayers() {
271 return MainApplication.getLayerManager().getLayersOfType(OsmDataLayer.class)
272 .stream().filter(OsmDataLayer::isDownloadable);
273 }
274
275 /**
276 * Returns the number of modifiable data layers
277 * @return number of modifiable data layers
278 * @since 13434
279 */
280 protected long getNumModifiableDataLayers() {
281 return getModifiableDataLayers().count();
282 }
283
284 /**
285 * Returns the first modifiable data layer
286 * @return the first modifiable data layer
287 * @since 13434
288 */
289 protected OsmDataLayer getFirstModifiableDataLayer() {
290 return getModifiableDataLayers().findFirst().orElse(null);
291 }
292
293 /**
294 * Creates a name for a new layer by utilizing the settings ({@link DownloadParams#getLayerName()}) or
295 * {@link OsmDataLayer#createNewName()} if the former option is {@code null}.
296 *
297 * @return a name for a new layer
298 * @since 14347
299 */
300 protected String generateLayerName() {
301 return Optional.ofNullable(settings.getLayerName())
302 .filter(not(Utils::isStripEmpty))
303 .orElse(OsmDataLayer.createNewName());
304 }
305
306 /**
307 * Can be overridden (e.g. by plugins) if a subclass of {@link OsmDataLayer} is needed.
308 * If you want to change how the name is determined, consider overriding
309 * {@link #generateLayerName()} instead.
310 *
311 * @param ds the dataset on which the layer is based, must be non-null
312 * @param layerName the name of the new layer, must be either non-blank or non-present
313 * @return a new instance of {@link OsmDataLayer} constructed with the given arguments
314 * @since 14347
315 */
316 protected OsmDataLayer createNewLayer(final DataSet ds, final Optional<String> layerName) {
317 if (layerName.filter(Utils::isStripEmpty).isPresent()) {
318 throw new IllegalArgumentException("Blank layer name!");
319 }
320 return new OsmDataLayer(
321 Objects.requireNonNull(ds, "dataset parameter"),
322 layerName.orElseGet(this::generateLayerName),
323 null
324 );
325 }
326
327 /**
328 * Convenience method for {@link #createNewLayer(DataSet, Optional)}, uses the dataset
329 * from field {@link #dataSet} and applies the settings from field {@link #settings}.
330 *
331 * @param layerName an optional layer name, must be non-blank if the [Optional] is present
332 * @return a newly constructed layer
333 * @since 14347
334 */
335 protected final OsmDataLayer createNewLayer(final Optional<String> layerName) {
336 Optional.ofNullable(settings.getDownloadPolicy())
337 .ifPresent(dataSet::setDownloadPolicy);
338 Optional.ofNullable(settings.getUploadPolicy())
339 .ifPresent(dataSet::setUploadPolicy);
340 if (dataSet.isLocked() && !settings.isLocked()) {
341 dataSet.unlock();
342 } else if (!dataSet.isLocked() && settings.isLocked()) {
343 dataSet.lock();
344 }
345 return createNewLayer(dataSet, layerName);
346 }
347
348 protected Optional<ProjectionBounds> computeBbox(Bounds bounds) {
349 BoundingXYVisitor v = new BoundingXYVisitor();
350 if (bounds != null) {
351 v.visit(bounds);
352 } else {
353 v.computeBoundingBox(dataSet.getNodes());
354 }
355 return Optional.ofNullable(v.getBounds());
356 }
357
358 protected OsmDataLayer addNewLayerIfRequired(String newLayerName) {
359 long numDataLayers = getNumModifiableDataLayers();
360 if (settings.isNewLayer() || numDataLayers == 0 || (numDataLayers > 1 && getEditLayer() == null)) {
361 // the user explicitly wants a new layer, we don't have any layer at all
362 // or it is not clear which layer to merge to
363 final OsmDataLayer layer = createNewLayer(Optional.ofNullable(newLayerName).filter(not(Utils::isStripEmpty)));
364 MainApplication.getLayerManager().addLayer(layer, zoomAfterDownload);
365 return layer;
366 }
367 return null;
368 }
369
370 protected void loadData(String newLayerName, Bounds bounds) {
371 OsmDataLayer layer = addNewLayerIfRequired(newLayerName);
372 if (layer == null) {
373 layer = getEditLayer();
374 if (layer == null || !layer.isDownloadable()) {
375 layer = getFirstModifiableDataLayer();
376 }
377 Collection<OsmPrimitive> primitivesToUpdate = searchPrimitivesToUpdate(bounds, layer.getDataSet());
378 layer.mergeFrom(dataSet);
379 MapFrame map = MainApplication.getMap();
380 if (map != null && zoomAfterDownload) {
381 computeBbox(bounds).map(ViewportData::new).ifPresent(map.mapView::zoomTo);
382 }
383 if (!primitivesToUpdate.isEmpty()) {
384 MainApplication.worker.submit(new UpdatePrimitivesTask(layer, primitivesToUpdate));
385 }
386 }
387 layer.onPostDownloadFromServer(); // for existing and newly added layer, see #19816
388 }
389
390 /**
391 * Look for primitives deleted on server (thus absent from downloaded data)
392 * but still present in existing data layer
393 * @param bounds download bounds
394 * @param ds existing data set
395 * @return the primitives to update
396 */
397 protected Collection<OsmPrimitive> searchPrimitivesToUpdate(Bounds bounds, DataSet ds) {
398 if (bounds == null)
399 return Collections.emptySet();
400 Collection<OsmPrimitive> col = new ArrayList<>();
401 ds.searchNodes(bounds.toBBox()).stream().filter(n -> !n.isNew() && !dataSet.containsNode(n)).forEachOrdered(col::add);
402 if (!col.isEmpty()) {
403 Set<Way> ways = new HashSet<>();
404 Set<Relation> rels = new HashSet<>();
405 for (OsmPrimitive n : col) {
406 for (OsmPrimitive ref : n.getReferrers()) {
407 if (ref.isNew()) {
408 continue;
409 } else if (ref instanceof Way) {
410 ways.add((Way) ref);
411 } else if (ref instanceof Relation) {
412 rels.add((Relation) ref);
413 }
414 }
415 }
416 ways.stream().filter(w -> !dataSet.containsWay(w)).forEachOrdered(col::add);
417 rels.stream().filter(r -> !dataSet.containsRelation(r)).forEachOrdered(col::add);
418 }
419 return Collections.unmodifiableCollection(col);
420 }
421 }
422
423 protected class DownloadTask extends AbstractInternalTask {
424 protected final OsmServerReader reader;
425 protected final Compression compression;
426
427 /**
428 * Constructs a new {@code DownloadTask}.
429 * @param settings download settings
430 * @param reader OSM data reader
431 * @param progressMonitor progress monitor
432 * @since 13927
433 */
434 public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor) {
435 this(settings, reader, progressMonitor, true);
436 }
437
438 /**
439 * Constructs a new {@code DownloadTask}.
440 * @param settings download settings
441 * @param reader OSM data reader
442 * @param progressMonitor progress monitor
443 * @param zoomAfterDownload If true, the map view will zoom to download area after download
444 * @since 13927
445 */
446 public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload) {
447 this(settings, reader, progressMonitor, zoomAfterDownload, Compression.NONE);
448 }
449
450 /**
451 * Constructs a new {@code DownloadTask}.
452 * @param settings download settings
453 * @param reader OSM data reader
454 * @param progressMonitor progress monitor
455 * @param zoomAfterDownload If true, the map view will zoom to download area after download
456 * @param compression compression to use
457 * @since 15784
458 */
459 public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload,
460 Compression compression) {
461 super(settings, tr("Downloading data"), progressMonitor, false, zoomAfterDownload);
462 this.reader = Objects.requireNonNull(reader);
463 this.compression = compression;
464 }
465
466 protected DataSet parseDataSet() throws OsmTransferException {
467 ProgressMonitor subTaskMonitor = progressMonitor.createSubTaskMonitor(ProgressMonitor.ALL_TICKS, false);
468 // Don't call parseOsm signature with compression if not needed, too many implementations to update before to avoid side effects
469 return compression != null && compression != Compression.NONE ?
470 reader.parseOsm(subTaskMonitor, compression) : reader.parseOsm(subTaskMonitor);
471 }
472
473 @Override
474 public void realRun() throws IOException, SAXException, OsmTransferException {
475 try {
476 if (isCanceled())
477 return;
478 dataSet = parseDataSet();
479 } catch (OsmTransferException e) {
480 if (isCanceled()) {
481 Logging.info(tr("Ignoring exception because download has been canceled. Exception was: {0}", e.toString()));
482 return;
483 }
484 if (e instanceof OsmTransferCanceledException) {
485 setCanceled(true);
486 return;
487 } else {
488 rememberException(e);
489 }
490 DownloadOsmTask.this.setFailed(true);
491 }
492 }
493
494 @Override
495 protected void finish() {
496 if (isFailed() || isCanceled())
497 return;
498 if (dataSet == null)
499 return; // user canceled download or error occurred
500 if (dataSet.allPrimitives().isEmpty()) {
501 if (warnAboutEmptyArea) {
502 rememberErrorMessage(NO_DATA_FOUND);
503 }
504 String remark = dataSet.getRemark();
505 if (!Utils.isEmpty(remark)) {
506 rememberErrorMessage(remark);
507 }
508 if (!(reader instanceof BoundingBoxDownloader)
509 || ((BoundingBoxDownloader) reader).considerAsFullDownload()) {
510 // need to synthesize a download bounds lest the visual indication of downloaded area doesn't work
511 dataSet.addDataSource(new DataSource(
512 currentBounds != null ? currentBounds : new Bounds(LatLon.ZERO), "OpenStreetMap server"));
513 }
514 }
515
516 rememberDownloadedBounds(currentBounds);
517 rememberDownloadedData(dataSet);
518 loadData(newLayerName, currentBounds);
519 }
520
521 @Override
522 protected void cancel() {
523 setCanceled(true);
524 if (reader != null) {
525 reader.cancel();
526 }
527 }
528 }
529
530 @Override
531 public String getConfirmationMessage(URL url) {
532 if (OsmUrlPattern.OSM_API_URL.matches(url)) {
533 Collection<String> items = new ArrayList<>();
534 items.add(tr("OSM Server URL:") + ' ' + url.getHost());
535 items.add(tr("Command")+": "+url.getPath());
536 if (url.getQuery() != null) {
537 items.add(tr("Request details: {0}", url.getQuery().replaceAll(",\\s*", ", ")));
538 }
539 return Utils.joinAsHtmlUnorderedList(items);
540 }
541 return null;
542 }
543}
Note: See TracBrowser for help on using the repository browser.