source: osm/applications/editors/josm/plugins/lakewalker/Lakewalker/lakewalker.py@ 8781

Last change on this file since 8781 was 6879, checked in by jrreid, 17 years ago

Update lakewalker to automatically create directory to store wms images in

File size: 21.9 KB
Line 
1#!/usr/bin/python
2
3# Lakewalker II.
4# 2007-07-09: Initial public release (v0.2) by Dshpak
5# 2007-07-10: v0.3: Added support for OSM input files. Also reduced start_radius_big, and fixed a division-by-zero error in the degenerate case of point_line_distance().
6# 2007-08-04: Added experimental non-recursive Douglas-Peucker algorithm (--dp-nr). Added --startdir option.
7# 2007-08-06: v0.4: Bounding box support (--left, --right, --top, and --bottom), as well as new --josm mode for JOSM integration. This isn't perfect yet, but it's a good start.
8
9"""Lakewalker II - A tool to automatically download and trace Landsat imagery, to generate OpenStreetMap data.
10
11Requires the Python Imaging Library, available from http://www.pythonware.com/products/pil/"""
12
13version="Lakewalker II v0.4"
14
15# TODO:
16# - Accept threshold tags in OSM input files.
17# - Command line options:
18# - direction (deosil/widdershins)
19# - layer to use (monochrome only?)
20# - additional tags for the way
21# - Landsat download retries (count and delay)
22# - radii for loop detection
23# - fname for z12 tile output (list or script)
24# - path to tilesGen for z12 script output
25# - debug mode (extra tags on nodes) (make this non-uploadable?)
26# - The "got stuck" message should output coords
27# - Better "got stuck" detection (detect mini loops)
28# - Offset nodes outwards by a half-pixel or so, to prevent duplicate segments
29# - Automatic threshold detection
30# - (Correct/tested) non-recursive DP simplification
31#
32#
33# For JOSM integration:
34# - Add a --tmpdir option that controls the location of the Landsat
35# tiles, the results text file, and the lake.osm file (unless it's
36# got an absolute path
37# - Add a --nocache option that keeps WMS tiles in memory but not on disk.
38
39import math
40import os
41import urllib
42from PIL import Image
43import OSMReader
44import time
45import optparse
46
47options = None
48
49start_radius_big = 0.001
50start_radius_small = 0.0002
51
52dirs = ((1, 0), (1, 1), (0, 1), (-1, 1), (-1, 0), (-1, -1), (0, -1), (1, -1))
53dirnames = ["east", "northeast", "north", "northwest", "west", "southwest", "south", "southeast"]
54dirabbrevs = ["e", "ne", "n", "nw", "w", "sw", "s", "se"]
55
56class FatalError(Exception):
57 pass
58
59def message(s):
60 if options.josm_mode:
61 print "m %s" % s
62 else:
63 print s
64
65def error(s):
66 if options.josm_mode:
67 print "e %s" % s
68 else:
69 print s
70
71class BBox:
72 def __init__(self, top = 90, left = -180, bottom = -90, right = 180):
73 self.left = left
74 self.right = right
75 self.top = top
76 self.bottom = bottom
77 def contains(self, loc):
78 (lat, lon) = loc
79 if lat > self.top or lat < self.bottom:
80 return False
81 if (self.right - self.left) % 360 == 0:
82 return True
83 return (lon - self.left) % 360 <= (self.right - self.left) % 360
84
85def download_landsat(c1, c2, width, height, fname):
86 layer = "global_mosaic_base"
87 #style = "IR1"
88 style = options.wmslayer
89
90 (min_lat, min_lon) = c1
91 (max_lat, max_lon) = c2
92
93 message("Downloading Landsat tile for (%.6f,%.6f)-(%.6f,%.6f)." % (min_lat, min_lon, max_lat, max_lon))
94 url = "http://onearth.jpl.nasa.gov/wms.cgi?request=GetMap&layers=%s&styles=%s&srs=EPSG:4326&format=image/png&bbox=%0.6f,%0.6f,%0.6f,%0.6f&width=%d&height=%d" % (layer, style, min_lon, min_lat, max_lon, max_lat, width, height)
95 #print url
96 try:
97 urllib.urlretrieve(url, fname)
98 except IOError, e:
99 raise FatalError("Error downloading tile: %s" % e.strerror)
100 if not os.path.exists(fname):
101 raise FatalError("Error: Could not retreive url %s" % url)
102
103def xy_to_geo(xy):
104 (x, y) = xy
105 (lat, lon) = (y / float(options.resolution), x / float(options.resolution))
106 return (lat, lon)
107
108def geo_to_xy(geo):
109 (lat, lon) = geo
110 coord = lambda L: math.floor(L * options.resolution + 0.5)
111 (x, y) = (coord(lon), coord(lat))
112 return (x, y)
113
114class WMSManager:
115 def __init__(self):
116 self.images = {}
117
118 def get_tile(self, xy):
119 fail_count = 0
120 im = None
121 while im is None and fail_count < 4:
122 (x, y) = xy
123 bottom_left_xy = (int(math.floor(x / options.tilesize)) * options.tilesize, int(math.floor(y / options.tilesize)) * options.tilesize)
124 top_right_xy = (bottom_left_xy[0] + options.tilesize, bottom_left_xy[1] + options.tilesize)
125 fname = options.wmslayer+"/landsat_%d_%d_xy_%d_%d.png" % (options.resolution, options.tilesize, bottom_left_xy[0], bottom_left_xy[1])
126
127 if not os.path.exists(options.wmslayer+"/"):
128 os.mkdir(options.wmslayer)
129
130 im = self.images.get(fname, None)
131 if im is None:
132 if not os.path.exists(fname):
133 bottom_left = xy_to_geo(bottom_left_xy)
134 top_right = xy_to_geo(top_right_xy)
135 download_landsat(bottom_left, top_right, options.tilesize, options.tilesize, fname)
136 if not os.path.exists(fname):
137 raise FatalError("Error: Could not get image file %s" % fname)
138 try:
139 im = Image.open(fname)
140 self.images[fname] = im
141 message("Using imagery in %s for %s." % (fname, xy_to_geo(xy)))
142 except IOError:
143 error("Download was corrupt...Deleting %s..." % fname)
144 os.unlink(fname)
145 im = None
146
147 if im is None:
148 message("Sleeping and retrying download...")
149 time.sleep(4)
150 fail_count = fail_count + 1
151
152 if im is None:
153 #if os.path.exists(fname):
154 # print open(fname).readlines()
155 raise FatalError("Couldn't get image file %s." % fname)
156
157 #return (im, top_left_xy)
158 return (im, bottom_left_xy)
159
160 def get_pixel(self, xy):
161 (x, y) = xy
162 (tile, (tx, ty)) = self.get_tile(xy)
163 tile_xy = (x - tx, (options.tilesize - 1) - (y - ty))
164 #print "%s maps to %s" % (xy, tile_xy)
165 return tile.getpixel(tile_xy)
166
167def trace_lake(loc, threshold, start_dir, bbox):
168 wms = WMSManager()
169 xy = geo_to_xy(loc)
170 nodelist = []
171
172 message("Starting coordinate: %.4f, %.4f" % loc)
173 message("Starting position: %d, %d" % xy)
174
175 if not bbox.contains(loc):
176 raise FatalError("Error: Starting location is outside bounding box!")
177
178 while True:
179 loc = xy_to_geo(xy)
180 if not bbox.contains(loc):
181 break
182
183 v = wms.get_pixel(xy)
184 if v > threshold:
185 break
186
187 xy = (xy[0] + dirs[start_dir][0], xy[1] + dirs[start_dir][1])
188
189 start_xy = xy
190 start_loc = xy_to_geo(xy)
191 message("Found shore at lat %.4f lon %.4f" % start_loc)
192
193 #dirs = ((1, 0), (1, -1), (0, -1), (-1, -1), (-1, 0), (-1, 1), (0, 1), (1, 1))
194 last_dir = start_dir
195
196 detect_loop = False
197
198 for i in xrange(options.maxnodes):
199 if i % 250 == 0:
200 if i > 0:
201 message("%s nodes so far..." % i)
202
203 for d in xrange(1, len(dirs)):
204 new_dir = dirs[(last_dir + d + 4) % 8]
205 test_xy = (xy[0] + new_dir[0], xy[1] + new_dir[1])
206 test_loc = xy_to_geo(test_xy)
207 if not bbox.contains(test_loc):
208 break
209
210 v = wms.get_pixel(test_xy)
211 #print "%s: %s: %s" % (new_dir, test_xy, v)
212 if v > threshold:
213 break
214
215 if d == 8:
216 error("Got stuck.")
217 break
218
219 #print "Moving to %s, direction %s (was %s)" % (test_xy, new_dir, dirs[last_dir])
220 last_dir = (last_dir + d + 4) % 8
221 xy = test_xy
222
223 if xy == start_xy:
224 break
225
226 loc = xy_to_geo(xy)
227 nodelist.append(loc)
228
229 start_proximity = (loc[0] - start_loc[0]) ** 2 + (loc[1] - start_loc[1]) ** 2
230 if detect_loop:
231 if start_proximity < start_radius_small ** 2:
232 break
233 else:
234 if start_proximity > start_radius_big ** 2:
235 detect_loop = True
236 return nodelist
237
238def vertex_reduce(nodes, proximity):
239 test_v = nodes[0]
240 reduced_nodes = [test_v]
241 prox_sq = proximity ** 2
242 for v in nodes:
243 if (v[0] - test_v[0]) ** 2 + (v[1] - test_v[1]) ** 2 > prox_sq:
244 reduced_nodes.append(v)
245 test_v = v
246 return reduced_nodes
247
248def point_line_distance(p0, p1, p2):
249 ((x0, y0), (x1, y1), (x2, y2)) = (p0, p1, p2)
250
251 if x2 == x1 and y2 == y1:
252 # Degenerate cast: the "line" is actually a point.
253 return math.sqrt((x1-x0)**2 + (y1-y0)**2)
254 else:
255 # I don't understand this at all. Thank you, Mathworld.
256 # http://mathworld.wolfram.com/Point-LineDistance2-Dimensional.html
257 return abs((x2-x1)*(y1-y0) - (x1-x0)*(y2-y1)) / math.sqrt((x2-x1)**2 + (y2-y1)**2)
258
259def douglas_peucker(nodes, epsilon):
260 #print "Running DP on %d nodes" % len(nodes)
261 farthest_node = None
262 farthest_dist = 0
263 first = nodes[0]
264 last = nodes[-1]
265
266 for i in xrange(1, len(nodes) - 1):
267 d = point_line_distance(nodes[i], first, last)
268 if d > farthest_dist:
269 farthest_dist = d
270 farthest_node = i
271
272 if farthest_dist > epsilon:
273 seg_a = douglas_peucker(nodes[0:farthest_node+1], epsilon)
274 seg_b = douglas_peucker(nodes[farthest_node:-1], epsilon)
275 #print "Minimized %d nodes to %d + %d nodes" % (len(nodes), len(seg_a), len(seg_b))
276 nodes = seg_a[:-1] + seg_b
277 else:
278 return [nodes[0], nodes[-1]]
279
280 return nodes
281
282def dp_findpoint(nodes, start, end):
283 farthest_node = None
284 farthest_dist = 0
285 #print "dp_findpoint(nodes, %s, %s)" % (start, end)
286 first = nodes[start]
287 last = nodes[end]
288
289 for i in xrange(start + 1, end):
290 d = point_line_distance(nodes[i], first, last)
291 if d > farthest_dist:
292 farthest_dist = d
293 farthest_node = i
294
295 return (farthest_node, farthest_dist)
296
297def douglas_peucker_nonrecursive(nodes, epsilon):
298 #print "Running DP on %d nodes" % len(nodes)
299 command_stack = [(0, len(nodes) - 1)]
300 result_stack = []
301
302 while len(command_stack) > 0:
303 cmd = command_stack.pop()
304 if type(cmd) == tuple:
305 (start, end) = cmd
306 (node, dist) = dp_findpoint(nodes, start, end)
307 if dist > epsilon:
308 command_stack.append("+")
309 command_stack.append((start, node))
310 command_stack.append((node, end))
311 else:
312 result_stack.append((start, end))
313 elif cmd == "+":
314 first = result_stack.pop()
315 second = result_stack.pop()
316 if first[-1] == second[0]:
317 result_stack.append(first + second[1:])
318 #print "Added %s and %s; result is %s" % (first, second, result_stack[-1])
319 else:
320 error("ERROR: Cannot connect nodestrings!")
321 #print first
322 #print second
323 return
324 else:
325 error("ERROR: Can't understand command \"%s\"" % (cmd,))
326 return
327
328 if len(result_stack) == 1:
329 return [nodes[x] for x in result_stack[0]]
330 else:
331 error("ERROR: Command stack is empty but result stack has %d nodes!" % len(result_stack))
332 return
333
334 farthest_node = None
335 farthest_dist = 0
336 first = nodes[0]
337 last = nodes[-1]
338
339 for i in xrange(1, len(nodes) - 1):
340 d = point_line_distance(nodes[i], first, last)
341 if d > farthest_dist:
342 farthest_dist = d
343 farthest_node = i
344
345 if farthest_dist > epsilon:
346 seg_a = douglas_peucker(nodes[0:farthest_node+1], epsilon)
347 seg_b = douglas_peucker(nodes[farthest_node:-1], epsilon)
348 #print "Minimized %d nodes to %d + %d nodes" % (len(nodes), len(seg_a), len(seg_b))
349 nodes = seg_a[:-1] + seg_b
350 else:
351 return [nodes[0], nodes[-1]]
352
353 return nodes
354
355def output_to_josm(lakelist):
356 # Description of JOSM output format:
357 # m text - Status message text, to be displayed in a display window
358 # e text - Error message text
359 # s nnn - Start full node list, nnn tracings following
360 # t nnn - Start tracing node list, nnn nodes following (i.e. there will be one of these for each lake where multiple start points specified)
361 # n lat lon - A node
362 # x - End of data
363 countnodes = 0
364
365 print "s %s" % len(lakelist)
366 for nodelist in lakelist:
367 print "t %s" % len(nodelist)
368 for node in nodelist:
369 print "n %.7f %.7f" % (node[0], node[1])
370
371 countnodes = countnodes + 1
372 if options.MAXLEN == countnodes:
373 print "x"
374 print "t %s" % len(nodelist)
375 print "n %.7f %.7f" % (node[0], node[1])
376 countnodes = 0
377
378 print "x"
379
380def write_osm(f, lakelist, waysize):
381 f.write('<osm version="0.4">')
382 cur_id = -1
383 way_count = 0
384 for nodelist in lakelist:
385 first_node_id = cur_id
386 cur_way = []
387 ways = [cur_way]
388 for loc in nodelist:
389 #f.write(' <node id="%d" lat="%.6f" lon="%.6f"><tag k="order" v="%d"/><tag k="x" v="%d"/><tag k="y" v="%d"/></node>\n' % (cur_id, loc[0], loc[1], -cur_id, geo_to_xy(loc)[0], geo_to_xy(loc)[1]))
390 f.write('<node id="%d" lat="%.6f" lon="%.6f"/>' % (cur_id, loc[0], loc[1]))
391 last_node_id = cur_id
392 cur_id = cur_id - 1
393
394 # print "Nodes: %d, %d" % (first_node_id, last_node_id)
395
396 first_segment_id = cur_id
397 for seg in xrange(first_node_id, last_node_id, -1):
398 f.write('<segment id="%d" from="%d" to="%d"/>' % (cur_id, seg, seg-1))
399 cur_id = cur_id - 1
400 f.write('<segment id="%d" from="%d" to="%d"/>' % (cur_id, last_node_id, first_node_id))
401 last_segment_id = cur_id
402 cur_id = cur_id - 1
403
404 # print "Segments: %d, %d" % (first_segment_id, last_segment_id)
405
406 for seg in xrange(first_segment_id, last_segment_id - 1, -1):
407 if len(cur_way) >= waysize:
408 cur_way = []
409 ways.append(cur_way)
410 cur_way.append(seg)
411 for way in ways:
412 f.write('<way id="%d">' % cur_id)
413 cur_id = cur_id - 1
414 for seg in way:
415 f.write('<seg id="%d"/>' % seg)
416 f.write('<tag k="natural" v="%s"/>' % options.natural_type)
417 f.write('<tag k="source" v="Dshpak_landsat_lakes"/>')
418 f.write('</way>')
419 way_count = way_count + len(ways)
420
421 f.write('</osm>')
422 message("Generated %d %s." % (way_count, ["way", "ways"][way_count > 0]))
423
424def get_locs(infile):
425 nodes = []
426 segments = []
427 ways = []
428 reader = OSMReader.OSMReader()
429 reader.nodeHandler = lambda x: nodes.append(x)
430 reader.segmentHandler = lambda x: segments.append(x)
431 reader.wayHandler = lambda x: ways.append(x)
432 reader.run(file(infile))
433 if len(segments) > 0 or len(ways) > 0:
434 raise FatalError("Error: Input file must only contain nodes -- no segments or ways.")
435 if len(nodes) == 0:
436 raise FatalError("Error: No nodes found in input file.")
437 return [((node.lat, node.lon), options.threshold) for node in nodes]
438
439def main():
440 parser = optparse.OptionParser(version=version)
441
442 parser.add_option("--lat", type="float", metavar="LATITUDE", help="Starting latitude. Required, unless --infile is used..")
443 parser.add_option("--lon", type="float", metavar="LONGITUDE", help="Starting longitude. Required, unless --infile is used.")
444 parser.add_option("--startdir", type="string", default="east", metavar="DIR", help="Direction to travel from start position when seeking land. Defaults to \"east\".")
445 parser.add_option("--infile", "-i", type="string", metavar="FILE", help="OSM file containing nodes representing starting points.")
446 parser.add_option("--out", "-o", default="lake.osm", dest="outfile", metavar="FILE", help="Output filename. Defaults to lake.osm.")
447 parser.add_option("--threshold", "-t", type="int", default="35", metavar="VALUE", help="Maximum gray value to accept as water (based on Landsat IR-1 data). Can be in the range 0-255. Defaults to 35.")
448 parser.add_option("--maxnodes", type="int", default="50000", metavar="N", help="Maximum number of nodes to generate before bailing out. Defaults to 50000.")
449 parser.add_option("--waylength", type="int", dest="MAXLEN", default=500, metavar="MAXLEN", help="Maximum nuber of nodes allowed in one way. Defaults to 250.")
450 parser.add_option("--landsat-res", type="int", default=4000, dest="resolution", metavar="RES", help="Resolution of Landsat tiles, measured in pixels per degree. Defaults to 4000.")
451 parser.add_option("--tilesize", type="int", default=2000, help="Size of one landsat tile, measured in pixels. Defaults to 2000.")
452 parser.add_option("--no-dp", action="store_false", dest="use_dp", default=True, help="Disable Douglas-Peucker line simplification (not recommended)")
453 parser.add_option("--dp-epsilon", type="float", metavar="EPSILON", default=0.0003, help="Accuracy of Douglas-Peucker line simplification, measured in degrees. Lower values give more nodes, and more accurate lines. Defaults to 0.0003.")
454 parser.add_option("--dp-nr", action="store_true", dest="dp_nr", default=False, help="Use experimental non-recursive DP implementation")
455 parser.add_option("--vr", action="store_true", dest="use_vr", default=False, help="Use vertex reduction before applying line simplification (off by default).")
456 parser.add_option("--vr-epsilon", type="float", default=0.0005, metavar="RADIUS", help="Radius used for vertex reduction (measured in degrees). Defaults to 0.0005.")
457 parser.add_option("--water", action="store_const", const="water", dest="natural_type", default="coastline", help="Tag ways as natural=water instead of natural=coastline")
458 parser.add_option("--left", type="float", metavar="LONGITUDE", default=-180, help="Left (west) longitude for bounding box")
459 parser.add_option("--right", type="float", metavar="LONGITUDE", default=180, help="Right (east) longitude for bounding box")
460 parser.add_option("--top", type="float", metavar="LATITUDE", default=90, help="Top (north) latitude for bounding box")
461 parser.add_option("--bottom", type="float", metavar="LATITUDE", default=-90, help="Bottom (south) latitude for bounding box")
462 parser.add_option("--josm", action="store_true", dest="josm_mode", default=False, help="Operate in JOSM plugin mode")
463 parser.add_option("--wms", type="string", dest="wmslayer", default="IR1", help="WMS layer to use")
464
465 global options # Ugly, I know...
466 (options, args) = parser.parse_args()
467
468 if len(args) > 0:
469 parser.print_help()
470 return
471
472 (start_lat, start_lon, infile) = (options.lat, options.lon, options.infile)
473
474 if (start_lat is None or start_lon is None) and infile is None:
475 if not options.josm_mode:
476 parser.print_help()
477 print
478 error("Error: you must specify a starting latitude and longitude.")
479 return
480
481 if infile is not None:
482 if start_lat is not None or start_lon is not None:
483 error("Error: you cannot use both --infile and --lat or --lon.")
484 return
485 try:
486 locs = get_locs(infile)
487 #print locs
488 except FatalError, e:
489 error("%s" % e)
490 return
491 else:
492 locs = [((start_lat, start_lon), options.threshold)]
493
494 dirname = options.startdir.lower()
495 if dirname in dirnames:
496 startdir = dirnames.index(dirname)
497 elif dirname in dirabbrevs:
498 startdir = dirabbrevs.index(dirname)
499 else:
500 error("Error: Can't understand starting direction \"%s\". Vaild options are %s." % (dirname, ", ".join(dirnames + dirabbrevs)))
501 return
502
503 message("Starting direction is %s." % dirnames[startdir])
504
505 bbox = BBox(options.top, options.left, options.bottom, options.right)
506
507 try:
508 lakes = []
509 for (loc, threshold) in locs:
510 nodes = trace_lake(loc, threshold, startdir, bbox)
511
512 message("%d nodes generated." % len(nodes))
513
514 if len(nodes) > 0:
515 if options.use_vr:
516 nodes = vertex_reduce(nodes, options.vr_epsilon)
517 message("After vertex reduction, %d nodes remain." % len(nodes))
518
519 if options.use_dp:
520 try:
521 if options.dp_nr:
522 nodes = douglas_peucker_nonrecursive(nodes, options.dp_epsilon)
523 #print "Final result: %s" % (nodes,)
524 else:
525 nodes = douglas_peucker(nodes, options.dp_epsilon)
526 message("After Douglas-Peucker approximation, %d nodes remain." % len(nodes))
527 except FatalError, e:
528 raise e
529 except:
530 raise FatalError("Line simplification failed -- there are probably too many nodes.")
531
532 lakes.append(nodes)
533
534 if options.josm_mode:
535 output_to_josm(lakes)
536 else:
537 message("Writing to %s" % options.outfile)
538 f = open(options.outfile, "w")
539 write_osm(f, lakes, options.waylength)
540 f.close()
541
542 #tiles = tilelist(nodes)
543
544 #for tile in tiles:
545 # print "./tilesGen.pl xy %d %d; ./upload.pl" % tile
546
547 #print tiles
548 except FatalError, e:
549 error("%s" % e)
550 error("Bailing out...")
551
552if __name__ == "__main__":
553 main()
554
Note: See TracBrowser for help on using the repository browser.