source: osm/applications/editors/josm/i18n/launchpad.pl@ 36423

Last change on this file since 36423 was 36406, checked in by stoecker, 3 weeks ago

I18n update
  • Property svn:executable set to *
  • Property svn:keywords set to Revision
File size: 13.8 KB
Line 
1#!/usr/bin/perl -w
2# -CDSL would be better than explicit encoding settings
3
4use strict;
5use utf8;
6
7my ($user, $pwd);
8
9# Three ways to handle login data:
10 # Enter data directly in these two lines (Be careful witn svn checkin later!)
11 # create a file "launchpad.pl_credits" or ~/.josm_i18n_launchpad.pl_credits
12 # containing the two lines with proper values
13 # leave credits empty and enter them on runtime
14$user = '';
15$pwd = '';
16
17# list of supported languages
18my %lang = map {$_ => 1} (
19"ar", "ast", "bg", "be", "ca", "ca\@valencia", "cs", "cy", "da", "de",
20"el", "en_AU", "en_CA", "en_GB", "eo", "es", "et", "fa", "fi", "fr", "gl",
21"hu", "id", "is", "it", "ja", "ko", "km", "lt", "mr", "nb",
22"nl", "pl", "pt", "pt_BR", "ru", "sk", "sr\@latin", "sv", "tr", "uk",
23"zh_CN", "zh_TW"
24);
25
26my $revision = '$Revision: 36406 $';
27$revision =~ s/^.*?(\d+).*$/$1/;
28my $agent = "JOSM_Launchpad/1.$revision";
29
30my $debugfile = 0;#1;
31my $cleanall = 0;#1;
32
33open TXTFILE,">","launchpad_debug.log" or die if $debugfile;
34
35if($#ARGV != 0)
36{
37 warn "No argument given (try Launchpad download URL, \"bzr\", \"bzronly\", \"upload\" or \"download\").";
38 system "ant";
39}
40elsif($ARGV[0] eq "bzr" || $ARGV[0] eq "bzronly")
41{
42 mkdir "build";
43 die "Could not change into new data dir." if !chdir "build";
44 system "bzr export -v josm_trans lp:~openstreetmap/josm/josm_trans";
45 chdir "..";
46 copypo("build/josm_trans/josm");
47 system "rm -rv build/josm_trans";
48 if($ARGV[0] ne "bzronly")
49 {
50 system "ant";
51 }
52}
53elsif($ARGV[0] eq "upload")
54{
55 potupload();
56}
57elsif($ARGV[0] eq "uploadall")
58{
59 makeupload();
60}
61elsif($ARGV[0] eq "uploadallsingle")
62{
63 makeuploadsingle();
64}
65elsif($ARGV[0] eq "approveall")
66{
67 approveall(dologin());
68}
69elsif($ARGV[0] eq "download")
70{
71 podownload();
72}
73elsif($ARGV[0] eq "stats")
74{
75 getstats();
76}
77else
78{
79 mkdir "build";
80 mkdir "build/josm_trans";
81 die "Could not change into new data dir." if !chdir "build/josm_trans";
82 system "wget $ARGV[0]";
83 system "tar -xf laun*";
84 chdir "../..";
85 copypo("build/josm_trans");
86 system "rm -rv build/josm_trans";
87 system "ant";
88}
89
90sub approveall
91{
92 my ($mech) = @_;
93 print "Trying to approve upload.\n";
94 $mech->get("https://translations.launchpad.net/josm/trunk/+imports?field.filter_status=NEEDS_REVIEW&field.filter_extension=all");
95 my @links;
96 foreach my $line (split("\n", $mech->content()))
97 {
98 push (@links, $1) if $line =~ /href="(\/\+imports\/\d+)"/;
99 }
100 if(!@links)
101 {
102 warn "No upload found in import list, upload possibly failed.";
103 }
104 else
105 {
106 foreach my $link (@links)
107 {
108 eval
109 {
110 print "Approve $link upload.\n";
111 $mech->get("https://translations.launchpad.net$link");
112 $mech->form_with_fields("field.potemplate");
113 $mech->select("field.potemplate", "josm");
114 $mech->click_button(name => "field.actions.approve");
115 };
116 print $@ if $@;
117 }
118 }
119}
120
121sub makeuploadsingle
122{
123 my $count = 1;
124 my $outdate = `date -u +"%Y-%m-%dT%H_%M_%S"`;
125 my $mech = dologin();
126 $outdate =~ s/[\r\n]+//;
127 mkdir "build/josm";
128 system "rm build/josm/*";
129 system "cp po/*.po po/josm.pot build/josm";
130 chdir "build";
131 print "Starting upload ($outdate).\n";
132 if(!$count)
133 {
134 system "tar -cjf launchpad_upload_josm_$outdate.tar.bz2 josm";
135 $mech->get("https://translations.launchpad.net/josm/trunk/+translations-upload");
136 $mech->submit_form(with_fields => {"file" => "launchpad_upload_josm_$outdate.tar.bz2"});
137 sleep(10);
138 }
139 else
140 {
141 my @files = sort glob("josm/*.po");
142 my $num = 1;
143 while($#files >= 0)
144 {
145 my @f = splice(@files, 0, $count);
146 my $file = "launchpad_upload_josm_${outdate}_$num.tar.bz2";
147 print "Create file $file (".join(" ",@f).").\n";
148 system "tar -cjf $file ".join(" ",@f);
149 print "Upload file $file.\n";
150 $mech->get("https://translations.launchpad.net/josm/trunk/+translations-upload");
151 $mech->submit_form(with_fields => {"file" => $file});
152 sleep(10);
153 ++$num;
154 }
155 }
156 system "rm -rv josm";
157 chdir "..";
158 approveall($mech);
159}
160
161sub makeupload
162{
163 my $count = 11;
164 my $outdate = `date -u +"%Y-%m-%dT%H_%M_%S"`;
165 my $mech = dologin();
166 $outdate =~ s/[\r\n]+//;
167 mkdir "build/josm";
168 system "rm build/josm/*";
169 system "cp po/*.po po/josm.pot build/josm";
170 chdir "build";
171 print "Starting upload ($outdate).\n";
172 if(!$count)
173 {
174 system "tar -cjf launchpad_upload_josm_$outdate.tar.bz2 josm";
175 $mech->get("https://translations.launchpad.net/josm/trunk/+translations-upload");
176 $mech->submit_form(with_fields => {"file" => "launchpad_upload_josm_$outdate.tar.bz2"});
177 sleep(10);
178 }
179 else
180 {
181 my @files = sort glob("josm/*.po");
182 my $num = 1;
183 while($#files >= 0)
184 {
185 my @f = splice(@files, 0, $count);
186 my $file = "launchpad_upload_josm_${outdate}_$num.tar.bz2";
187 print "Create file $file (".join(" ",@f).").\n";
188 system "tar -cjf $file ".join(" ",@f);
189 print "Upload file $file.\n";
190 $mech->get("https://translations.launchpad.net/josm/trunk/+translations-upload");
191 $mech->submit_form(with_fields => {"file" => $file});
192 sleep(10);
193 ++$num;
194 }
195 }
196 system "rm -rv josm";
197 chdir "..";
198 approveall($mech);
199}
200
201sub copypo
202{
203 my ($path) = @_;
204 foreach my $name (sort split("\n", `find $path -name "*.po"`))
205 {
206 $name =~ /([a-zA-Z_@]+)\.po/;
207 if($lang{$1})
208 {
209 system("cp", "-v", $name, "po/$1.po");
210 }
211 elsif($cleanall)
212 {
213 local $/; undef $/;
214 open FILE,"<",$name or die;
215 my $x = <FILE>;
216 close FILE;
217 $x =~ s/\n\n.*$/\n/s;
218 open FILE,">","po/$1.po" or die;
219 print FILE $x;
220 close FILE;
221 }
222 }
223}
224
225sub dologin
226{
227 require WWW::Mechanize;
228
229 my $mech = WWW::Mechanize->new("agent" => $agent);
230
231 #$mech->add_handler("request_send" => sub {
232 # my($request, $ua, $h) = @_;
233 # printf "FORM: %s\n", $request->content();
234 # return undef;
235 #});
236 $mech->get("https://translations.launchpad.net/josm/trunk/+login");
237 #print $mech->status() ." - ". $mech->uri()."\n";
238 $mech->submit_form(form_number => 1);
239 getcredits();
240 #print $mech->status() ." - ". $mech->uri()."\n";
241 $mech->submit_form(form_id => "login-form", fields => {"email" => $user, "password" => $pwd});
242 #$mech->dump_headers();
243 #print $mech->status() ." - ". $mech->uri()."\n";
244 #print $mech->content();
245 my $form = $mech->form_name("decideform");
246 die "Could not login.\n" if !$form;
247 my %par = ("yes" => ""); # We need to add "yes" or it does not work
248 foreach my $p ($form->param)
249 {
250 $par{$p} = $form->value($p);
251 }
252 $mech->post($form->action, \%par);
253 #$mech->dump_headers();
254 #print $mech->content();
255 #print $mech->status() ." - ". $mech->uri()."\n";
256 #$mech->dump_forms();
257 return $mech;
258}
259
260sub potupload
261{
262 my $mech = dologin();
263 print "Starting upload.\n";
264 sleep(2);
265 $mech->get("https://translations.launchpad.net/josm/trunk/+translations-upload");
266 chdir("po");
267 $mech->submit_form(with_fields => {"file" => "josm.pot"});
268 print "Start approving of upload.\n";
269 for(1..10)
270 {
271 eval
272 {
273 sleep(10);
274 print "Trying to approve upload.\n";
275 $mech->get("https://translations.launchpad.net/josm/trunk/+imports?field.filter_status=NEEDS_REVIEW&field.filter_extension=pot");
276 my @links;
277 foreach my $line (split("\n", $mech->content()))
278 {
279 if($line =~ /href="(\/\+imports\/\d+)"/)
280 {
281 push (@links, $1);
282 print TXTFILE $line if $debugfile;
283 }
284 }
285 if(!@links)
286 {
287 print TXTFILE $mech->content() if $debugfile;
288 die "Upload not found in import list, upload possibly failed.";
289 }
290 elsif(@links > 1)
291 {
292 die "More than one upload found in import list, cannot approve.";
293 }
294 else
295 {
296 $mech->get("https://translations.launchpad.net$links[0]");
297 if($debugfile)
298 {
299 print TXTFILE "LINK: $links[0]\n";
300 $mech->dump_headers(\*TXTFILE);
301 print TXTFILE $mech->content();
302 print TXTFILE $mech->status() ." - ". $mech->uri()."\n";
303 $mech->dump_forms(\*TXTFILE);
304 }
305 $mech->submit_form(form_name => "launchpadform", button => "field.actions.approve");
306 if(!($mech->content() =~ /There are no entries that match this filtering/))
307 {
308 die "Approving possibly failed.";
309 }
310 }
311 };
312 print $@ if $@;
313 last if !$@;
314 }
315
316 chdir("..");
317}
318
319sub podownload
320{
321 my $mech = dologin();
322 $mech->get("https://translations.launchpad.net/josm/trunk/+export");
323 $mech->submit_form(with_fields => {"format" => "PO"});
324 if(!($mech->content() =~ /receive an email shortly/))
325 {
326 warn "Error requesting file\n";
327 }
328}
329
330sub getcredits
331{
332 if(!$user || !$pwd)
333 {
334 require Term::ReadKey;
335 local undef $/;
336 if(open FILE, "launchpad.pl_credits")
337 {
338 eval <FILE>;
339 close FILE;
340 }
341 elsif(open FILE, "$ENV{HOME}/.josm_i18n_launchpad.pl_credits")
342 {
343 eval <FILE>;
344 close FILE;
345 }
346
347 if(!$user)
348 {
349 Term::ReadKey::ReadMode(4); # Turn off controls keys
350 printf("Enter username: ");
351 for(;;)
352 {
353 my $c = getc();
354 print $c;
355 last if $c eq "\n";
356 $user .= $c;
357 }
358 Term::ReadKey::ReadMode(0);
359 }
360
361 if(!$pwd)
362 {
363 Term::ReadKey::ReadMode(4); # Turn off controls keys
364 printf("Enter password: ");
365 for(;;)
366 {
367 my $c = getc();
368 last if $c eq "\n";
369 print "*";
370 $pwd .= $c;
371 }
372 print "\n";
373 Term::ReadKey::ReadMode(0);
374 }
375 }
376}
377
378sub doget
379{
380 my ($mech, $page, $arg) = @_;
381 for(my $i = 1; $i <= 5; $i++)
382 {
383 $mech->timeout(30);
384 eval
385 {
386 $mech->get($page);
387 };
388 my $code = $mech->status();
389 print "Try $i: ($code) $@" if $@;
390 return $mech if !$@;
391 sleep(30+5*$i);
392 last if $arg && $arg eq "no503" and $code == 503;
393 $mech = WWW::Mechanize->new("agent" => $agent);
394 }
395 return $mech;
396}
397
398sub getstats
399{
400 my %results;
401 require WWW::Mechanize;
402 require Data::Dumper;
403 require URI::Escape;
404 my $mech = WWW::Mechanize->new("agent" => $agent);
405
406 if(open DFILE,"<:utf8","launchpadtrans.data")
407 {
408 local $/;
409 $/ = undef;
410 my $val = <DFILE>;
411 eval $val;
412 close DFILE;
413 }
414
415 binmode STDOUT, ":utf8";
416
417 open FILE,">:utf8","launchpadtrans.txt" or die "Could not open output file.";
418
419 for my $lang (sort keys %lang)
420 {
421 doget($mech, "https://translations.launchpad.net/josm/trunk/+pots/josm/$lang/");
422 sleep(1);
423 my $cont = $mech->content();
424 while($cont =~ /<a href="https?:\/\/launchpad.net\/~(.*?)" class="sprite person(-inactive)?">(.*?)<\/a>/g)
425 {
426 my ($code, $inactive, $name) = ($1, $2, $3);
427 if(exists($results{$code}{$lang}{count}) && exists($results{$code}{$lang}{time}) && time()-$results{$code}{$lang}{time} < 5*24*60*60)
428 {
429 printf "%-5s - %-30s - Found - %s\n", $lang,$code,$name;
430 next;
431 }
432 my $urlcode = URI::Escape::uri_escape($code);
433 $mech = doget($mech, "https://translations.launchpad.net/josm/trunk/+pots/josm/$lang/+filter?person=$urlcode", "no503");
434 sleep(1);
435 my $cont = $mech->content() || "";
436 my ($count) = $cont =~ /of[\r\n\t ]+?(\d+)[\r\n\t ]+?result/;
437 if($count && $mech->status == 200)
438 {
439 my $t = time();
440 my $old = "";
441 if(exists($results{$code}{$lang}{count}))
442 {
443 if($results{$code}{$lang}{count} != $count)
444 {
445 $old .= sprintf " %d %s",abs($count-$results{$code}{$lang}{count}),$count-$results{$code}{$lang}{count} > 0 ? "more" : "less";
446 }
447 else
448 {
449 $old .= " unchanged";
450 }
451 }
452 if(exists($results{$code}{$lang}{time}))
453 {
454 $old .= sprintf " %.2f days later",($t-$results{$code}{$lang}{time})/86400.0;
455 }
456 $results{$code}{NAME} = $name;
457 $results{$code}{TOTAL} += $count-($results{$code}{$lang}{count}||0);
458 $results{$code}{$lang}{count} = $count;
459 $results{$code}{$lang}{time} = $t;
460 $results{$code}{$lang}{$t} = $count;
461 if(open DFILE,">:utf8","launchpadtrans.data")
462 {
463 print DFILE Data::Dumper->Dump([\%results],['*results']);
464 close DFILE;
465 }
466 if($old)
467 {
468 printf "%-5s - %-30s - %5d - %-70s%s\n", $lang,$code,$count,$name,$old;
469 }
470 else
471 {
472 printf "%-5s - %-30s - %5d - %s\n", $lang,$code,$count,$name;
473 }
474 }
475 else
476 {
477 printf "%-5s - %-30s - Skip - %s\n", $lang,$code,$name;
478 }
479 }
480 }
481 for my $code (sort {($results{$b}{TOTAL}||0) <=> ($results{$a}{TOTAL}||0)} keys %results)
482 {
483 next if !$results{$code}{TOTAL};
484 print FILE "$results{$code}{NAME}:$results{$code}{TOTAL}";
485 printf "%5d - %-50s",$results{$code}{TOTAL}, $results{$code}{NAME};
486 for my $lang (sort keys %{$results{$code}})
487 {
488 next if $lang eq "NAME" or $lang eq "TOTAL";
489 next if !exists($results{$code}{$lang}{time}) || time()-$results{$code}{$lang}{time} > 6*24*60*60;
490 print FILE ";$lang=$results{$code}{$lang}{count}";
491 printf " - %-5s=%5d",$lang, $results{$code}{$lang}{count};
492 }
493 print FILE "\n";
494 print "\n";
495 }
496}
Note: See TracBrowser for help on using the repository browser.