source: osm/applications/editors/josm/i18n/launchpad.pl@ 34151

Last change on this file since 34151 was 33169, checked in by stoecker, 8 years ago

fix if

  • Property svn:executable set to *
  • Property svn:keywords set to Revision
File size: 12.4 KB
RevLine 
[16400]1#!/usr/bin/perl -w
[27208]2# -CDSL would be better than explicit encoding settings
[16400]3
4use strict;
[27210]5use utf8;
[16400]6
[27020]7my ($user, $pwd);
8
9# Three ways to handle login data:
10 # Enter data directly in these two lines (Be careful witn svn checkin later!)
[33159]11 # create a file "launchpad.pl_credits" or ~/.josm_i18n_launchpad.pl_credits
12 # containing the two lines with proper values
[27020]13 # leave credits empty and enter them on runtime
14$user = '';
15$pwd = '';
16
17# list of supported languages
[26928]18my %lang = map {$_ => 1} (
[31139]19"ast", "bg", "be", "ca", "ca\@valencia", "cs", "da", "de", "el", "en_AU", "en_GB",
[30399]20"es", "et", "fi", "fr", "gl", "hu", "id",
[31724]21"it", "ja", "km", "lt", "nb", "nl", "pl", "pt", "pt_BR", "ru", "sk",
[31126]22"sv", "uk", "vi", "zh_CN", "zh_TW"
[26928]23);
24
[27210]25my $revision = '$Revision: 33169 $';
26$revision =~ s/^.*?(\d+).*$/$1/;
[27208]27my $agent = "JOSM_Launchpad/1.$revision";
28
[30179]29my $debugfile = 0;#1;
[25528]30my $cleanall = 0;#1;
[16400]31
[30179]32open TXTFILE,">","launchpad_debug.log" or die if $debugfile;
33
[16973]34if($#ARGV != 0)
[16400]35{
[27020]36 warn "No argument given (try Launchpad download URL, \"bzr\", \"bzronly\", \"upload\" or \"download\").";
37 system "ant";
[16400]38}
[26928]39elsif($ARGV[0] eq "bzr" || $ARGV[0] eq "bzronly")
40{
41 mkdir "build";
42 die "Could not change into new data dir." if !chdir "build";
43 system "bzr export -v josm_trans lp:~openstreetmap/josm/josm_trans";
44 chdir "..";
45 copypo("build/josm_trans/josm");
46 system "rm -rv build/josm_trans";
[27020]47 if($ARGV[0] ne "bzronly")
48 {
49 system "ant";
50 }
[26928]51}
[27020]52elsif($ARGV[0] eq "upload")
53{
54 potupload();
55}
[28769]56elsif($ARGV[0] eq "uploadall")
57{
58 makeupload();
59}
60elsif($ARGV[0] eq "approveall")
61{
[28806]62 approveall(dologin());
[28769]63}
[27020]64elsif($ARGV[0] eq "download")
65{
66 podownload();
67}
[27208]68elsif($ARGV[0] eq "stats")
69{
70 getstats();
71}
[16973]72else
73{
[26849]74 mkdir "build";
[26928]75 mkdir "build/josm_trans";
76 die "Could not change into new data dir." if !chdir "build/josm_trans";
[16973]77 system "wget $ARGV[0]";
78 system "tar -xf laun*";
[26849]79 chdir "../..";
[26928]80 copypo("build/josm_trans");
81 system "rm -rv build/josm_trans";
[27020]82 system "ant";
[26928]83}
84
[28769]85sub approveall
[26928]86{
[28769]87 my ($mech) = @_;
88 print "Trying to approve upload.\n";
89 $mech->get("https://translations.launchpad.net/josm/trunk/+imports?field.filter_status=NEEDS_REVIEW&field.filter_extension=all");
90 my @links;
91 foreach my $line (split("\n", $mech->content()))
[16973]92 {
[28769]93 push (@links, $1) if $line =~ /href="(\/\+imports\/\d+)"/;
94 }
95 if(!@links)
96 {
97 warn "No upload found in import list, upload possibly failed.";
98 }
99 else
100 {
101 foreach my $link (@links)
[27020]102 {
[28769]103 eval
104 {
105 print "Approve $link upload.\n";
106 $mech->get("https://translations.launchpad.net$link");
107 $mech->form_with_fields("field.potemplate");
108 $mech->select("field.potemplate", "josm");
109 $mech->click_button(name => "field.actions.approve");
110 };
111 print $@ if $@;
[27020]112 }
[26928]113 }
114}
115
[28769]116sub makeupload
117{
118 my $count = 11;
119 my $outdate = `date -u +"%Y-%m-%dT%H_%M_%S"`;
120 my $mech = dologin();
121 $outdate =~ s/[\r\n]+//;
122 mkdir "build/josm";
123 system "cp po/*.po po/josm.pot build/josm";
124 chdir "build";
125 print "Starting upload ($outdate).\n";
126 if(!$count)
127 {
128 system "tar -cjf launchpad_upload_josm_$outdate.tar.bz2 josm";
129 $mech->get("https://translations.launchpad.net/josm/trunk/+translations-upload");
130 $mech->submit_form(with_fields => {"file" => "launchpad_upload_josm_$outdate.tar.bz2"});
131 sleep(10);
132 }
133 else
134 {
135 my @files = sort glob("josm/*.po");
136 my $num = 1;
137 while($#files >= 0)
138 {
139 my @f = splice(@files, 0, $count);
140 my $file = "launchpad_upload_josm_${outdate}_$num.tar.bz2";
141 print "Create file $file.\n";
142 system "tar -cjf $file ".join(" ",@f);
143 print "Upload file $file.\n";
144 $mech->get("https://translations.launchpad.net/josm/trunk/+translations-upload");
145 $mech->submit_form(with_fields => {"file" => $file});
146 sleep(10);
147 ++$num;
148 }
149 }
150 system "rm -rv josm";
151 chdir "..";
152 approveall($mech);
153}
154
[26928]155sub copypo
156{
157 my ($path) = @_;
158 foreach my $name (split("\n", `find $path -name "*.po"`))
159 {
[31086]160 $name =~ /([a-zA-Z_@]+)\.po/;
[27020]161 if($lang{$1})
162 {
[31086]163 system("cp", "-v", $name, "po/$1.po");
[27020]164 }
165 elsif($cleanall)
166 {
167 local $/; undef $/;
168 open FILE,"<",$name or die;
169 my $x = <FILE>;
170 close FILE;
171 $x =~ s/\n\n.*$/\n/s;
172 open FILE,">","po/$1.po" or die;
173 print FILE $x;
174 close FILE;
175 }
[16973]176 }
177}
[27020]178
179sub dologin
180{
181 require WWW::Mechanize;
182
[27208]183 my $mech = WWW::Mechanize->new("agent" => $agent);
[27020]184
185 #$mech->add_handler("request_send" => sub {
186 # my($request, $ua, $h) = @_;
187 # printf "FORM: %s\n", $request->content();
188 # return undef;
189 #});
190 $mech->get("https://translations.launchpad.net/josm/trunk/+login");
191 #print $mech->status() ." - ". $mech->uri()."\n";
192 $mech->submit_form(form_number => 1);
193 getcredits();
194 #print $mech->status() ." - ". $mech->uri()."\n";
[31033]195 $mech->submit_form(form_id => "login-form", fields => {"email" => $user, "password" => $pwd});
[27020]196 #$mech->dump_headers();
197 #print $mech->status() ." - ". $mech->uri()."\n";
198 #print $mech->content();
199 my $form = $mech->form_name("decideform");
[27026]200 die "Could not login.\n" if !$form;
[27020]201 my %par = ("yes" => ""); # We need to add "yes" or it does not work
202 foreach my $p ($form->param)
203 {
204 $par{$p} = $form->value($p);
205 }
206 $mech->post($form->action, \%par);
207 #$mech->dump_headers();
208 #print $mech->content();
209 #print $mech->status() ." - ". $mech->uri()."\n";
210 #$mech->dump_forms();
211 return $mech;
212}
213
214sub potupload
215{
216 my $mech = dologin();
[28491]217 print "Starting upload.\n";
[28490]218 sleep(2);
[27020]219 $mech->get("https://translations.launchpad.net/josm/trunk/+translations-upload");
220 chdir("po");
221 $mech->submit_form(with_fields => {"file" => "josm.pot"});
[30179]222 print "Start approving of upload.\n";
223 for(1..10)
[28424]224 {
[30179]225 eval
[28424]226 {
[30179]227 sleep(10);
228 print "Trying to approve upload.\n";
229 $mech->get("https://translations.launchpad.net/josm/trunk/+imports?field.filter_status=NEEDS_REVIEW&field.filter_extension=pot");
230 my @links;
231 foreach my $line (split("\n", $mech->content()))
232 {
233 if($line =~ /href="(\/\+imports\/\d+)"/)
234 {
235 push (@links, $1);
236 print TXTFILE $line if $debugfile;
237 }
238 }
239 if(!@links)
240 {
241 print TXTFILE $mech->content() if $debugfile;
242 die "Upload not found in import list, upload possibly failed.";
243 }
244 elsif(@links > 1)
245 {
246 die "More than one upload found in import list, cannot approve.";
247 }
248 else
249 {
250 $mech->get("https://translations.launchpad.net$links[0]");
251 if($debugfile)
252 {
253 print TXTFILE "LINK: $links[0]\n";
254 $mech->dump_headers(\*TXTFILE);
255 print TXTFILE $mech->content();
256 print TXTFILE $mech->status() ." - ". $mech->uri()."\n";
257 $mech->dump_forms(\*TXTFILE);
258 }
259 $mech->submit_form(form_name => "launchpadform", button => "field.actions.approve");
260 if(!($mech->content() =~ /There are no entries that match this filtering/))
261 {
262 die "Approving possibly failed.";
263 }
264 }
265 };
266 print $@ if $@;
267 last if !$@;
[28424]268 }
269
[27020]270 chdir("..");
271}
272
273sub podownload
274{
275 my $mech = dologin();
276 $mech->get("https://translations.launchpad.net/josm/trunk/+export");
277 $mech->submit_form(with_fields => {"format" => "PO"});
278 if(!($mech->content() =~ /receive an email shortly/))
279 {
280 warn "Error requesting file\n";
281 }
282}
283
284sub getcredits
285{
286 if(!$user || !$pwd)
287 {
288 require Term::ReadKey;
289 local undef $/;
290 if(open FILE, "launchpad.pl_credits")
291 {
292 eval <FILE>;
293 close FILE;
294 }
[33169]295 elsif(open FILE, "$ENV{HOME}/.josm_i18n_launchpad.pl_credits")
[33159]296 {
297 eval <FILE>;
298 close FILE;
299 }
[27020]300
301 if(!$user)
302 {
303 Term::ReadKey::ReadMode(4); # Turn off controls keys
304 printf("Enter username: ");
305 for(;;)
306 {
307 my $c = getc();
308 print $c;
309 last if $c eq "\n";
310 $user .= $c;
311 }
312 Term::ReadKey::ReadMode(0);
313 }
314
315 if(!$pwd)
316 {
317 Term::ReadKey::ReadMode(4); # Turn off controls keys
318 printf("Enter password: ");
319 for(;;)
320 {
321 my $c = getc();
322 last if $c eq "\n";
323 print "*";
324 $pwd .= $c;
325 }
326 print "\n";
327 Term::ReadKey::ReadMode(0);
328 }
329 }
330}
[27208]331
332sub doget
333{
[27211]334 my ($mech, $page, $arg) = @_;
335 for(my $i = 1; $i <= 5; $i++)
[27208]336 {
[27210]337 $mech->timeout(30);
[27208]338 eval
339 {
340 $mech->get($page);
341 };
[27210]342 my $code = $mech->status();
343 print "Try $i: ($code) $@" if $@;
[27208]344 return $mech if !$@;
345 sleep(30+5*$i);
[27211]346 last if $arg && $arg eq "no503" and $code == 503;
[27208]347 $mech = WWW::Mechanize->new("agent" => $agent);
348 }
349 return $mech;
350}
351
352sub getstats
353{
354 my %results;
355 require WWW::Mechanize;
356 require Data::Dumper;
357 require URI::Escape;
358 my $mech = WWW::Mechanize->new("agent" => $agent);
359
360 if(open DFILE,"<:utf8","launchpadtrans.data")
361 {
362 local $/;
363 $/ = undef;
364 my $val = <DFILE>;
365 eval $val;
366 close DFILE;
367 }
368
369 binmode STDOUT, ":utf8";
370
371 open FILE,">:utf8","launchpadtrans.txt" or die "Could not open output file.";
372
373 for my $lang (sort keys %lang)
374 {
[27210]375 doget($mech, "https://translations.launchpad.net/josm/trunk/+pots/josm/$lang/");
[27208]376 sleep(1);
[27210]377 my $cont = $mech->content();
[27208]378 while($cont =~ /<a href="https?:\/\/launchpad.net\/~(.*?)" class="sprite person(-inactive)?">(.*?)<\/a>/g)
379 {
380 my ($code, $inactive, $name) = ($1, $2, $3);
[30229]381 if(exists($results{$code}{$lang}{count}) && exists($results{$code}{$lang}{time}) && time()-$results{$code}{$lang}{time} < 5*24*60*60)
[27208]382 {
383 printf "%-5s - %-30s - Found - %s\n", $lang,$code,$name;
384 next;
385 }
386 my $urlcode = URI::Escape::uri_escape($code);
[27211]387 $mech = doget($mech, "https://translations.launchpad.net/josm/trunk/+pots/josm/$lang/+filter?person=$urlcode", "no503");
[27208]388 sleep(1);
[27211]389 my $cont = $mech->content() || "";
390 my ($count) = $cont =~ /of[\r\n\t ]+?(\d+)[\r\n\t ]+?result/;
[27208]391 if($count && $mech->status == 200)
392 {
[30229]393 my $t = time();
394 my $old = "";
395 if(exists($results{$code}{$lang}{count}))
396 {
397 if($results{$code}{$lang}{count} != $count)
398 {
399 $old .= sprintf " %d %s",abs($count-$results{$code}{$lang}{count}),$count-$results{$code}{$lang}{count} > 0 ? "more" : "less";
400 }
401 else
402 {
403 $old .= " unchanged";
404 }
405 }
406 if(exists($results{$code}{$lang}{time}))
407 {
408 $old .= sprintf " %.2f days later",($t-$results{$code}{$lang}{time})/86400.0;
409 }
[27208]410 $results{$code}{NAME} = $name;
[30229]411 $results{$code}{TOTAL} += $count-($results{$code}{$lang}{count}||0);
412 $results{$code}{$lang}{count} = $count;
413 $results{$code}{$lang}{time} = $t;
414 $results{$code}{$lang}{$t} = $count;
[27208]415 if(open DFILE,">:utf8","launchpadtrans.data")
416 {
417 print DFILE Data::Dumper->Dump([\%results],['*results']);
418 close DFILE;
419 }
[30229]420 if($old)
421 {
422 printf "%-5s - %-30s - %5d - %-70s%s\n", $lang,$code,$count,$name,$old;
423 }
424 else
425 {
426 printf "%-5s - %-30s - %5d - %s\n", $lang,$code,$count,$name;
427 }
[27208]428 }
429 else
430 {
431 printf "%-5s - %-30s - Skip - %s\n", $lang,$code,$name;
432 }
433 }
434 }
[30233]435 for my $code (sort {($results{$b}{TOTAL}||0) <=> ($results{$a}{TOTAL}||0)} keys %results)
[27208]436 {
[30233]437 next if !$results{$code}{TOTAL};
[27208]438 print FILE "$results{$code}{NAME}:$results{$code}{TOTAL}";
439 printf "%5d - %-50s",$results{$code}{TOTAL}, $results{$code}{NAME};
440 for my $lang (sort keys %{$results{$code}})
441 {
442 next if $lang eq "NAME" or $lang eq "TOTAL";
[30229]443 next if !exists($results{$code}{$lang}{time}) || time()-$results{$code}{$lang}{time} > 6*24*60*60;
444 print FILE ";$lang=$results{$code}{$lang}{count}";
445 printf " - %-5s=%5d",$lang, $results{$code}{$lang}{count};
[27208]446 }
447 print FILE "\n";
448 print "\n";
449 }
450}
Note: See TracBrowser for help on using the repository browser.