[16400] | 1 | #!/usr/bin/perl -w
|
---|
[27208] | 2 | # -CDSL would be better than explicit encoding settings
|
---|
[16400] | 3 |
|
---|
| 4 | use strict;
|
---|
[27210] | 5 | use utf8;
|
---|
[16400] | 6 |
|
---|
[27020] | 7 | my ($user, $pwd);
|
---|
| 8 |
|
---|
| 9 | # Three ways to handle login data:
|
---|
| 10 | # Enter data directly in these two lines (Be careful witn svn checkin later!)
|
---|
[33159] | 11 | # create a file "launchpad.pl_credits" or ~/.josm_i18n_launchpad.pl_credits
|
---|
| 12 | # containing the two lines with proper values
|
---|
[27020] | 13 | # leave credits empty and enter them on runtime
|
---|
| 14 | $user = '';
|
---|
| 15 | $pwd = '';
|
---|
| 16 |
|
---|
| 17 | # list of supported languages
|
---|
[26928] | 18 | my %lang = map {$_ => 1} (
|
---|
[31139] | 19 | "ast", "bg", "be", "ca", "ca\@valencia", "cs", "da", "de", "el", "en_AU", "en_GB",
|
---|
[30399] | 20 | "es", "et", "fi", "fr", "gl", "hu", "id",
|
---|
[34205] | 21 | "it", "ja", "ko", "km", "lt", "nb", "nl", "pl", "pt", "pt_BR", "ru", "sk",
|
---|
[31126] | 22 | "sv", "uk", "vi", "zh_CN", "zh_TW"
|
---|
[26928] | 23 | );
|
---|
| 24 |
|
---|
[27210] | 25 | my $revision = '$Revision: 34205 $';
|
---|
| 26 | $revision =~ s/^.*?(\d+).*$/$1/;
|
---|
[27208] | 27 | my $agent = "JOSM_Launchpad/1.$revision";
|
---|
| 28 |
|
---|
[30179] | 29 | my $debugfile = 0;#1;
|
---|
[25528] | 30 | my $cleanall = 0;#1;
|
---|
[16400] | 31 |
|
---|
[30179] | 32 | open TXTFILE,">","launchpad_debug.log" or die if $debugfile;
|
---|
| 33 |
|
---|
[16973] | 34 | if($#ARGV != 0)
|
---|
[16400] | 35 | {
|
---|
[27020] | 36 | warn "No argument given (try Launchpad download URL, \"bzr\", \"bzronly\", \"upload\" or \"download\").";
|
---|
| 37 | system "ant";
|
---|
[16400] | 38 | }
|
---|
[26928] | 39 | elsif($ARGV[0] eq "bzr" || $ARGV[0] eq "bzronly")
|
---|
| 40 | {
|
---|
| 41 | mkdir "build";
|
---|
| 42 | die "Could not change into new data dir." if !chdir "build";
|
---|
| 43 | system "bzr export -v josm_trans lp:~openstreetmap/josm/josm_trans";
|
---|
| 44 | chdir "..";
|
---|
| 45 | copypo("build/josm_trans/josm");
|
---|
| 46 | system "rm -rv build/josm_trans";
|
---|
[27020] | 47 | if($ARGV[0] ne "bzronly")
|
---|
| 48 | {
|
---|
| 49 | system "ant";
|
---|
| 50 | }
|
---|
[26928] | 51 | }
|
---|
[27020] | 52 | elsif($ARGV[0] eq "upload")
|
---|
| 53 | {
|
---|
| 54 | potupload();
|
---|
| 55 | }
|
---|
[28769] | 56 | elsif($ARGV[0] eq "uploadall")
|
---|
| 57 | {
|
---|
| 58 | makeupload();
|
---|
| 59 | }
|
---|
| 60 | elsif($ARGV[0] eq "approveall")
|
---|
| 61 | {
|
---|
[28806] | 62 | approveall(dologin());
|
---|
[28769] | 63 | }
|
---|
[27020] | 64 | elsif($ARGV[0] eq "download")
|
---|
| 65 | {
|
---|
| 66 | podownload();
|
---|
| 67 | }
|
---|
[27208] | 68 | elsif($ARGV[0] eq "stats")
|
---|
| 69 | {
|
---|
| 70 | getstats();
|
---|
| 71 | }
|
---|
[16973] | 72 | else
|
---|
| 73 | {
|
---|
[26849] | 74 | mkdir "build";
|
---|
[26928] | 75 | mkdir "build/josm_trans";
|
---|
| 76 | die "Could not change into new data dir." if !chdir "build/josm_trans";
|
---|
[16973] | 77 | system "wget $ARGV[0]";
|
---|
| 78 | system "tar -xf laun*";
|
---|
[26849] | 79 | chdir "../..";
|
---|
[26928] | 80 | copypo("build/josm_trans");
|
---|
| 81 | system "rm -rv build/josm_trans";
|
---|
[27020] | 82 | system "ant";
|
---|
[26928] | 83 | }
|
---|
| 84 |
|
---|
[28769] | 85 | sub approveall
|
---|
[26928] | 86 | {
|
---|
[28769] | 87 | my ($mech) = @_;
|
---|
| 88 | print "Trying to approve upload.\n";
|
---|
| 89 | $mech->get("https://translations.launchpad.net/josm/trunk/+imports?field.filter_status=NEEDS_REVIEW&field.filter_extension=all");
|
---|
| 90 | my @links;
|
---|
| 91 | foreach my $line (split("\n", $mech->content()))
|
---|
[16973] | 92 | {
|
---|
[28769] | 93 | push (@links, $1) if $line =~ /href="(\/\+imports\/\d+)"/;
|
---|
| 94 | }
|
---|
| 95 | if(!@links)
|
---|
| 96 | {
|
---|
| 97 | warn "No upload found in import list, upload possibly failed.";
|
---|
| 98 | }
|
---|
| 99 | else
|
---|
| 100 | {
|
---|
| 101 | foreach my $link (@links)
|
---|
[27020] | 102 | {
|
---|
[28769] | 103 | eval
|
---|
| 104 | {
|
---|
| 105 | print "Approve $link upload.\n";
|
---|
| 106 | $mech->get("https://translations.launchpad.net$link");
|
---|
| 107 | $mech->form_with_fields("field.potemplate");
|
---|
| 108 | $mech->select("field.potemplate", "josm");
|
---|
| 109 | $mech->click_button(name => "field.actions.approve");
|
---|
| 110 | };
|
---|
| 111 | print $@ if $@;
|
---|
[27020] | 112 | }
|
---|
[26928] | 113 | }
|
---|
| 114 | }
|
---|
| 115 |
|
---|
[28769] | 116 | sub makeupload
|
---|
| 117 | {
|
---|
| 118 | my $count = 11;
|
---|
| 119 | my $outdate = `date -u +"%Y-%m-%dT%H_%M_%S"`;
|
---|
| 120 | my $mech = dologin();
|
---|
| 121 | $outdate =~ s/[\r\n]+//;
|
---|
| 122 | mkdir "build/josm";
|
---|
| 123 | system "cp po/*.po po/josm.pot build/josm";
|
---|
| 124 | chdir "build";
|
---|
| 125 | print "Starting upload ($outdate).\n";
|
---|
| 126 | if(!$count)
|
---|
| 127 | {
|
---|
| 128 | system "tar -cjf launchpad_upload_josm_$outdate.tar.bz2 josm";
|
---|
| 129 | $mech->get("https://translations.launchpad.net/josm/trunk/+translations-upload");
|
---|
| 130 | $mech->submit_form(with_fields => {"file" => "launchpad_upload_josm_$outdate.tar.bz2"});
|
---|
| 131 | sleep(10);
|
---|
| 132 | }
|
---|
| 133 | else
|
---|
| 134 | {
|
---|
| 135 | my @files = sort glob("josm/*.po");
|
---|
| 136 | my $num = 1;
|
---|
| 137 | while($#files >= 0)
|
---|
| 138 | {
|
---|
| 139 | my @f = splice(@files, 0, $count);
|
---|
| 140 | my $file = "launchpad_upload_josm_${outdate}_$num.tar.bz2";
|
---|
| 141 | print "Create file $file.\n";
|
---|
| 142 | system "tar -cjf $file ".join(" ",@f);
|
---|
| 143 | print "Upload file $file.\n";
|
---|
| 144 | $mech->get("https://translations.launchpad.net/josm/trunk/+translations-upload");
|
---|
| 145 | $mech->submit_form(with_fields => {"file" => $file});
|
---|
| 146 | sleep(10);
|
---|
| 147 | ++$num;
|
---|
| 148 | }
|
---|
| 149 | }
|
---|
| 150 | system "rm -rv josm";
|
---|
| 151 | chdir "..";
|
---|
| 152 | approveall($mech);
|
---|
| 153 | }
|
---|
| 154 |
|
---|
[26928] | 155 | sub copypo
|
---|
| 156 | {
|
---|
| 157 | my ($path) = @_;
|
---|
| 158 | foreach my $name (split("\n", `find $path -name "*.po"`))
|
---|
| 159 | {
|
---|
[31086] | 160 | $name =~ /([a-zA-Z_@]+)\.po/;
|
---|
[27020] | 161 | if($lang{$1})
|
---|
| 162 | {
|
---|
[31086] | 163 | system("cp", "-v", $name, "po/$1.po");
|
---|
[27020] | 164 | }
|
---|
| 165 | elsif($cleanall)
|
---|
| 166 | {
|
---|
| 167 | local $/; undef $/;
|
---|
| 168 | open FILE,"<",$name or die;
|
---|
| 169 | my $x = <FILE>;
|
---|
| 170 | close FILE;
|
---|
| 171 | $x =~ s/\n\n.*$/\n/s;
|
---|
| 172 | open FILE,">","po/$1.po" or die;
|
---|
| 173 | print FILE $x;
|
---|
| 174 | close FILE;
|
---|
| 175 | }
|
---|
[16973] | 176 | }
|
---|
| 177 | }
|
---|
[27020] | 178 |
|
---|
| 179 | sub dologin
|
---|
| 180 | {
|
---|
| 181 | require WWW::Mechanize;
|
---|
| 182 |
|
---|
[27208] | 183 | my $mech = WWW::Mechanize->new("agent" => $agent);
|
---|
[27020] | 184 |
|
---|
| 185 | #$mech->add_handler("request_send" => sub {
|
---|
| 186 | # my($request, $ua, $h) = @_;
|
---|
| 187 | # printf "FORM: %s\n", $request->content();
|
---|
| 188 | # return undef;
|
---|
| 189 | #});
|
---|
| 190 | $mech->get("https://translations.launchpad.net/josm/trunk/+login");
|
---|
| 191 | #print $mech->status() ." - ". $mech->uri()."\n";
|
---|
| 192 | $mech->submit_form(form_number => 1);
|
---|
| 193 | getcredits();
|
---|
| 194 | #print $mech->status() ." - ". $mech->uri()."\n";
|
---|
[31033] | 195 | $mech->submit_form(form_id => "login-form", fields => {"email" => $user, "password" => $pwd});
|
---|
[27020] | 196 | #$mech->dump_headers();
|
---|
| 197 | #print $mech->status() ." - ". $mech->uri()."\n";
|
---|
| 198 | #print $mech->content();
|
---|
| 199 | my $form = $mech->form_name("decideform");
|
---|
[27026] | 200 | die "Could not login.\n" if !$form;
|
---|
[27020] | 201 | my %par = ("yes" => ""); # We need to add "yes" or it does not work
|
---|
| 202 | foreach my $p ($form->param)
|
---|
| 203 | {
|
---|
| 204 | $par{$p} = $form->value($p);
|
---|
| 205 | }
|
---|
| 206 | $mech->post($form->action, \%par);
|
---|
| 207 | #$mech->dump_headers();
|
---|
| 208 | #print $mech->content();
|
---|
| 209 | #print $mech->status() ." - ". $mech->uri()."\n";
|
---|
| 210 | #$mech->dump_forms();
|
---|
| 211 | return $mech;
|
---|
| 212 | }
|
---|
| 213 |
|
---|
| 214 | sub potupload
|
---|
| 215 | {
|
---|
| 216 | my $mech = dologin();
|
---|
[28491] | 217 | print "Starting upload.\n";
|
---|
[28490] | 218 | sleep(2);
|
---|
[27020] | 219 | $mech->get("https://translations.launchpad.net/josm/trunk/+translations-upload");
|
---|
| 220 | chdir("po");
|
---|
| 221 | $mech->submit_form(with_fields => {"file" => "josm.pot"});
|
---|
[30179] | 222 | print "Start approving of upload.\n";
|
---|
| 223 | for(1..10)
|
---|
[28424] | 224 | {
|
---|
[30179] | 225 | eval
|
---|
[28424] | 226 | {
|
---|
[30179] | 227 | sleep(10);
|
---|
| 228 | print "Trying to approve upload.\n";
|
---|
| 229 | $mech->get("https://translations.launchpad.net/josm/trunk/+imports?field.filter_status=NEEDS_REVIEW&field.filter_extension=pot");
|
---|
| 230 | my @links;
|
---|
| 231 | foreach my $line (split("\n", $mech->content()))
|
---|
| 232 | {
|
---|
| 233 | if($line =~ /href="(\/\+imports\/\d+)"/)
|
---|
| 234 | {
|
---|
| 235 | push (@links, $1);
|
---|
| 236 | print TXTFILE $line if $debugfile;
|
---|
| 237 | }
|
---|
| 238 | }
|
---|
| 239 | if(!@links)
|
---|
| 240 | {
|
---|
| 241 | print TXTFILE $mech->content() if $debugfile;
|
---|
| 242 | die "Upload not found in import list, upload possibly failed.";
|
---|
| 243 | }
|
---|
| 244 | elsif(@links > 1)
|
---|
| 245 | {
|
---|
| 246 | die "More than one upload found in import list, cannot approve.";
|
---|
| 247 | }
|
---|
| 248 | else
|
---|
| 249 | {
|
---|
| 250 | $mech->get("https://translations.launchpad.net$links[0]");
|
---|
| 251 | if($debugfile)
|
---|
| 252 | {
|
---|
| 253 | print TXTFILE "LINK: $links[0]\n";
|
---|
| 254 | $mech->dump_headers(\*TXTFILE);
|
---|
| 255 | print TXTFILE $mech->content();
|
---|
| 256 | print TXTFILE $mech->status() ." - ". $mech->uri()."\n";
|
---|
| 257 | $mech->dump_forms(\*TXTFILE);
|
---|
| 258 | }
|
---|
| 259 | $mech->submit_form(form_name => "launchpadform", button => "field.actions.approve");
|
---|
| 260 | if(!($mech->content() =~ /There are no entries that match this filtering/))
|
---|
| 261 | {
|
---|
| 262 | die "Approving possibly failed.";
|
---|
| 263 | }
|
---|
| 264 | }
|
---|
| 265 | };
|
---|
| 266 | print $@ if $@;
|
---|
| 267 | last if !$@;
|
---|
[28424] | 268 | }
|
---|
| 269 |
|
---|
[27020] | 270 | chdir("..");
|
---|
| 271 | }
|
---|
| 272 |
|
---|
| 273 | sub podownload
|
---|
| 274 | {
|
---|
| 275 | my $mech = dologin();
|
---|
| 276 | $mech->get("https://translations.launchpad.net/josm/trunk/+export");
|
---|
| 277 | $mech->submit_form(with_fields => {"format" => "PO"});
|
---|
| 278 | if(!($mech->content() =~ /receive an email shortly/))
|
---|
| 279 | {
|
---|
| 280 | warn "Error requesting file\n";
|
---|
| 281 | }
|
---|
| 282 | }
|
---|
| 283 |
|
---|
| 284 | sub getcredits
|
---|
| 285 | {
|
---|
| 286 | if(!$user || !$pwd)
|
---|
| 287 | {
|
---|
| 288 | require Term::ReadKey;
|
---|
| 289 | local undef $/;
|
---|
| 290 | if(open FILE, "launchpad.pl_credits")
|
---|
| 291 | {
|
---|
| 292 | eval <FILE>;
|
---|
| 293 | close FILE;
|
---|
| 294 | }
|
---|
[33169] | 295 | elsif(open FILE, "$ENV{HOME}/.josm_i18n_launchpad.pl_credits")
|
---|
[33159] | 296 | {
|
---|
| 297 | eval <FILE>;
|
---|
| 298 | close FILE;
|
---|
| 299 | }
|
---|
[27020] | 300 |
|
---|
| 301 | if(!$user)
|
---|
| 302 | {
|
---|
| 303 | Term::ReadKey::ReadMode(4); # Turn off controls keys
|
---|
| 304 | printf("Enter username: ");
|
---|
| 305 | for(;;)
|
---|
| 306 | {
|
---|
| 307 | my $c = getc();
|
---|
| 308 | print $c;
|
---|
| 309 | last if $c eq "\n";
|
---|
| 310 | $user .= $c;
|
---|
| 311 | }
|
---|
| 312 | Term::ReadKey::ReadMode(0);
|
---|
| 313 | }
|
---|
| 314 |
|
---|
| 315 | if(!$pwd)
|
---|
| 316 | {
|
---|
| 317 | Term::ReadKey::ReadMode(4); # Turn off controls keys
|
---|
| 318 | printf("Enter password: ");
|
---|
| 319 | for(;;)
|
---|
| 320 | {
|
---|
| 321 | my $c = getc();
|
---|
| 322 | last if $c eq "\n";
|
---|
| 323 | print "*";
|
---|
| 324 | $pwd .= $c;
|
---|
| 325 | }
|
---|
| 326 | print "\n";
|
---|
| 327 | Term::ReadKey::ReadMode(0);
|
---|
| 328 | }
|
---|
| 329 | }
|
---|
| 330 | }
|
---|
[27208] | 331 |
|
---|
| 332 | sub doget
|
---|
| 333 | {
|
---|
[27211] | 334 | my ($mech, $page, $arg) = @_;
|
---|
| 335 | for(my $i = 1; $i <= 5; $i++)
|
---|
[27208] | 336 | {
|
---|
[27210] | 337 | $mech->timeout(30);
|
---|
[27208] | 338 | eval
|
---|
| 339 | {
|
---|
| 340 | $mech->get($page);
|
---|
| 341 | };
|
---|
[27210] | 342 | my $code = $mech->status();
|
---|
| 343 | print "Try $i: ($code) $@" if $@;
|
---|
[27208] | 344 | return $mech if !$@;
|
---|
| 345 | sleep(30+5*$i);
|
---|
[27211] | 346 | last if $arg && $arg eq "no503" and $code == 503;
|
---|
[27208] | 347 | $mech = WWW::Mechanize->new("agent" => $agent);
|
---|
| 348 | }
|
---|
| 349 | return $mech;
|
---|
| 350 | }
|
---|
| 351 |
|
---|
| 352 | sub getstats
|
---|
| 353 | {
|
---|
| 354 | my %results;
|
---|
| 355 | require WWW::Mechanize;
|
---|
| 356 | require Data::Dumper;
|
---|
| 357 | require URI::Escape;
|
---|
| 358 | my $mech = WWW::Mechanize->new("agent" => $agent);
|
---|
| 359 |
|
---|
| 360 | if(open DFILE,"<:utf8","launchpadtrans.data")
|
---|
| 361 | {
|
---|
| 362 | local $/;
|
---|
| 363 | $/ = undef;
|
---|
| 364 | my $val = <DFILE>;
|
---|
| 365 | eval $val;
|
---|
| 366 | close DFILE;
|
---|
| 367 | }
|
---|
| 368 |
|
---|
| 369 | binmode STDOUT, ":utf8";
|
---|
| 370 |
|
---|
| 371 | open FILE,">:utf8","launchpadtrans.txt" or die "Could not open output file.";
|
---|
| 372 |
|
---|
| 373 | for my $lang (sort keys %lang)
|
---|
| 374 | {
|
---|
[27210] | 375 | doget($mech, "https://translations.launchpad.net/josm/trunk/+pots/josm/$lang/");
|
---|
[27208] | 376 | sleep(1);
|
---|
[27210] | 377 | my $cont = $mech->content();
|
---|
[27208] | 378 | while($cont =~ /<a href="https?:\/\/launchpad.net\/~(.*?)" class="sprite person(-inactive)?">(.*?)<\/a>/g)
|
---|
| 379 | {
|
---|
| 380 | my ($code, $inactive, $name) = ($1, $2, $3);
|
---|
[30229] | 381 | if(exists($results{$code}{$lang}{count}) && exists($results{$code}{$lang}{time}) && time()-$results{$code}{$lang}{time} < 5*24*60*60)
|
---|
[27208] | 382 | {
|
---|
| 383 | printf "%-5s - %-30s - Found - %s\n", $lang,$code,$name;
|
---|
| 384 | next;
|
---|
| 385 | }
|
---|
| 386 | my $urlcode = URI::Escape::uri_escape($code);
|
---|
[27211] | 387 | $mech = doget($mech, "https://translations.launchpad.net/josm/trunk/+pots/josm/$lang/+filter?person=$urlcode", "no503");
|
---|
[27208] | 388 | sleep(1);
|
---|
[27211] | 389 | my $cont = $mech->content() || "";
|
---|
| 390 | my ($count) = $cont =~ /of[\r\n\t ]+?(\d+)[\r\n\t ]+?result/;
|
---|
[27208] | 391 | if($count && $mech->status == 200)
|
---|
| 392 | {
|
---|
[30229] | 393 | my $t = time();
|
---|
| 394 | my $old = "";
|
---|
| 395 | if(exists($results{$code}{$lang}{count}))
|
---|
| 396 | {
|
---|
| 397 | if($results{$code}{$lang}{count} != $count)
|
---|
| 398 | {
|
---|
| 399 | $old .= sprintf " %d %s",abs($count-$results{$code}{$lang}{count}),$count-$results{$code}{$lang}{count} > 0 ? "more" : "less";
|
---|
| 400 | }
|
---|
| 401 | else
|
---|
| 402 | {
|
---|
| 403 | $old .= " unchanged";
|
---|
| 404 | }
|
---|
| 405 | }
|
---|
| 406 | if(exists($results{$code}{$lang}{time}))
|
---|
| 407 | {
|
---|
| 408 | $old .= sprintf " %.2f days later",($t-$results{$code}{$lang}{time})/86400.0;
|
---|
| 409 | }
|
---|
[27208] | 410 | $results{$code}{NAME} = $name;
|
---|
[30229] | 411 | $results{$code}{TOTAL} += $count-($results{$code}{$lang}{count}||0);
|
---|
| 412 | $results{$code}{$lang}{count} = $count;
|
---|
| 413 | $results{$code}{$lang}{time} = $t;
|
---|
| 414 | $results{$code}{$lang}{$t} = $count;
|
---|
[27208] | 415 | if(open DFILE,">:utf8","launchpadtrans.data")
|
---|
| 416 | {
|
---|
| 417 | print DFILE Data::Dumper->Dump([\%results],['*results']);
|
---|
| 418 | close DFILE;
|
---|
| 419 | }
|
---|
[30229] | 420 | if($old)
|
---|
| 421 | {
|
---|
| 422 | printf "%-5s - %-30s - %5d - %-70s%s\n", $lang,$code,$count,$name,$old;
|
---|
| 423 | }
|
---|
| 424 | else
|
---|
| 425 | {
|
---|
| 426 | printf "%-5s - %-30s - %5d - %s\n", $lang,$code,$count,$name;
|
---|
| 427 | }
|
---|
[27208] | 428 | }
|
---|
| 429 | else
|
---|
| 430 | {
|
---|
| 431 | printf "%-5s - %-30s - Skip - %s\n", $lang,$code,$name;
|
---|
| 432 | }
|
---|
| 433 | }
|
---|
| 434 | }
|
---|
[30233] | 435 | for my $code (sort {($results{$b}{TOTAL}||0) <=> ($results{$a}{TOTAL}||0)} keys %results)
|
---|
[27208] | 436 | {
|
---|
[30233] | 437 | next if !$results{$code}{TOTAL};
|
---|
[27208] | 438 | print FILE "$results{$code}{NAME}:$results{$code}{TOTAL}";
|
---|
| 439 | printf "%5d - %-50s",$results{$code}{TOTAL}, $results{$code}{NAME};
|
---|
| 440 | for my $lang (sort keys %{$results{$code}})
|
---|
| 441 | {
|
---|
| 442 | next if $lang eq "NAME" or $lang eq "TOTAL";
|
---|
[30229] | 443 | next if !exists($results{$code}{$lang}{time}) || time()-$results{$code}{$lang}{time} > 6*24*60*60;
|
---|
| 444 | print FILE ";$lang=$results{$code}{$lang}{count}";
|
---|
| 445 | printf " - %-5s=%5d",$lang, $results{$code}{$lang}{count};
|
---|
[27208] | 446 | }
|
---|
| 447 | print FILE "\n";
|
---|
| 448 | print "\n";
|
---|
| 449 | }
|
---|
| 450 | }
|
---|