[16400] | 1 | #!/usr/bin/perl -w
|
---|
[27208] | 2 | # -CDSL would be better than explicit encoding settings
|
---|
[16400] | 3 |
|
---|
| 4 | use strict;
|
---|
[27210] | 5 | use utf8;
|
---|
[16400] | 6 |
|
---|
[27020] | 7 | my ($user, $pwd);
|
---|
| 8 |
|
---|
| 9 | # Three ways to handle login data:
|
---|
| 10 | # Enter data directly in these two lines (Be careful witn svn checkin later!)
|
---|
[28253] | 11 | # create a file "launchpad.pl_credits" containing the two lines with proper values
|
---|
[27020] | 12 | # leave credits empty and enter them on runtime
|
---|
| 13 | $user = '';
|
---|
| 14 | $pwd = '';
|
---|
| 15 |
|
---|
| 16 | # list of supported languages
|
---|
[26928] | 17 | my %lang = map {$_ => 1} (
|
---|
[28409] | 18 | "bg", "ca", "cs", "da", "de", "el", "en_AU", "en_GB",
|
---|
[28251] | 19 | "es", "et", "eu", "fi", "fr", "gl", "hu", "id",
|
---|
[28408] | 20 | "it", "ja", "nb", "nl", "pl", "pt", "pt_BR", "ru", "sk",
|
---|
[26928] | 21 | "sv", "tr", "uk", "zh_CN", "zh_TW"
|
---|
| 22 | );
|
---|
| 23 |
|
---|
[27210] | 24 | my $revision = '$Revision: 28806 $';
|
---|
| 25 | $revision =~ s/^.*?(\d+).*$/$1/;
|
---|
[27208] | 26 | my $agent = "JOSM_Launchpad/1.$revision";
|
---|
| 27 |
|
---|
[25528] | 28 | my $cleanall = 0;#1;
|
---|
[16400] | 29 |
|
---|
[16973] | 30 | if($#ARGV != 0)
|
---|
[16400] | 31 | {
|
---|
[27020] | 32 | warn "No argument given (try Launchpad download URL, \"bzr\", \"bzronly\", \"upload\" or \"download\").";
|
---|
| 33 | system "ant";
|
---|
[16400] | 34 | }
|
---|
[26928] | 35 | elsif($ARGV[0] eq "bzr" || $ARGV[0] eq "bzronly")
|
---|
| 36 | {
|
---|
| 37 | mkdir "build";
|
---|
| 38 | die "Could not change into new data dir." if !chdir "build";
|
---|
| 39 | system "bzr export -v josm_trans lp:~openstreetmap/josm/josm_trans";
|
---|
| 40 | chdir "..";
|
---|
| 41 | copypo("build/josm_trans/josm");
|
---|
| 42 | system "rm -rv build/josm_trans";
|
---|
[27020] | 43 | if($ARGV[0] ne "bzronly")
|
---|
| 44 | {
|
---|
| 45 | system "ant";
|
---|
| 46 | }
|
---|
[26928] | 47 | }
|
---|
[27020] | 48 | elsif($ARGV[0] eq "upload")
|
---|
| 49 | {
|
---|
| 50 | potupload();
|
---|
| 51 | }
|
---|
[28769] | 52 | elsif($ARGV[0] eq "uploadall")
|
---|
| 53 | {
|
---|
| 54 | makeupload();
|
---|
| 55 | }
|
---|
| 56 | elsif($ARGV[0] eq "approveall")
|
---|
| 57 | {
|
---|
[28806] | 58 | approveall(dologin());
|
---|
[28769] | 59 | }
|
---|
[27020] | 60 | elsif($ARGV[0] eq "download")
|
---|
| 61 | {
|
---|
| 62 | podownload();
|
---|
| 63 | }
|
---|
[27208] | 64 | elsif($ARGV[0] eq "stats")
|
---|
| 65 | {
|
---|
| 66 | getstats();
|
---|
| 67 | }
|
---|
[16973] | 68 | else
|
---|
| 69 | {
|
---|
[26849] | 70 | mkdir "build";
|
---|
[26928] | 71 | mkdir "build/josm_trans";
|
---|
| 72 | die "Could not change into new data dir." if !chdir "build/josm_trans";
|
---|
[16973] | 73 | system "wget $ARGV[0]";
|
---|
| 74 | system "tar -xf laun*";
|
---|
[26849] | 75 | chdir "../..";
|
---|
[26928] | 76 | copypo("build/josm_trans");
|
---|
| 77 | system "rm -rv build/josm_trans";
|
---|
[27020] | 78 | system "ant";
|
---|
[26928] | 79 | }
|
---|
| 80 |
|
---|
[28769] | 81 | sub approveall
|
---|
[26928] | 82 | {
|
---|
[28769] | 83 | my ($mech) = @_;
|
---|
| 84 | print "Trying to approve upload.\n";
|
---|
| 85 | $mech->get("https://translations.launchpad.net/josm/trunk/+imports?field.filter_status=NEEDS_REVIEW&field.filter_extension=all");
|
---|
| 86 | my @links;
|
---|
| 87 | foreach my $line (split("\n", $mech->content()))
|
---|
[16973] | 88 | {
|
---|
[28769] | 89 | push (@links, $1) if $line =~ /href="(\/\+imports\/\d+)"/;
|
---|
| 90 | }
|
---|
| 91 | if(!@links)
|
---|
| 92 | {
|
---|
| 93 | warn "No upload found in import list, upload possibly failed.";
|
---|
| 94 | }
|
---|
| 95 | else
|
---|
| 96 | {
|
---|
| 97 | foreach my $link (@links)
|
---|
[27020] | 98 | {
|
---|
[28769] | 99 | eval
|
---|
| 100 | {
|
---|
| 101 | print "Approve $link upload.\n";
|
---|
| 102 | $mech->get("https://translations.launchpad.net$link");
|
---|
| 103 | $mech->form_with_fields("field.potemplate");
|
---|
| 104 | $mech->select("field.potemplate", "josm");
|
---|
| 105 | $mech->click_button(name => "field.actions.approve");
|
---|
| 106 | };
|
---|
| 107 | print $@ if $@;
|
---|
[27020] | 108 | }
|
---|
[26928] | 109 | }
|
---|
| 110 | }
|
---|
| 111 |
|
---|
[28769] | 112 | sub makeupload
|
---|
| 113 | {
|
---|
| 114 | my $count = 11;
|
---|
| 115 | my $outdate = `date -u +"%Y-%m-%dT%H_%M_%S"`;
|
---|
| 116 | my $mech = dologin();
|
---|
| 117 | $outdate =~ s/[\r\n]+//;
|
---|
| 118 | mkdir "build/josm";
|
---|
| 119 | system "cp po/*.po po/josm.pot build/josm";
|
---|
| 120 | chdir "build";
|
---|
| 121 | print "Starting upload ($outdate).\n";
|
---|
| 122 | if(!$count)
|
---|
| 123 | {
|
---|
| 124 | system "tar -cjf launchpad_upload_josm_$outdate.tar.bz2 josm";
|
---|
| 125 | $mech->get("https://translations.launchpad.net/josm/trunk/+translations-upload");
|
---|
| 126 | $mech->submit_form(with_fields => {"file" => "launchpad_upload_josm_$outdate.tar.bz2"});
|
---|
| 127 | sleep(10);
|
---|
| 128 | }
|
---|
| 129 | else
|
---|
| 130 | {
|
---|
| 131 | my @files = sort glob("josm/*.po");
|
---|
| 132 | my $num = 1;
|
---|
| 133 | while($#files >= 0)
|
---|
| 134 | {
|
---|
| 135 | my @f = splice(@files, 0, $count);
|
---|
| 136 | my $file = "launchpad_upload_josm_${outdate}_$num.tar.bz2";
|
---|
| 137 | print "Create file $file.\n";
|
---|
| 138 | system "tar -cjf $file ".join(" ",@f);
|
---|
| 139 | print "Upload file $file.\n";
|
---|
| 140 | $mech->get("https://translations.launchpad.net/josm/trunk/+translations-upload");
|
---|
| 141 | $mech->submit_form(with_fields => {"file" => $file});
|
---|
| 142 | sleep(10);
|
---|
| 143 | ++$num;
|
---|
| 144 | }
|
---|
| 145 | }
|
---|
| 146 | system "rm -rv josm";
|
---|
| 147 | chdir "..";
|
---|
| 148 | approveall($mech);
|
---|
| 149 | }
|
---|
| 150 |
|
---|
[26928] | 151 | sub copypo
|
---|
| 152 | {
|
---|
| 153 | my ($path) = @_;
|
---|
| 154 | foreach my $name (split("\n", `find $path -name "*.po"`))
|
---|
| 155 | {
|
---|
[27020] | 156 | $name =~ /([a-zA-Z_]+)\.po/;
|
---|
| 157 | if($lang{$1})
|
---|
| 158 | {
|
---|
| 159 | system "cp -v $name po/$1.po";
|
---|
| 160 | }
|
---|
| 161 | elsif($cleanall)
|
---|
| 162 | {
|
---|
| 163 | local $/; undef $/;
|
---|
| 164 | open FILE,"<",$name or die;
|
---|
| 165 | my $x = <FILE>;
|
---|
| 166 | close FILE;
|
---|
| 167 | $x =~ s/\n\n.*$/\n/s;
|
---|
| 168 | open FILE,">","po/$1.po" or die;
|
---|
| 169 | print FILE $x;
|
---|
| 170 | close FILE;
|
---|
| 171 | }
|
---|
[16973] | 172 | }
|
---|
| 173 | }
|
---|
[27020] | 174 |
|
---|
| 175 | sub dologin
|
---|
| 176 | {
|
---|
| 177 | require WWW::Mechanize;
|
---|
| 178 |
|
---|
[27208] | 179 | my $mech = WWW::Mechanize->new("agent" => $agent);
|
---|
[27020] | 180 |
|
---|
| 181 | #$mech->add_handler("request_send" => sub {
|
---|
| 182 | # my($request, $ua, $h) = @_;
|
---|
| 183 | # printf "FORM: %s\n", $request->content();
|
---|
| 184 | # return undef;
|
---|
| 185 | #});
|
---|
| 186 | $mech->get("https://translations.launchpad.net/josm/trunk/+login");
|
---|
| 187 | #print $mech->status() ." - ". $mech->uri()."\n";
|
---|
| 188 | $mech->submit_form(form_number => 1);
|
---|
| 189 | getcredits();
|
---|
| 190 | #print $mech->status() ." - ". $mech->uri()."\n";
|
---|
| 191 | $mech->submit_form(with_fields => {"email" => $user, "password" => $pwd});
|
---|
| 192 | #$mech->dump_headers();
|
---|
| 193 | #print $mech->status() ." - ". $mech->uri()."\n";
|
---|
| 194 | #print $mech->content();
|
---|
| 195 | my $form = $mech->form_name("decideform");
|
---|
[27026] | 196 | die "Could not login.\n" if !$form;
|
---|
[27020] | 197 | my %par = ("yes" => ""); # We need to add "yes" or it does not work
|
---|
| 198 | foreach my $p ($form->param)
|
---|
| 199 | {
|
---|
| 200 | $par{$p} = $form->value($p);
|
---|
| 201 | }
|
---|
| 202 | $mech->post($form->action, \%par);
|
---|
| 203 | #$mech->dump_headers();
|
---|
| 204 | #print $mech->content();
|
---|
| 205 | #print $mech->status() ." - ". $mech->uri()."\n";
|
---|
| 206 | #$mech->dump_forms();
|
---|
| 207 | return $mech;
|
---|
| 208 | }
|
---|
| 209 |
|
---|
| 210 | sub potupload
|
---|
| 211 | {
|
---|
| 212 | my $mech = dologin();
|
---|
[28491] | 213 | print "Starting upload.\n";
|
---|
[28490] | 214 | sleep(2);
|
---|
[27020] | 215 | $mech->get("https://translations.launchpad.net/josm/trunk/+translations-upload");
|
---|
| 216 | chdir("po");
|
---|
| 217 | $mech->submit_form(with_fields => {"file" => "josm.pot"});
|
---|
[28490] | 218 | sleep(10);
|
---|
| 219 | print "Trying to approve upload.\n";
|
---|
[28424] | 220 | $mech->get("https://translations.launchpad.net/josm/trunk/+imports?field.filter_status=NEEDS_REVIEW&field.filter_extension=pot");
|
---|
| 221 | my @links;
|
---|
| 222 | foreach my $line (split("\n", $mech->content()))
|
---|
| 223 | {
|
---|
| 224 | push (@links, $1) if $line =~ /href="(\/\+imports\/\d+)"/;
|
---|
| 225 | }
|
---|
| 226 | if(!@links)
|
---|
| 227 | {
|
---|
| 228 | warn "Upload not found in import list, upload possibly failed.";
|
---|
| 229 | }
|
---|
| 230 | elsif(@links > 1)
|
---|
| 231 | {
|
---|
| 232 | warn "More than one upload found in import list, cannot approve.";
|
---|
| 233 | }
|
---|
| 234 | else
|
---|
| 235 | {
|
---|
| 236 | $mech->get("https://translations.launchpad.net$links[0]");
|
---|
| 237 | $mech->submit_form(form_name => "launchpadform", button => "field.actions.approve");
|
---|
| 238 | if(!($mech->content() =~ /There are no entries that match this filtering/))
|
---|
| 239 | {
|
---|
| 240 | warn "Approving possibly failed.";
|
---|
| 241 | }
|
---|
| 242 | }
|
---|
| 243 |
|
---|
[27020] | 244 | chdir("..");
|
---|
| 245 | }
|
---|
| 246 |
|
---|
| 247 | sub podownload
|
---|
| 248 | {
|
---|
| 249 | my $mech = dologin();
|
---|
| 250 | $mech->get("https://translations.launchpad.net/josm/trunk/+export");
|
---|
| 251 | $mech->submit_form(with_fields => {"format" => "PO"});
|
---|
| 252 | if(!($mech->content() =~ /receive an email shortly/))
|
---|
| 253 | {
|
---|
| 254 | warn "Error requesting file\n";
|
---|
| 255 | }
|
---|
| 256 | }
|
---|
| 257 |
|
---|
| 258 | sub getcredits
|
---|
| 259 | {
|
---|
| 260 | if(!$user || !$pwd)
|
---|
| 261 | {
|
---|
| 262 | require Term::ReadKey;
|
---|
| 263 | local undef $/;
|
---|
| 264 | if(open FILE, "launchpad.pl_credits")
|
---|
| 265 | {
|
---|
| 266 | eval <FILE>;
|
---|
| 267 | close FILE;
|
---|
| 268 | }
|
---|
| 269 |
|
---|
| 270 | if(!$user)
|
---|
| 271 | {
|
---|
| 272 | Term::ReadKey::ReadMode(4); # Turn off controls keys
|
---|
| 273 | printf("Enter username: ");
|
---|
| 274 | for(;;)
|
---|
| 275 | {
|
---|
| 276 | my $c = getc();
|
---|
| 277 | print $c;
|
---|
| 278 | last if $c eq "\n";
|
---|
| 279 | $user .= $c;
|
---|
| 280 | }
|
---|
| 281 | Term::ReadKey::ReadMode(0);
|
---|
| 282 | }
|
---|
| 283 |
|
---|
| 284 | if(!$pwd)
|
---|
| 285 | {
|
---|
| 286 | Term::ReadKey::ReadMode(4); # Turn off controls keys
|
---|
| 287 | printf("Enter password: ");
|
---|
| 288 | for(;;)
|
---|
| 289 | {
|
---|
| 290 | my $c = getc();
|
---|
| 291 | last if $c eq "\n";
|
---|
| 292 | print "*";
|
---|
| 293 | $pwd .= $c;
|
---|
| 294 | }
|
---|
| 295 | print "\n";
|
---|
| 296 | Term::ReadKey::ReadMode(0);
|
---|
| 297 | }
|
---|
| 298 | }
|
---|
| 299 | }
|
---|
[27208] | 300 |
|
---|
| 301 | sub doget
|
---|
| 302 | {
|
---|
[27211] | 303 | my ($mech, $page, $arg) = @_;
|
---|
| 304 | for(my $i = 1; $i <= 5; $i++)
|
---|
[27208] | 305 | {
|
---|
[27210] | 306 | $mech->timeout(30);
|
---|
[27208] | 307 | eval
|
---|
| 308 | {
|
---|
| 309 | $mech->get($page);
|
---|
| 310 | };
|
---|
[27210] | 311 | my $code = $mech->status();
|
---|
| 312 | print "Try $i: ($code) $@" if $@;
|
---|
[27208] | 313 | return $mech if !$@;
|
---|
| 314 | sleep(30+5*$i);
|
---|
[27211] | 315 | last if $arg && $arg eq "no503" and $code == 503;
|
---|
[27208] | 316 | $mech = WWW::Mechanize->new("agent" => $agent);
|
---|
| 317 | }
|
---|
| 318 | return $mech;
|
---|
| 319 | }
|
---|
| 320 |
|
---|
| 321 | sub getstats
|
---|
| 322 | {
|
---|
| 323 | my %results;
|
---|
| 324 | require WWW::Mechanize;
|
---|
| 325 | require Data::Dumper;
|
---|
| 326 | require URI::Escape;
|
---|
| 327 | my $mech = WWW::Mechanize->new("agent" => $agent);
|
---|
| 328 |
|
---|
| 329 | if(open DFILE,"<:utf8","launchpadtrans.data")
|
---|
| 330 | {
|
---|
| 331 | local $/;
|
---|
| 332 | $/ = undef;
|
---|
| 333 | my $val = <DFILE>;
|
---|
| 334 | eval $val;
|
---|
| 335 | close DFILE;
|
---|
| 336 | }
|
---|
| 337 |
|
---|
| 338 | binmode STDOUT, ":utf8";
|
---|
| 339 |
|
---|
| 340 | open FILE,">:utf8","launchpadtrans.txt" or die "Could not open output file.";
|
---|
| 341 |
|
---|
| 342 | for my $lang (sort keys %lang)
|
---|
| 343 | {
|
---|
[27210] | 344 | doget($mech, "https://translations.launchpad.net/josm/trunk/+pots/josm/$lang/");
|
---|
[27208] | 345 | sleep(1);
|
---|
[27210] | 346 | my $cont = $mech->content();
|
---|
[27208] | 347 | while($cont =~ /<a href="https?:\/\/launchpad.net\/~(.*?)" class="sprite person(-inactive)?">(.*?)<\/a>/g)
|
---|
| 348 | {
|
---|
| 349 | my ($code, $inactive, $name) = ($1, $2, $3);
|
---|
| 350 | if(exists($results{$code}{$lang}))
|
---|
| 351 | {
|
---|
| 352 | printf "%-5s - %-30s - Found - %s\n", $lang,$code,$name;
|
---|
| 353 | next;
|
---|
| 354 | }
|
---|
| 355 | my $urlcode = URI::Escape::uri_escape($code);
|
---|
[27211] | 356 | $mech = doget($mech, "https://translations.launchpad.net/josm/trunk/+pots/josm/$lang/+filter?person=$urlcode", "no503");
|
---|
[27208] | 357 | sleep(1);
|
---|
[27211] | 358 | my $cont = $mech->content() || "";
|
---|
| 359 | my ($count) = $cont =~ /of[\r\n\t ]+?(\d+)[\r\n\t ]+?result/;
|
---|
[27208] | 360 | if($count && $mech->status == 200)
|
---|
| 361 | {
|
---|
| 362 | $results{$code}{NAME} = $name;
|
---|
| 363 | $results{$code}{$lang} = $count;
|
---|
| 364 | $results{$code}{TOTAL} += $count;
|
---|
| 365 | if(open DFILE,">:utf8","launchpadtrans.data")
|
---|
| 366 | {
|
---|
| 367 | print DFILE Data::Dumper->Dump([\%results],['*results']);
|
---|
| 368 | close DFILE;
|
---|
| 369 | }
|
---|
| 370 | printf "%-5s - %-30s - %5d - %s\n", $lang,$code,$count,$name;
|
---|
| 371 |
|
---|
| 372 | }
|
---|
| 373 | else
|
---|
| 374 | {
|
---|
| 375 | printf "%-5s - %-30s - Skip - %s\n", $lang,$code,$name;
|
---|
| 376 | }
|
---|
| 377 | }
|
---|
| 378 | }
|
---|
| 379 | for my $code (sort {$results{$b}{TOTAL} <=> $results{$a}{TOTAL}} keys %results)
|
---|
| 380 | {
|
---|
| 381 | print FILE "$results{$code}{NAME}:$results{$code}{TOTAL}";
|
---|
| 382 | printf "%5d - %-50s",$results{$code}{TOTAL}, $results{$code}{NAME};
|
---|
| 383 | for my $lang (sort keys %{$results{$code}})
|
---|
| 384 | {
|
---|
| 385 | next if $lang eq "NAME" or $lang eq "TOTAL";
|
---|
| 386 | print FILE ";$lang=$results{$code}{$lang}";
|
---|
| 387 | printf " - %-5s=%5d",$lang, $results{$code}{$lang};
|
---|
| 388 | }
|
---|
| 389 | print FILE "\n";
|
---|
| 390 | print "\n";
|
---|
| 391 | }
|
---|
| 392 | }
|
---|