blob: 483a784b154cd0165db9c029183671a8ded03020 [file] [log] [blame]
Samuel Mendoza-Jonas52bd75b2018-02-28 10:59:02 +11001#!/usr/bin/env perl
Stewart Smithf76e7f92017-05-19 10:42:01 +10002
Patrick Williams37567b72015-06-08 15:25:40 -05003use strict;
4
Stewart Smithf76e7f92017-05-19 10:42:01 +10005use Getopt::Long;
6use DBI;
7
8# We use an in-memory SQLite database as SQL is good for doing queries
9# of differences between data sets, and doing that in perl is annoying.
10my $db = "dbi:SQLite:dbname=release-notes.sqlite";
11#my $dbh = DBI->connect("dbi:SQLite:dbname=:memory:","","");
12my $dbh = DBI->connect($db,undef,undef, {AutoCommit => 1, RaiseError=>1});
13
Patrick Williams37567b72015-06-08 15:25:40 -050014my $repos =
15{
Samuel Mendoza-Jonasa6262a02017-08-17 14:46:02 +100016 'op-build' => { REPO => "https://github.com/open-power/op-build" },
17 'hostboot' => { REPO => "https://github.com/open-power/hostboot" ,
Stewart Smithf76e7f92017-05-19 10:42:01 +100018 DIR => "openpower/package/hostboot" },
Samuel Mendoza-Jonasa6262a02017-08-17 14:46:02 +100019 'sbe' => { REPO => "https://github.com/open-power/sbe" ,
Stewart Smithf76e7f92017-05-19 10:42:01 +100020 DIR => "openpower/package/sbe"},
Samuel Mendoza-Jonasa6262a02017-08-17 14:46:02 +100021 'skiboot' => { REPO => "https://github.com/open-power/skiboot" ,
Stewart Smithf76e7f92017-05-19 10:42:01 +100022 DIR => "openpower/package/skiboot"},
Samuel Mendoza-Jonasa6262a02017-08-17 14:46:02 +100023 'occ' => { REPO => "https://github.com/open-power/occ" ,
Stewart Smithf76e7f92017-05-19 10:42:01 +100024 DIR => "openpower/package/occ"},
Samuel Mendoza-Jonasa6262a02017-08-17 14:46:02 +100025 'pnor' => { REPO => "https://github.com/open-power/pnor" ,
Stewart Smithf76e7f92017-05-19 10:42:01 +100026 DIR => "openpower/package/openpower-pnor"},
Samuel Mendoza-Jonasa6262a02017-08-17 14:46:02 +100027 'petitboot' => { REPO => "https://github.com/open-power/petitboot" ,
Stewart Smithf76e7f92017-05-19 10:42:01 +100028 DIR => "openpower/package/petitboot"},
Patrick Williams37567b72015-06-08 15:25:40 -050029};
30
Stewart Smithf76e7f92017-05-19 10:42:01 +100031my $begin_release;
32my $end_release;
33my $begin_worktree;
34my $end_worktree;
35my $platform;
36
37GetOptions("begin-release=s" => \$begin_release,
38 "end-release=s" => \$end_release,
39 "begin-worktree=s" => \$begin_worktree,
40 "platform=s" => \$platform,
41 "end-worktree=s" => \$end_worktree)
42 or die("Error in command line arguments");
43
44die "Required argument missing" unless ($begin_release and $end_release and $begin_worktree and $end_worktree);
45
46open(OUTPUT, "> RELEASE.md") || die "Failed to open RELEASE.md";
47
48print OUTPUT "# Release Notes for OpenPower Firmware $end_release\n";
49
50if (-e $begin_worktree) {
51 system("cd $begin_worktree && git checkout $begin_release && git submodule update --reference ../buildroot && git submodule update") and die "Could not update $begin_worktree";
52} else {
53 system("git worktree add $begin_worktree $begin_release && cd $begin_worktree&& git submodule update --reference ../buildroot && git submodule update && rm -rf dl && ln -s ../dl dl") and die "Couldn't init $begin_worktree";
54}
55
56if (-e $end_worktree) {
57 system("cd $end_worktree && git checkout $end_release && git submodule update --reference ../buildroot && git submodule update") and die "Could not update $end_worktree";
58} else {
59 system("git worktree add $end_worktree $end_release && cd $end_worktree && git submodule update --reference ../buildroot && git submodule update && rm -rf dl && ln -s ../dl dl") and die "Couldn't init $end_worktree";
60}
61
62opendir (my $dh, "$begin_worktree/openpower/configs")
63 or die "can't scan $begin_worktree defconfigs";
64my @begin_platforms = grep { /.*_defconfig/ } readdir($dh);
65closedir $dh;
66
67opendir (my $dh, "$end_worktree/openpower/configs")
68 or die "can't scan $end_worktree defconfigs";
69my @end_platforms = grep { /.*_defconfig/ } readdir($dh);
70closedir $dh;
71
72s/_defconfig// foreach (@begin_platforms);
73s/_defconfig// foreach (@end_platforms);
74
Stewart Smith24f82552017-07-14 15:53:22 +100075my $witherspoon_insanity;
76
Samuel Mendoza-Jonas93c5a542017-09-21 13:55:00 +100077# If both witherspoon and witherspoon-sequoia exist we've switched back
78# to a single witherspoon platform and the -sequoia platform is just to
79# keep Jenkins happy - ignore it.
80if ("witherspoon" ~~ @begin_platforms
81 && "witherspoon-sequoia" ~~ @begin_platforms) {
82 my $index = 0;
83 $index++ until @begin_platforms[$index] eq "witherspoon-sequoia";
84 splice(@begin_platforms, $index, 1);
85}
86
87if ("witherspoon" ~~ @end_platforms
88 && "witherspoon-sequoia" ~~ @end_platforms) {
89 my $index = 0;
90 $index++ until @end_platforms[$index] eq "witherspoon-sequoia";
91 splice(@end_platforms, $index, 1);
92}
93
Stewart Smith88c7e352017-07-14 15:12:36 +100094if (($platform && $platform eq 'witherspoon')
95 && -f "$end_worktree/openpower/configs/witherspoon-sequoia_defconfig") {
96 @begin_platforms = ('witherspoon');
97 @end_platforms = ('witherspoon-sequoia', 'witherspoon-redbud');
Stewart Smith24f82552017-07-14 15:53:22 +100098 $witherspoon_insanity = 1;
Stewart Smith88c7e352017-07-14 15:12:36 +100099} elsif ($platform) {
Stewart Smithf76e7f92017-05-19 10:42:01 +1000100 @begin_platforms = ($platform);
101 @end_platforms = ($platform);
102}
103
104$dbh->do("CREATE TABLE platforms (platform TEXT, version TEXT);") or die "$!";
105{
106 my $q = "INSERT INTO platforms (platform,version) VALUES (?,?)";
107 my $sth = $dbh->prepare($q) or die "$!";
108 $sth->execute($_, $begin_release) foreach (@begin_platforms);
109 $sth->execute($_, $end_release) foreach (@end_platforms);
110}
111
112{
Stewart Smith88c7e352017-07-14 15:12:36 +1000113 my $q = "SELECT platform as p FROM platforms WHERE version is ? AND platform NOT IN (SELECT platform FROM platforms WHERE version is ? and platform=p)";
Stewart Smithf76e7f92017-05-19 10:42:01 +1000114 my $sth = $dbh->prepare($q) or die $!;
115 $sth->execute($begin_release, $end_release);
116 my $r;
Stewart Smith59d3cb42018-02-12 11:52:12 +1100117 print OUTPUT "\n## Removed platforms\n\n- ".$r->{p}."\n" if $r = $sth->fetchrow_hashref;
Stewart Smith88c7e352017-07-14 15:12:36 +1000118 print OUTPUT "- ".$r->{p}."\n" while ($r = $sth->fetchrow_hashref);
Stewart Smithf76e7f92017-05-19 10:42:01 +1000119
120 $sth->execute($end_release, $begin_release);
Stewart Smith59d3cb42018-02-12 11:52:12 +1100121 print OUTPUT "\n## New platforms\n\n- ".$r->{p}."\n" if $r = $sth->fetchrow_hashref;
Stewart Smith88c7e352017-07-14 15:12:36 +1000122 print OUTPUT "- ".$r->{p}."\n" while($r = $sth->fetchrow_hashref);
Stewart Smithf76e7f92017-05-19 10:42:01 +1000123}
124
125my @common_platforms;
126{
Stewart Smith88c7e352017-07-14 15:12:36 +1000127 my $q = "SELECT platform as p FROM platforms WHERE version is ? AND EXISTS (select platform from platforms where version is ? and platform=p)";
Stewart Smithf76e7f92017-05-19 10:42:01 +1000128 my $sth = $dbh->prepare($q) or die $!;
129 $sth->execute($begin_release, $end_release);
130 my $r;
Stewart Smith88c7e352017-07-14 15:12:36 +1000131 push @common_platforms, $r->{p} while ($r = $sth->fetchrow_hashref);
Stewart Smithf76e7f92017-05-19 10:42:01 +1000132}
133
Stewart Smith88c7e352017-07-14 15:12:36 +1000134use Data::Dumper;
135
136print "# COMMON PLATFORMS\n";
137print Dumper(\@common_platforms);
138
Stewart Smithf76e7f92017-05-19 10:42:01 +1000139foreach my $p (@common_platforms) {
140 next if $p =~ /firenze/;
141 next if $p =~ /^zz$/;
142 next if $p =~ /mambo/;
Samuel Mendoza-Jonas84f3d252017-07-12 13:26:55 +1000143 next if $p =~ /redbud/;
Samuel Mendoza-Jonas264a6cd2017-08-04 11:42:41 +1000144 next if $p =~ /pseries/;
145 next if $p =~ /witherspoon-redbud/;
146 if ($p =~ /witherspoon-sequoia/) {
147 $p = "witherspoon";
148 $witherspoon_insanity = 1;
149 }
Samuel Mendoza-Jonasa6262a02017-08-17 14:46:02 +1000150 $repos->{"$p-xml"} = { REPO => "https://github.com/open-power/$p-xml" ,
Stewart Smithf76e7f92017-05-19 10:42:01 +1000151 DIR => "openpower/package/$p-xml" };
152}
153
154foreach my $p (@begin_platforms) {
155 system("bash -c '(cd $begin_worktree && . op-build-env && op-build ".$p."_defconfig && op-build legal-info)'");
Stewart Smith88c7e352017-07-14 15:12:36 +1000156 # Work-around bug in op-build v1.17, fixed 2 commits later
157 system('sed -e \'s/,""\([^",]\)/,"\1/; s/\([^,]\)"",/\1",/; s/machine-xml-"\(.*\)".tar/machine-xml-\1.tar/\' -i '. $begin_worktree .'/output/legal-info/manifest.csv');
Stewart Smithf76e7f92017-05-19 10:42:01 +1000158 # Forgive me for this....
Stewart Smithcd96a992017-07-14 15:40:19 +1000159 system("sqlite3 release-notes.sqlite \".mode csv\" \".import $begin_worktree/output/legal-info/manifest.csv i\"");
160 $dbh->do("ALTER TABLE i RENAME to 'begin_".$p."_manifest'") or die $!;
Stewart Smithf76e7f92017-05-19 10:42:01 +1000161}
162
163foreach my $p (@end_platforms) {
Stewart Smith88c7e352017-07-14 15:12:36 +1000164 print "# END PLATFORMS LEGAL-INFO $p\n";
Stewart Smithf76e7f92017-05-19 10:42:01 +1000165 system("bash -c '(cd $end_worktree && . op-build-env && op-build ".$p."_defconfig && op-build legal-info)'");
166 # Forgive me for this....
Stewart Smithcd96a992017-07-14 15:40:19 +1000167 print "# loading manifest\n";
168 system("sqlite3 release-notes.sqlite \".mode csv\" \".import $end_worktree/output/legal-info/manifest.csv i\"");
169 $dbh->do("ALTER TABLE i RENAME to 'end_".$p."_manifest'") or die $!;
Stewart Smithf76e7f92017-05-19 10:42:01 +1000170}
171
Stewart Smith24f82552017-07-14 15:53:22 +1000172if ($witherspoon_insanity) {
173 $dbh->do("ALTER TABLE 'end_witherspoon-sequoia_manifest' RENAME to end_witherspoon_manifest");
Samuel Mendoza-Jonas264a6cd2017-08-04 11:42:41 +1000174 $dbh->do("ALTER TABLE 'begin_witherspoon-sequoia_manifest' RENAME to begin_witherspoon_manifest");
Stewart Smith24f82552017-07-14 15:53:22 +1000175 @end_platforms = grep { $_ != 'witherspoon-sequoia' } @end_platforms;
Samuel Mendoza-Jonas264a6cd2017-08-04 11:42:41 +1000176 @begin_platforms = grep { $_ != 'witherspoon-sequoia' } @begin_platforms;
Stewart Smith24f82552017-07-14 15:53:22 +1000177 push @end_platforms, 'witherspoon';
Samuel Mendoza-Jonas264a6cd2017-08-04 11:42:41 +1000178 push @begin_platforms, 'witherspoon';
Stewart Smith24f82552017-07-14 15:53:22 +1000179 push @common_platforms, 'witherspoon';
180}
181
Stewart Smithf76e7f92017-05-19 10:42:01 +1000182$dbh->do(<<'SQL') or die "$!";
183CREATE TABLE package_upgrades (
184 PACKAGE TEXT,
185 OLDVERSION TEXT,
186 NEWVERSION TEXT,
187 PLATFORM TEXT
188)
189SQL
190
191foreach my $p (@common_platforms) {
Stewart Smith88c7e352017-07-14 15:12:36 +1000192 $dbh->do("INSERT INTO package_upgrades select b.package,b.version,e.version,'$p' from 'begin_".$p."_manifest' as b LEFT JOIN 'end_".$p."_manifest' AS e ON b.package=e.package WHERE b.version != e.version") or die $!;
Stewart Smithf76e7f92017-05-19 10:42:01 +1000193}
194
195$dbh->do(<<'SQL') or die "$!";
196CREATE TABLE new_package (
197 PACKAGE TEXT,
198 VERSION TEXT,
199 PLATFORM TEXT
200)
201SQL
202
203foreach my $p (@common_platforms) {
Stewart Smith88c7e352017-07-14 15:12:36 +1000204 $dbh->do("INSERT INTO new_package select b.package,b.version,'$p' from 'end_".$p."_manifest' as b WHERE NOT EXISTS(SELECT package FROM 'begin_".$p."_manifest' AS e WHERE b.package=e.package)") or die $!;
Stewart Smithf76e7f92017-05-19 10:42:01 +1000205}
206
207$dbh->do(<<'SQL') or die "$!";
208CREATE TABLE removed_package (
209 PACKAGE TEXT,
210 VERSION TEXT,
211 PLATFORM TEXT
212)
213SQL
214
215foreach my $p (@common_platforms) {
Stewart Smith88c7e352017-07-14 15:12:36 +1000216 $dbh->do("INSERT INTO removed_package select b.package,b.version,'$p' from 'begin_".$p."_manifest' as b WHERE NOT EXISTS(SELECT package FROM 'end_".$p."_manifest' AS e WHERE b.package=e.package)") or die $!;
Stewart Smithf76e7f92017-05-19 10:42:01 +1000217}
218
219my $old_level = {};
220my $new_level = {};
221
222{
223 my $q = <<'SQL';
224select package as pk ,oldversion as o ,newversion as n,
Stewart Smith59d3cb42018-02-12 11:52:12 +1100225 GROUP_CONCAT(platform,', ') as ps
Stewart Smithf76e7f92017-05-19 10:42:01 +1000226FROM package_upgrades
227 GROUP BY package,oldversion,newversion
228ORDER BY package,platform
229SQL
230 my $sth = $dbh->prepare($q) or die $!;
231 $sth->execute();
232
Stewart Smith59d3cb42018-02-12 11:52:12 +1100233 print OUTPUT "\n## Updated Packages\n\n";
Stewart Smithf76e7f92017-05-19 10:42:01 +1000234 print OUTPUT "Package | Old Version | New Version | Platforms\n";
Stewart Smith59d3cb42018-02-12 11:52:12 +1100235 my $t;
236 my ($plen,$olen,$nlen,$platlen) = (3,3,3,3);
Stewart Smithf76e7f92017-05-19 10:42:01 +1000237 while (my $r = $sth->fetchrow_hashref) {
Stewart Smith59d3cb42018-02-12 11:52:12 +1100238 $t.= join(' | ',($r->{pk}, $r->{o}, $r->{n}, $r->{ps}))."\n" ;
239 $plen = length($r->{pk}) if $plen < length($r->{pk});
240 $olen = length($r->{o}) if $olen < length($r->{o});
241 $nlen = length($r->{n}) if $nlen < length($r->{n});
242 $platlen = length($r->{ps}) if $platlen < length($r->{ps});
243
Stewart Smithf76e7f92017-05-19 10:42:01 +1000244 if ($r->{pk} eq 'machine-xml') {
245 $old_level->{$r->{ps}."-xml"} = $r->{o};
246 $new_level->{$r->{ps}."-xml"} = $r->{n};
247 } else {
248 $old_level->{$r->{pk}} = $r->{o};
249 $new_level->{$r->{pk}} = $r->{n};
250 }
251 }
Stewart Smith59d3cb42018-02-12 11:52:12 +1100252 print OUTPUT "-"x$plen." | "."-"x$olen." | "."-"x$nlen." | ".
253 "-"x$platlen."\n";
254 print OUTPUT $t;
Stewart Smithf76e7f92017-05-19 10:42:01 +1000255}
256
257{
258 my $q = <<'SQL';
259select package as pk ,version as v,
260 GROUP_CONCAT(platform) as ps
261FROM new_package
262 GROUP BY package,version
263ORDER BY package,platform
264SQL
265 my $sth = $dbh->prepare($q) or die $!;
266 $sth->execute();
267
268 print OUTPUT "\n\n## New Packages\n\n";
269 print OUTPUT "Package | Version | Platforms\n";
270 print OUTPUT "--- | --- | ---\n";
271 while (my $r = $sth->fetchrow_hashref) {
272 print OUTPUT join(' | ',($r->{pk}, $r->{v}, $r->{ps}))."\n" ;
273 if ($r->{pk} eq 'machine-xml') {
274 $new_level->{$r->{ps}."-xml"} = $r->{v};
275 }
276 }
277}
278
279{
280 my $q = <<'SQL';
281select package as pk ,version as v,
282 GROUP_CONCAT(platform) as ps
283FROM removed_package
284 GROUP BY package,version
285ORDER BY package,platform
286SQL
287 my $sth = $dbh->prepare($q) or die $!;
288 $sth->execute();
289
290 print OUTPUT "\n\n## Removed Packages\n\n";
291 print OUTPUT "Package | Version | Platforms\n";
292 print OUTPUT "--- | --- | ---\n";
293 while (my $r = $sth->fetchrow_hashref) {
294 print OUTPUT join(' | ',($r->{pk}, $r->{v}, $r->{ps}))."\n" ;
295 if ($r->{pk} eq $r->{ps}."-xml") {
296 $old_level->{$r->{ps}."-xml"} = $r->{v};
297 }
298 }
299 print OUTPUT "\n\n";
300}
Patrick Williams37567b72015-06-08 15:25:40 -0500301
302foreach my $repo (keys %{$repos})
303{
304 if (-e $repo)
305 {
306 system("cd $repo; git fetch") && die "Could not fetch $repo";
307 }
308 else
309 {
310 system("git clone $repos->{$repo}->{REPO} $repo") &&
311 die "Could not clone $repo";
312 }
313}
314
315system("cd op-build; git checkout $end_release --force; git reset HEAD --hard");
316
317
Patrick Williams37567b72015-06-08 15:25:40 -0500318my $op_url = $repos->{'op-build'}->{REPO};
319
320foreach my $repo (sort keys %{$repos})
321{
Stewart Smithf76e7f92017-05-19 10:42:01 +1000322 my $package = $repo;
Patrick Williams37567b72015-06-08 15:25:40 -0500323 my $url = $repos->{$repo}->{REPO};
324 my $dir = $repos->{$repo}->{DIR};
325
Stewart Smith59d3cb42018-02-12 11:52:12 +1100326 print OUTPUT "\n## Package: $repo\n";
Patrick Williams37567b72015-06-08 15:25:40 -0500327 print OUTPUT "[Repository]($url)\n";
328 print OUTPUT "\n";
329
Patrick Williams37567b72015-06-08 15:25:40 -0500330 # Display patches.
331 if (open(LSLOG, "ls op-build/$dir/*.patch | ".
332 "xargs -n1 --no-run-if-empty basename |"))
333 {
334 print OUTPUT "### Patches\n";
335 while (my $logline = <LSLOG>)
336 {
337 chomp $logline;
338 print OUTPUT "* [$logline]".
339 "($op_url/tree/$end_release/$dir/$logline)\n";
340 }
341 close LSLOG;
342 print OUTPUT "\n";
343 }
344 else
345 {
346 print OUTPUT "None.\n";
347 }
348
349 # Display changes.
350 print OUTPUT "### Commits\n";
351 if ((not exists $old_level->{$package}) &&
352 (not exists $new_level->{$package}))
353 {
354 # No change identified.
355 print "No changes: $repo\n";
Stewart Smith59d3cb42018-02-12 11:52:12 +1100356 print OUTPUT "No changes.\n\n";
Patrick Williams37567b72015-06-08 15:25:40 -0500357 next;
358 }
359
360 if ((exists $old_level->{$package}) &&
361 (exists $new_level->{$package}))
362 {
363 print "Changes in $repo...\n";
Patrick Williams37567b72015-06-08 15:25:40 -0500364 open(GITLOG, "cd $repo; git shortlog $old_level->{$package}...".
365 "$new_level->{$package} --no-merges --format=".
Stewart Smith59d3cb42018-02-12 11:52:12 +1100366 "\"- [%h]($url/commit/%h) %s\" |");
Patrick Williams37567b72015-06-08 15:25:40 -0500367
368 while (my $logline = <GITLOG>)
369 {
370 chomp $logline;
371 $logline =~ s/^[[:space:]]*//;
Stewart Smith59d3cb42018-02-12 11:52:12 +1100372 $logline =~ s/:$/:\n/;
Patrick Williams37567b72015-06-08 15:25:40 -0500373 print OUTPUT "$logline\n";
374 }
375 close GITLOG;
376 print OUTPUT "\n";
377 next;
378 }
379
380 if (not exists $old_level->{$package})
381 {
382 print "New package $repo.\n";
383 print OUTPUT "New package.\n";
384 next;
385 }
386
387 if (not exists $new_level->{$package})
388 {
389 print "Deleted package $repo.\n";
390 print OUTPUT "Package removed.\n";
391 next;
392 }
393}