scripts/release-notes: Update for machine-xml changes, diff package list
A bit of a decent rework to the release-notes tool, coping with the
machine-xml rework (to be one package with different repos/versions per
platform, see cd986e4cf84a526ee9653f1282e7889a9312f658)
We also do a legal-info build for each platform, as the resulting
manifest.csv allows us to work out what packages have been
updated/added/removed for each platform.
Also add the '--platform' option, to produce release notes for only
one platform.
Signed-off-by: Stewart Smith <stewart@linux.vnet.ibm.com>
diff --git a/openpower/scripts/release-notes b/openpower/scripts/release-notes
index 5df6154..4231aa0 100755
--- a/openpower/scripts/release-notes
+++ b/openpower/scripts/release-notes
@@ -1,55 +1,242 @@
#!/bin/env perl
+
use strict;
+use Getopt::Long;
+use DBI;
+
+# We use an in-memory SQLite database as SQL is good for doing queries
+# of differences between data sets, and doing that in perl is annoying.
+my $db = "dbi:SQLite:dbname=release-notes.sqlite";
+#my $dbh = DBI->connect("dbi:SQLite:dbname=:memory:","","");
+my $dbh = DBI->connect($db,undef,undef, {AutoCommit => 1, RaiseError=>1});
+
my $repos =
{
'op-build' => { REPO => "http://github.com/open-power/op-build" },
'hostboot' => { REPO => "http://github.com/open-power/hostboot" ,
- DIR => "openpower/package/hostboot",
- PACKAGE => "HOSTBOOT" },
+ DIR => "openpower/package/hostboot" },
'sbe' => { REPO => "http://github.com/open-power/sbe" ,
- DIR => "openpower/package/sbe",
- PACKAGE => "SBE" },
+ DIR => "openpower/package/sbe"},
'skiboot' => { REPO => "http://github.com/open-power/skiboot" ,
- DIR => "openpower/package/skiboot",
- PACKAGE => "SKIBOOT" },
+ DIR => "openpower/package/skiboot"},
'occ' => { REPO => "http://github.com/open-power/occ" ,
- DIR => "openpower/package/occ",
- PACKAGE => "OCC" },
+ DIR => "openpower/package/occ"},
'pnor' => { REPO => "http://github.com/open-power/pnor" ,
- DIR => "openpower/package/openpower-pnor",
- PACKAGE => "OPENPOWER_PNOR" },
- 'palmetto-xml' => { REPO => "http://github.com/open-power/palmetto-xml" ,
- DIR => "openpower/package/palmetto-xml",
- PACKAGE => "PALMETTO_XML" },
- 'habanero-xml' => { REPO => "http://github.com/open-power/habanero-xml" ,
- DIR => "openpower/package/habanero-xml",
- PACKAGE => "HABANERO_XML" },
- 'firestone-xml' => { REPO => "http://github.com/open-power/firestone-xml" ,
- DIR => "openpower/package/firestone-xml",
- PACKAGE => "FIRESTONE_XML" },
- 'garrison-xml' => { REPO => "http://github.com/open-power/garrison-xml" ,
- DIR => "openpower/package/garrison-xml",
- PACKAGE => "GARRISON_XML" },
- 'barreleye-xml' => { REPO => "http://github.com/open-power/barreleye-xml" ,
- DIR => "openpower/package/barreleye-xml",
- PACKAGE => "BARRELEYE_XML" },
- 'romulus-xml' => { REPO => "http://github.com/open-power/romulus-xml" ,
- DIR => "openpower/package/romulus-xml",
- PACKAGE => "ROMULUS_XML" },
- 'witherspoon-xml' => { REPO => "http://github.com/open-power/witherspoon-xml" ,
- DIR => "openpower/package/witherspoon-xml",
- PACKAGE => "WITHERSPOON_XML" },
- 'zaius-xml' => { REPO => "http://github.com/open-power/zaius-xml" ,
- DIR => "openpower/package/zaius-xml",
- PACKAGE => "ZAIUS_XML" },
+ DIR => "openpower/package/openpower-pnor"},
'petitboot' => { REPO => "http://github.com/open-power/petitboot" ,
- DIR => "openpower/package/petitboot",
- PACKAGE => "PETITBOOT" },
+ DIR => "openpower/package/petitboot"},
};
-my $begin_release = shift;
-my $end_release = shift;
+my $begin_release;
+my $end_release;
+my $begin_worktree;
+my $end_worktree;
+my $platform;
+
+GetOptions("begin-release=s" => \$begin_release,
+ "end-release=s" => \$end_release,
+ "begin-worktree=s" => \$begin_worktree,
+ "platform=s" => \$platform,
+ "end-worktree=s" => \$end_worktree)
+ or die("Error in command line arguments");
+
+die "Required argument missing" unless ($begin_release and $end_release and $begin_worktree and $end_worktree);
+
+open(OUTPUT, "> RELEASE.md") || die "Failed to open RELEASE.md";
+
+print OUTPUT "# Release Notes for OpenPower Firmware $end_release\n";
+
+if (-e $begin_worktree) {
+ system("cd $begin_worktree && git checkout $begin_release && git submodule update --reference ../buildroot && git submodule update") and die "Could not update $begin_worktree";
+} else {
+ system("git worktree add $begin_worktree $begin_release && cd $begin_worktree&& git submodule update --reference ../buildroot && git submodule update && rm -rf dl && ln -s ../dl dl") and die "Couldn't init $begin_worktree";
+}
+
+if (-e $end_worktree) {
+ system("cd $end_worktree && git checkout $end_release && git submodule update --reference ../buildroot && git submodule update") and die "Could not update $end_worktree";
+} else {
+ system("git worktree add $end_worktree $end_release && cd $end_worktree && git submodule update --reference ../buildroot && git submodule update && rm -rf dl && ln -s ../dl dl") and die "Couldn't init $end_worktree";
+}
+
+opendir (my $dh, "$begin_worktree/openpower/configs")
+ or die "can't scan $begin_worktree defconfigs";
+my @begin_platforms = grep { /.*_defconfig/ } readdir($dh);
+closedir $dh;
+
+opendir (my $dh, "$end_worktree/openpower/configs")
+ or die "can't scan $end_worktree defconfigs";
+my @end_platforms = grep { /.*_defconfig/ } readdir($dh);
+closedir $dh;
+
+s/_defconfig// foreach (@begin_platforms);
+s/_defconfig// foreach (@end_platforms);
+
+if ($platform) {
+ @begin_platforms = ($platform);
+ @end_platforms = ($platform);
+}
+
+$dbh->do("CREATE TABLE platforms (platform TEXT, version TEXT);") or die "$!";
+{
+ my $q = "INSERT INTO platforms (platform,version) VALUES (?,?)";
+ my $sth = $dbh->prepare($q) or die "$!";
+ $sth->execute($_, $begin_release) foreach (@begin_platforms);
+ $sth->execute($_, $end_release) foreach (@end_platforms);
+}
+
+{
+ my $q = "SELECT platform FROM platforms WHERE version is ? AND platform NOT IN (SELECT platform FROM platforms WHERE version is ?)";
+ my $sth = $dbh->prepare($q) or die $!;
+ $sth->execute($begin_release, $end_release);
+ my $r;
+ print OUTPUT "## Removed platforms\n\n- ".$r->{platform}."\n" if $r = $sth->fetchrow_hashref;
+ print OUTPUT "- ".$_->{platform}."\n" while ($r = $sth->fetchrow_hashref);
+
+ $sth->execute($end_release, $begin_release);
+ print OUTPUT "## New platforms\n\n- ".$r->{platform}."\n" if $r = $sth->fetchrow_hashref;
+ print OUTPUT "- ".$_->{platform}."\n" while($r = $sth->fetchrow_hashref);
+}
+
+my @common_platforms;
+{
+ my $q = "SELECT platform FROM platforms WHERE version is ? AND EXISTS (select platform from platforms where version is ?)";
+ my $sth = $dbh->prepare($q) or die $!;
+ $sth->execute($begin_release, $end_release);
+ my $r;
+ push @common_platforms, $r->{platform} while ($r = $sth->fetchrow_hashref);
+}
+
+foreach my $p (@common_platforms) {
+ next if $p =~ /firenze/;
+ next if $p =~ /^zz$/;
+ next if $p =~ /mambo/;
+ $repos->{"$p-xml"} = { REPO => "http://github.com/open-power/$p-xml" ,
+ DIR => "openpower/package/$p-xml" };
+}
+
+foreach my $p (@begin_platforms) {
+ system("bash -c '(cd $begin_worktree && . op-build-env && op-build ".$p."_defconfig && op-build legal-info)'");
+ # Forgive me for this....
+ system("sqlite3 release-notes.sqlite \".mode csv\" \".import $begin_worktree/output/legal-info/manifest.csv begin_".$p."_manifest\"");
+}
+
+foreach my $p (@end_platforms) {
+ system("bash -c '(cd $end_worktree && . op-build-env && op-build ".$p."_defconfig && op-build legal-info)'");
+ # Forgive me for this....
+ system("sqlite3 release-notes.sqlite \".mode csv\" \".import $end_worktree/output/legal-info/manifest.csv end_".$p."_manifest\"");
+}
+
+$dbh->do(<<'SQL') or die "$!";
+CREATE TABLE package_upgrades (
+ PACKAGE TEXT,
+ OLDVERSION TEXT,
+ NEWVERSION TEXT,
+ PLATFORM TEXT
+)
+SQL
+
+foreach my $p (@common_platforms) {
+ $dbh->do("INSERT INTO package_upgrades select b.package,b.version,e.version,'$p' from begin_".$p."_manifest as b LEFT JOIN end_".$p."_manifest AS e ON b.package=e.package WHERE b.version != e.version") or die $!;
+}
+
+$dbh->do(<<'SQL') or die "$!";
+CREATE TABLE new_package (
+ PACKAGE TEXT,
+ VERSION TEXT,
+ PLATFORM TEXT
+)
+SQL
+
+foreach my $p (@common_platforms) {
+ $dbh->do("INSERT INTO new_package select b.package,b.version,'$p' from end_".$p."_manifest as b WHERE NOT EXISTS(SELECT package FROM begin_".$p."_manifest AS e WHERE b.package=e.package)") or die $!;
+}
+
+$dbh->do(<<'SQL') or die "$!";
+CREATE TABLE removed_package (
+ PACKAGE TEXT,
+ VERSION TEXT,
+ PLATFORM TEXT
+)
+SQL
+
+foreach my $p (@common_platforms) {
+ $dbh->do("INSERT INTO removed_package select b.package,b.version,'$p' from begin_".$p."_manifest as b WHERE NOT EXISTS(SELECT package FROM end_".$p."_manifest AS e WHERE b.package=e.package)") or die $!;
+}
+
+my $old_level = {};
+my $new_level = {};
+
+{
+ my $q = <<'SQL';
+select package as pk ,oldversion as o ,newversion as n,
+ GROUP_CONCAT(platform) as ps
+FROM package_upgrades
+ GROUP BY package,oldversion,newversion
+ORDER BY package,platform
+SQL
+ my $sth = $dbh->prepare($q) or die $!;
+ $sth->execute();
+
+ print OUTPUT "## Updated Packages\n\n";
+ print OUTPUT "Package | Old Version | New Version | Platforms\n";
+ print OUTPUT "--- | --- | --- | ---\n";
+ while (my $r = $sth->fetchrow_hashref) {
+ print OUTPUT join(' | ',($r->{pk}, $r->{o}, $r->{n}, $r->{ps}))."\n" ;
+ if ($r->{pk} eq 'machine-xml') {
+ $old_level->{$r->{ps}."-xml"} = $r->{o};
+ $new_level->{$r->{ps}."-xml"} = $r->{n};
+ } else {
+ $old_level->{$r->{pk}} = $r->{o};
+ $new_level->{$r->{pk}} = $r->{n};
+ }
+ }
+}
+
+{
+ my $q = <<'SQL';
+select package as pk ,version as v,
+ GROUP_CONCAT(platform) as ps
+FROM new_package
+ GROUP BY package,version
+ORDER BY package,platform
+SQL
+ my $sth = $dbh->prepare($q) or die $!;
+ $sth->execute();
+
+ print OUTPUT "\n\n## New Packages\n\n";
+ print OUTPUT "Package | Version | Platforms\n";
+ print OUTPUT "--- | --- | ---\n";
+ while (my $r = $sth->fetchrow_hashref) {
+ print OUTPUT join(' | ',($r->{pk}, $r->{v}, $r->{ps}))."\n" ;
+ if ($r->{pk} eq 'machine-xml') {
+ $new_level->{$r->{ps}."-xml"} = $r->{v};
+ }
+ }
+}
+
+{
+ my $q = <<'SQL';
+select package as pk ,version as v,
+ GROUP_CONCAT(platform) as ps
+FROM removed_package
+ GROUP BY package,version
+ORDER BY package,platform
+SQL
+ my $sth = $dbh->prepare($q) or die $!;
+ $sth->execute();
+
+ print OUTPUT "\n\n## Removed Packages\n\n";
+ print OUTPUT "Package | Version | Platforms\n";
+ print OUTPUT "--- | --- | ---\n";
+ while (my $r = $sth->fetchrow_hashref) {
+ print OUTPUT join(' | ',($r->{pk}, $r->{v}, $r->{ps}))."\n" ;
+ if ($r->{pk} eq $r->{ps}."-xml") {
+ $old_level->{$r->{ps}."-xml"} = $r->{v};
+ }
+ }
+ print OUTPUT "\n\n";
+}
foreach my $repo (keys %{$repos})
{
@@ -67,67 +254,11 @@
system("cd op-build; git checkout $end_release --force; git reset HEAD --hard");
-open(OP_SUMMARY, "cd op-build; ".
- "git diff $begin_release...$end_release --name-only |".
- "grep -e \"\.mk\" -e \"Config.in\" |".
- "xargs git diff $begin_release...$end_release -- |".
- "grep -e '^[+-][^[:space:]]*_VERSION' ".
- "-e'^[+-][[:space:]]*default.*if.*LATEST_VERSION' |") || die;
-
-my $old_level = {};
-my $new_level = {};
-
-while(my $line = <OP_SUMMARY>)
-{
- chomp $line;
- if ($line =~ m/\$/) # Sanity check there are not variables here.
- {
- next;
- }
- if ($line =~ m/-([^[:space:]]*)_VERSION([[:space:]]*\?*=[[:space:]]*)(.*)/)
- {
- print "Old $1:$3\n";
- $old_level->{$1} = $3;
- }
- if ($line =~ m/-([^[:space:]]*)_VERSION_BRANCH_MASTER([[:space:]]*\?*=[[:space:]]*)(.*)/)
- {
- print "Old $1:$3\n";
- $old_level->{$1} = $3;
- }
- if ($line =~ m/\+([^[:space:]]*)_VERSION([[:space:]]*\?*=[[:space:]]*)(.*)/)
- {
- print "New $1:$3\n";
- $new_level->{$1} = $3;
- }
- if ($line =~ m/\+([^[:space:]]*)_VERSION_BRANCH_MASTER([[:space:]]*\?*=[[:space:]]*)(.*)/)
- {
- print "New $1:$3\n";
- $new_level->{$1} = $3;
- }
-
- if ($line =~ m/-[[:space:]]*default[[:space:]]*"([^[:space:]]*)"[[:space:]]*if[[:space:]]BR2_([^[:space:]]*)_LATEST_VERSION/)
- {
- print "Old $2:$1\n";
- $old_level->{$2} = $1;
- }
- if ($line =~ m/\+[[:space:]]*default[[:space:]]*"([^[:space:]]*)"[[:space:]]*if[[:space:]]BR2_([^[:space:]]*)_LATEST_VERSION/)
- {
- print "New $2:$1\n";
- $new_level->{$2} = $1;
- }
-}
-
-open(OUTPUT, "> RELEASE.md") || die "Failed to open RELEASE.md";
-
-print OUTPUT "# Release Notes for OpenPower Firmware $end_release\n";
-
my $op_url = $repos->{'op-build'}->{REPO};
foreach my $repo (sort keys %{$repos})
{
- next if (not exists $repos->{$repo}->{PACKAGE});
-
- my $package = $repos->{$repo}->{PACKAGE};
+ my $package = $repo;
my $url = $repos->{$repo}->{REPO};
my $dir = $repos->{$repo}->{DIR};
@@ -135,8 +266,6 @@
print OUTPUT "[Repository]($url)\n";
print OUTPUT "\n";
- my $package = $repos->{$repo}->{PACKAGE};
-
# Display patches.
if (open(LSLOG, "ls op-build/$dir/*.patch | ".
"xargs -n1 --no-run-if-empty basename |"))
@@ -171,7 +300,6 @@
(exists $new_level->{$package}))
{
print "Changes in $repo...\n";
-
open(GITLOG, "cd $repo; git shortlog $old_level->{$package}...".
"$new_level->{$package} --no-merges --format=".
"\"* [%h]($url/commit/%h) %s\" |");