[flatpaks/flatpak-runtime] f45: generate-report: show packages pulled in by dependencies

[Owen W. Taylor <otaylor@fishsoup.net>]

A new commit has been pushed.

Repo   : flatpaks/flatpak-runtime
Branch : f45
Commit : adac2c90a454b6cee70f50aa7564020414282604
Author : Owen W. Taylor <otaylor@fishsoup.net>
Date   : 2018-01-31T11:21:01+01:00
Stats  : +113/-47 in 4 file(s)
URL    : https://src.fedoraproject.org/flatpaks/flatpak-runtime/c/adac2c90a454b6cee70f50aa7564020414282604?branch=f45

Log:
generate-report: show packages pulled in by dependencies

---
diff --git a/README.md b/README.md
index f7f6053..d3a6037 100644
--- a/README.md
+++ b/README.md
@@ -80,6 +80,8 @@ can compare to the package sets of the desktop modules.
 You also need `python3-jinja2 python3-dnf` and possibly a few other Python
 packages installed.
 
+And finally, you'll need to have [fedmod](https://pagure.io/modularity/fedmod) installed.
+
 *Report generation*: run `make`. The first run will download f27 data
 into the user-specific DNF cache and also generate a big index of all files
 in Fedora. Subsequent runs are faster. Then open `report.html` in your

diff --git a/generate-report.py b/generate-report.py
index 0cb1adf..58a8a72 100755
--- a/generate-report.py
+++ b/generate-report.py
@@ -2,20 +2,30 @@
 
 from jinja2 import Environment, FileSystemLoader, select_autoescape
 import locale
+import os
+import subprocess
+import sys
+
+def start(msg):
+    print("{}: \033[90m{} ... \033[39m".format(os.path.basename(sys.argv[0]), msg), file=sys.stderr, end="")
+    sys.stderr.flush()
+
+def done():
+    print("\033[90mdone\033[39m", file=sys.stderr)
 
 class Package(object):
     def __init__(self, name):
         self.name = name
-        self.freedesktop_platform = False
+        self.freedesktop_platform = 0
         self.freedesktop_platform_files = None
-        self.gnome_platform = False
+        self.gnome_platform = 0
         self.gnome_platform_files = None
-        self.freedesktop_sdk = False
+        self.freedesktop_sdk = 0
         self.freedesktop_sdk_files = None
-        self.gnome_sdk = False
+        self.gnome_sdk = 0
         self.gnome_sdk_files = None
-        self.live = False
-        self.rf26 = False
+        self.live = 0
+        self.rf26 = 0
 
     @property
     def runtimes(self):
@@ -23,17 +33,27 @@ class Package(object):
 
     @property
     def klass(self):
+        k = ""
         if self.live and not self.runtimes:
-            return "live-only"
+            k = "live-only"
         if self.gnome_platform and not self.live:
-            return "not-on-live"
-        return ""
+            k = "not-on-live"
+
+        if self.modules in ("", "installer"):
+            k += " build"
+
+        return k
+
+    @property
+    def modules(self):
+        return package_to_module.get(self.name, "")
 
     @property
     def note(self):
-        return {
-            "not-on-live": "platform package not on Live image"
-        }.get(self.klass, "")
+        if self.gnome_platform and not self.live:
+            return "platform package not on Live image"
+        else:
+            return ""
 
     def files_str(self, which):
         files = getattr(self, which + '_files')
@@ -65,25 +85,34 @@ class Letter(object):
         self.letter = letter
         self.packages = []
 
-env = Environment(
-    loader=FileSystemLoader('.'),
-    autoescape=select_autoescape(['html', 'xml']),
-    trim_blocks=True,
-    lstrip_blocks=True
-)
+#
+# Get information about packages
+#
 
 packages = dict()
-def add_package(name, which):
+def add_package(name, which, level):
     pkg = packages.get(name, None)
     if pkg is None:
         pkg = Package(name)
         packages[name] = pkg
-    setattr(pkg,which, True)
+    if getattr(pkg, which) < level:
+        setattr(pkg, which, level)
 
-def add_packages(filename, which):
+def add_packages(filename, which, resolve_deps=False):
+    start("Adding packages from {}".format(filename))
     with open(filename) as f:
-        for line in f:
-            add_package(line.strip(), which)
+        packages = list(line.strip() for line in f)
+
+    for package in packages:
+        add_package(package, which, level=2)
+
+    if resolve_deps:
+        output = subprocess.check_output(['fedmod', 'resolve-deps'] + packages, encoding='utf-8')
+        dep_packages = list(line.strip() for line in output.split())
+
+        for package in dep_packages:
+            add_package(package, which, level=1)
+    done()
 
 def add_package_files(filename, which):
     with open(filename) as f:
@@ -97,10 +126,10 @@ def add_package_files(filename, which):
             else:
                 setattr(pkg, which + '_files', [f])
 
-add_packages('out/freedesktop-Platform.packages', 'freedesktop_platform')
-add_packages('out/freedesktop-Sdk.packages', 'freedesktop_sdk')
-add_packages('out/gnome-Platform.packages', 'gnome_platform')
-add_packages('out/gnome-Sdk.packages', 'gnome_sdk')
+add_packages('out/freedesktop-Platform.packages', 'freedesktop_platform', resolve_deps=True)
+add_packages('out/freedesktop-Sdk.packages', 'freedesktop_sdk', resolve_deps=True)
+add_packages('out/gnome-Platform.packages', 'gnome_platform', resolve_deps=True)
+add_packages('out/gnome-Sdk.packages', 'gnome_sdk', resolve_deps=True)
 add_packages('f27-live.packages', 'live')
 add_packages('f26-flatpak-runtime.packages', 'rf26')
 
@@ -109,17 +138,6 @@ add_package_files('out/freedesktop-Sdk.matched', 'freedesktop_sdk')
 add_package_files('out/gnome-Platform.matched', 'gnome_platform')
 add_package_files('out/gnome-Sdk.matched', 'gnome_sdk')
 
-def count_lines(fname):
-    with open(fname) as f:
-        return len(list(f))
-
-unmatched_counts = {
-    'freedesktop_platform': count_lines('out/freedesktop-Platform.unmatched'),
-    'gnome_platform': count_lines('out/gnome-Platform.unmatched'),
-    'freedesktop_sdk': count_lines('out/freedesktop-Sdk.unmatched'),
-    'gnome_sdk': count_lines('out/gnome-Sdk.unmatched'),
-}
-
 letters_map = dict()
 for k, v in packages.items():
     l = v.name[0].upper()
@@ -135,8 +153,44 @@ for k in sorted(letters_map.keys()):
     letter.packages.sort(key=lambda p: locale.strxfrm(p.name))
     letters.append(letter)
 
+start("Loading package to module map")
+package_to_module = dict()
+
+output = subprocess.check_output(['fedmod', 'list-rpms', '--list-modules'], encoding='utf-8')
+for l in output.split('\n'):
+    fields = l.strip().split()
+    if len(fields) != 2:
+        continue
+    package_to_module[fields[0]] = fields[1][1:-1]
+done()
+
+#
+# Get summary information for unmatched files
+#
+
+def count_lines(fname):
+    with open(fname) as f:
+        return len(list(f))
+
+unmatched_counts = {
+    'freedesktop_platform': count_lines('out/freedesktop-Platform.unmatched'),
+    'gnome_platform': count_lines('out/gnome-Platform.unmatched'),
+    'freedesktop_sdk': count_lines('out/freedesktop-Sdk.unmatched'),
+    'gnome_sdk': count_lines('out/gnome-Sdk.unmatched'),
+}
+
+#
+# Generate the report
+#
+
+env = Environment(
+    loader=FileSystemLoader('.'),
+    autoescape=select_autoescape(['html', 'xml']),
+    trim_blocks=True,
+    lstrip_blocks=True
+)
+
 template = env.get_template('report-template.html')
 
 with open('report.html', 'w') as f:
-    f.write(template.render(letters=letters, unmatched=unmatched_counts))
-
+    f.write(template.render(letters=letters, unmatched=unmatched_counts, package_to_module=package_to_module))

diff --git a/report-template.html b/report-template.html
index 86272a5..fd47747 100644
--- a/report-template.html
+++ b/report-template.html
@@ -76,16 +76,18 @@
 	  <th>LIVE</th>
 	  <th>RF26</th>
 	  <th></th>
+	  <th></th>
 	</tr>
 	{% for package in letter.packages %}
 	<tr class="package {{ package.klass }} {{ 'hidden' if not package.runtimes }}" {{ 'data-in-runtimes' if package.runtimes else '' }} {{ 'data-in-live' if package.live else '' }} {{ 'data-in-rf26' if package.rf26 else '' }}>
 	  <td>{{ package.name }}</td>
-	  <td class="{{'present' if package.freedesktop_platform else 'absent'}}" title="{{ package.freedesktop_platform_files_str }}"></td>
-	  <td class="{{'present' if package.gnome_platform else 'absent'}}" title="{{ package.gnome_platform_files_str }}"></td>
-	  <td class="{{'present' if package.freedesktop_sdk else 'absent'}}" title="{{ package.freedesktop_sdk_files_str }}"></td>
-	  <td class="{{'present' if package.gnome_sdk else 'absent'}}" title="{{ package.gnome_sdk_files_str }}"></td>
-	  <td class="{{'present' if package.live else 'absent'}}"></td>
-	  <td class="{{'present' if package.rf26 else 'absent'}}"></td>
+	  <td class="{{('absent','dep','present')[package.freedesktop_platform]}}" title="{{ package.freedesktop_platform_files_str }}"></td>
+	  <td class="{{('absent','dep','present')[package.gnome_platform]}}" title="{{ package.gnome_platform_files_str }}"></td>
+	  <td class="{{('absent','dep','present')[package.freedesktop_sdk]}}" title="{{ package.freedesktop_sdk_files_str }}"></td>
+	  <td class="{{('absent','dep','present')[package.gnome_sdk]}}" title="{{ package.gnome_sdk_files_str }}"></td>
+	  <td class="{{('absent','dep','present')[package.live]}}"></td>
+	  <td class="{{('absent','dep','present')[package.rf26]}}"></td>
+	  <td>{{ package.modules}} </td>
 	  <td>{{ package.note }}</td>
 	</tr>
 	{% endfor %}

diff --git a/report.css b/report.css
index 3317647..0b27390 100644
--- a/report.css
+++ b/report.css
@@ -62,7 +62,12 @@ tr {
     border-right: 1px solid #aa8888;
 }
 td.present {
-    background: #bbbbbb;
+    background: #aaaaaa;
+    border: 1px solid #888888;
+    padding: 0px;
+}
+td.dep {
+    background: #cccccc;
     border: 1px solid #888888;
     padding: 0px;
 }
@@ -76,6 +81,9 @@ tr.not-on-live {
 tr.live-only {
     color: #aaaaaa;
 }
+tr.build {
+    font-weight: bold;
+}
 .hidden {
     display: none;
 }