diff --git a/README.md b/README.md index 7680a6b..c339019 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,25 @@ The archive is generated in the rpm package directory, and can be committed to for [openSUSE](https://www.opensuse.org), [SUSE](https://www.suse.com), and numerous other distributions. +Upon request, `obs-service-go_modules` will update module versions in the +`go.mod` file before downloading the vendor modules, regenerate `go.sum` +file, update the `.changes` file by adding any missing version updates and +create a patch file for these that can be applied when building the package. +This feature makes it easy to apply security fixes by updating the vendor +module list before downloading the modules. + +Taking the version information from a YAML file `obs-service-go_modules` will +pass every version update to `go get`: + +``` +go get @ +``` + +one-by-one to the unpacked source package before downloading the vendor +modules. This command will update the package entry and recalculate the +checksum. + + ## Usage for packagers Presently it is assumed the Go application source is distributed as a compressed tarball named @@ -116,6 +135,102 @@ The `zstd` format has a clear advantage in speed with reasonable compression rat and is likely to become the default compression method in a future release. Decompression timings are closely matched among `gz`, `xz`, and `zstd` compression methods. +## Update Vendor Module Versions + +Optionally, `obs-service-go_modules` will update module versions in `go.mod`, +recreate `go.sum`, update the `*.changes` file with any change that's missing +and create a patch that can be applied when building the package. The version +changes to be applied need to be specified in a YAML file. + +### Service Parameters + +To do so, specify the name of the YAML file using the service parameter +`modupdates`. The default is `none` in which case no version update will be +performed. You can specify the name of the patch file to generate using the +service parameter `moddiff`. The default is `go-modules.patch`. If multiple +`.changes` files exist, missing comments will be added to all files. To apply +this update to a single `.changes` file, you may use the parameter +`changesfile`. This parameter can be provided multiple times. + +``` + + + go-module-updates.yaml + go-module.patch + mypackage.changes + + +``` + +### YAML File + +The YAML file has the following format: + +``` +go_module: + module_updates: + - uri: @ + comment: +``` + +The module information is represented as a list of associative arrays where +each array represents a single change so multiple changes can be applied. The +value to the `uri:` key in each array represents the module/version +information that is passed to `go get` verbatim. + +The optional `comment:` key represents additional information that will help +to determine the purpose of the version update which will be used to generate +an entry for the `.changes` file. If no comment is specified, a default comment +will be used. + +### Example + +Using the [apptainer](https://github.com/apptainer/apptainer) HPC container +platform as an example to demonstrate module version updates the `_service` +file: + +``` + + + go-module-updates.yaml + go-module.patch + + + +``` + +together with the YAML file `go-module-updates.yaml` (as specified in the +`_service` file): + +``` +go_module: + module_updates: + - uri: golang.org/x/net@v0.23.0 + comment: "This prevents an attacker from sending an arbitrary amount of\n +http/2 header data CVE-2023-45288 (bnc#1236527)." +``` + +produce: +1. a file `go-module.patch` +2. an entry in the file `apptainer.changes`: + +``` +------------------------------------------------------------------- +Wed Jan 29 09:47:49 UTC 2025 - Egbert Eich + +- Update vendor module dependencies: + * Update golang.org/x/net to v0.23: + This prevents an attacker from sending an arbitrary amount of + http/2 header data CVE-2023-45288 (bnc#1236527). + * Update go-module.patch. +``` + +3. a `vendor.tar.gz` file with updated vendor modules. + +Note, that the `moddiff` parameter in the service file is set to the default +value and thus could be omitted. + + ## OBS Source Service Build Mode support OBS Source Services can run in one of several modes as shown in diff --git a/go_modules b/go_modules index 828b9a7..e1191b4 100755 --- a/go_modules +++ b/go_modules @@ -27,6 +27,7 @@ vendor/ directory populated by go mod vendor. See README.md for additional documentation. """ +import difflib import logging import argparse import re @@ -34,6 +35,11 @@ import libarchive import os import sys import tempfile +import yaml +import glob +import subprocess +from osc import conf +from osc import core from pathlib import Path from subprocess import run @@ -44,6 +50,7 @@ description = __doc__ DEFAULT_COMPRESSION = "gz" DEFAULT_VENDOR_STEM = "vendor" +DEFAULT_MOD_DIFF = "go-modules.patch" def get_archive_parameters(args): @@ -115,6 +122,8 @@ def archive_autodetect(): - .tar.lz - .tar.xz - .tar.zst + - .obscpio + Returns str with filename of the archive or subdirectory """ log.info("Autodetecting archive since no archive param provided in _service") specs = sorted(Path.cwd().glob("*.spec"), reverse=True) @@ -195,7 +204,8 @@ def cmd_go_mod(cmd, moddir): else: cp = run(["go", "mod", cmd], cwd=moddir) if cp.returncode: - log.error(cp.stderr.strip()) + if cp.stderr is not None: + log.error(cp.stderr.strip()) return cp @@ -207,7 +217,22 @@ def sanitize_subdir(basedir, subdir): exit(1) +def match_comment(string, changesfile): + lines = string.splitlines() + index = 0 + with open(changesfile) as f: + for line_f in f.readlines(): + if lines[index] in line_f: + index += 1 + if index == len(lines): + return True + else: + index = 0 + return False + + def main(): + mod_updates = {} log.info(f"Running OBS Source Service: {app_name}") parser = argparse.ArgumentParser( @@ -220,13 +245,15 @@ def main(): parser.add_argument("--basename") parser.add_argument("--vendorname", default=DEFAULT_VENDOR_STEM) parser.add_argument("--subdir") + parser.add_argument("--modupdates") + parser.add_argument("--moddiff", default=DEFAULT_MOD_DIFF) + parser.add_argument("--changesfile", action="append") args = parser.parse_args() outdir = args.outdir subdir = args.subdir archive_args = get_archive_parameters(args) - vendor_tarname = f"{archive_args['vendorname']}.{archive_args['ext']}" if args.archive: archive_matches = sorted(Path.cwd().glob(args.archive), reverse=True) if not archive_matches: @@ -237,6 +264,16 @@ def main(): archive = archive_autodetect() log.info(f"Using archive {archive}") + if args.modupdates: + modupdatepath = sanitize_subdir( + os.getcwd(), os.path.join(os.getcwd(), args.modupdates) + ) + if os.path.exists(modupdatepath): + with open(modupdatepath, "r") as f: + mod_updates = yaml.safe_load(f) + + moddiff = sanitize_subdir(outdir, os.path.join(outdir, args.moddiff)) + with tempfile.TemporaryDirectory() as tempdir: extract(archive, tempdir) @@ -246,13 +283,10 @@ def main(): or basename_from_archive_name(archive) ) if subdir: - go_mod_path = sanitize_subdir( - tempdir, os.path.join(tempdir, basename, subdir, "go.mod") - ) + basedir = os.path.join(basename, subdir) else: - go_mod_path = sanitize_subdir( - tempdir, os.path.join(tempdir, basename, "go.mod") - ) + basedir = basename + go_mod_path = sanitize_subdir(tempdir, os.path.join(tempdir, basedir, "go.mod")) if go_mod_path and os.path.exists(go_mod_path): go_mod_dir = os.path.dirname(go_mod_path) log.info(f"Using go.mod found at {go_mod_path}") @@ -260,6 +294,83 @@ def main(): log.error(f"File go.mod not found under {os.path.join(tempdir, basename)}") exit(1) + if ( + mod_updates + and "go_module" in mod_updates + and "module_updates" in mod_updates["go_module"] + ): + new_comments = [] + flines = {} + if args.changesfile: + changesfiles = args.changesfile + else: + changesfiles = glob.glob("./*.changes") + log.info(f"Found changesfiles: {changesfiles}") + for _cf in changesfiles: + new_comments.append([]) + for file in ["go.mod", "go.sum"]: + with open(os.path.join(go_mod_dir, file), "r") as f: + flines[file] = f.readlines() + if not mod_updates["go_module"]["module_updates"] is None: + for modupdate in mod_updates["go_module"]["module_updates"]: + log.info(f"Processing: `go get \"{modupdate['uri']}\"`") + if ( + re.fullmatch( + r"[a-zA-Z0-9\~\-_.:/]+(@[a-zA-Z0-9\+\~\-_.:/]+){0,1}", + modupdate["uri"], + ) + is None + ): + log.error( + f"uri: {modupdate['uri']} contains invalid characters" + ) + exit(1) + if sys.version_info >= (3, 7): + cp = run( + ["go", "get", f"{modupdate['uri']}"], + cwd=go_mod_dir, + capture_output=True, + text=True, + ) + else: + cp = run(["go", "get", f"{modupdate['uri']}"], cwd=go_mod_dir) + if cp.returncode: + if cp.stderr is not None: + log.error(cp.stderr.strip()) + exit(1) + s_tmp = modupdate["uri"].split("@", 1) + if len(s_tmp) > 1: + comment = comment_0 = f"Update {s_tmp[0]} to version {s_tmp[1]}" + else: + comment = comment_0 = f"Update {s_tmp[0]}" + if "comment" in modupdate: + comment += ":\n" + modupdate["comment"] + for i in range(len(changesfiles)): + cf = changesfiles[i] + nc = new_comments[i] + if not match_comment(comment_0, cf): + nc.append(comment) + log.info(f"adding comment: {comment}") + else: + log.info(f"comment {comment_0} matched") + # Sanitze after updating modules + cp = cmd_go_mod("tidy", go_mod_dir) + if cp.returncode: + log.error("go mod tidy failed") + exit(1) + + with open(moddiff, "w") as diff_f: + for file in ["go.mod", "go.sum"]: + with open(os.path.join(go_mod_dir, file), "r") as f: + tlines = f.readlines() + diff = difflib.unified_diff( + flines[file], + tlines, + os.path.join("a", file), + os.path.join("b", file), + ) + diff_f.writelines(diff) + if args.strategy == "vendor": # go subcommand sequence: # - go mod download @@ -293,6 +404,7 @@ def main(): log.error("go mod verify failed") exit(1) + vendor_tarname = f"{archive_args['vendorname']}.{archive_args['ext']}" log.info(f"Vendor go.mod dependencies to {vendor_tarname}") vendor_tarfile = os.path.join(outdir, vendor_tarname) cwd = os.getcwd() @@ -313,9 +425,48 @@ def main(): archive_args["compression"], options=",".join(options), ) as new_archive: - new_archive.add_files(vendor_dir, mtime=mtime, ctime=mtime, atime=mtime) + try: + new_archive.add_files( + vendor_dir, mtime=mtime, ctime=mtime, atime=mtime + ) + except ( + TypeError + ): # If using old libarchive fallback to old non reproducible behavior + log.warning( + "python libarchive is too old, unable to produce reproducible output" + ) + new_archive.add_files(vendor_dir) os.chdir(cwd) + if args.modupdates: + for i in range(len(changesfiles)): + cf = changesfiles[i] + nc = new_comments[i] + if nc: + if args.moddiff: + nc.append(f"Update {args.moddiff}.") + cmd_list = [conf.Options()["vc-cmd"]] + if core.which(cmd_list[0]) is None: + log.error(f"vc ('{cmd_list[0]}') command not found") + exit(1) + cmd_list.append("-m") + argstr = "" + prefix = "" + argstr = "Update vendor module dependencies:" + for comment in nc: + argstr += "\n" + log.info(f"Write comment {comment}") + prefix = " * " + for line in comment.splitlines(keepends=True): + argstr += prefix + line + prefix = " " + log.info(f"adding comment: {argstr} to {cf}") + cmd_list.append(argstr) + cmd_list.append(cf) + vc = subprocess.Popen(cmd_list) + vc.wait() + log.info(f"{cmd_list[0]} returned {vc.returncode}") + if __name__ == "__main__": logging.basicConfig(level=logging.DEBUG) diff --git a/go_modules.service b/go_modules.service index f2edd20..e9e03ca 100644 --- a/go_modules.service +++ b/go_modules.service @@ -19,4 +19,10 @@ Specify the name for the generated vendor tarball. Default: vendor + + YAML file listing module updates. Default: none + + + Name of the patch for go.mod/sum to write to outdir. Default: go-module.patch +