Add `make source_archive_complete` target

Add a `source_archive_complete` target for `make` that creates a source
archive including the source packages of Blender's dependencies.

This expands `make_source_archive.py` to include files from
`${BUILD_DIR}/source_archive/packages/` as well.

Reviewed By: dfelinto

Maniphest Tasks: T86124

Differential Revision: https://developer.blender.org/D10727

GNUmakefile

@@ -128,6 +128,9 @@ Utilities
    * source_archive:
      Create a compressed archive of the source code.
 
+   * source_archive_complete:
+     Create a compressed archive of the source code and all the libraries of dependencies.
+
    * update:
      Updates git and all submodules and svn.
 
@@ -511,6 +514,13 @@ check_descriptions: .FORCE
 source_archive: .FORCE
 	python3 ./build_files/utils/make_source_archive.py
 
+source_archive_complete: .FORCE
+	cmake -S "$(BLENDER_DIR)/build_files/build_environment" -B"$(BUILD_DIR)/source_archive" \
+		-DCMAKE_BUILD_TYPE_INIT:STRING=$(BUILD_TYPE) -DPACKAGE_USE_UPSTREAM_SOURCES=OFF
+# This assumes CMake is still using a default `PACKAGE_DIR` variable:
+	python3 ./build_files/utils/make_source_archive.py --include-packages "$(BUILD_DIR)/source_archive/packages"
+
+
 INKSCAPE_BIN?="inkscape"
 icons: .FORCE
 	BLENDER_BIN=$(BLENDER_BIN) INKSCAPE_BIN=$(INKSCAPE_BIN) \

build_files/build_environment/cmake/options.cmake

@@ -37,14 +37,8 @@ else(BUILD_MODE STREQUAL "Debug")
 endif()
 
 set(DOWNLOAD_DIR "${CMAKE_CURRENT_BINARY_DIR}/downloads" CACHE STRING "Path for downloaded files")
-# look in blenders source folder for packages directory, if that exists
-# it will our package folder, otherwise it will be in the build folder
-if(EXISTS "${CMAKE_SOURCE_DIR}/../../packages")
-  set(PACKAGE_DIR_DEFAULT "${CMAKE_SOURCE_DIR}/../../packages")
-else()
-  set(PACKAGE_DIR_DEFAULT "${CMAKE_CURRENT_BINARY_DIR}/packages")
-endif()
-set(PACKAGE_DIR ${PACKAGE_DIR_DEFAULT} CACHE STRING "Path for downloaded source files")
+# This path must be hard-coded like this, so that the GNUmakefile knows where it is and can pass it to make_source_archive.py:
+set(PACKAGE_DIR "${CMAKE_CURRENT_BINARY_DIR}/packages")
 option(PACKAGE_USE_UPSTREAM_SOURCES "Use soures upstream to download the package sources, when OFF the blender mirror will be used" ON)
 
 file(TO_CMAKE_PATH ${DOWNLOAD_DIR} DOWNLOAD_DIR)

build_files/utils/make_source_archive.py

 #!/usr/bin/env python3
 
+import argparse
 import dataclasses
 import os
 import re
 import subprocess
 from pathlib import Path
-from typing import Iterable, TextIO
+from typing import Iterable, TextIO, Optional, Any
 
 # This script can run from any location,
 # output is created in the $CWD
@@ -18,21 +19,43 @@ SKIP_NAMES = {
     ".gitignore",
     ".gitmodules",
     ".arcconfig",
+    ".svn",
 }
 
 
 def main() -> None:
-    output_dir = Path(".").absolute()
     blender_srcdir = Path(__file__).absolute().parent.parent.parent
+
+    cli_parser = argparse.ArgumentParser(
+        description=f"Create a tarball of the Blender sources, optionally including sources of dependencies.",
+        epilog="This script is intended to be run by `make source_archive_complete`.",
+    )
+    cli_parser.add_argument(
+        "-p",
+        "--include-packages",
+        type=Path,
+        default=None,
+        metavar="PACKAGE_PATH",
+        help="Include all source files from the given package directory as well.",
+    )
+
+    cli_args = cli_parser.parse_args()
+
     print(f"Source dir: {blender_srcdir}")
 
+    curdir = blender_srcdir.parent
+    os.chdir(curdir)
+    blender_srcdir = blender_srcdir.relative_to(curdir)
+
+    print(f"Output dir: {curdir}")
+
     version = parse_blender_version(blender_srcdir)
-    manifest = output_dir / f"blender-{version}-manifest.txt"
-    tarball = output_dir / f"blender-{version}.tar.xz"
+    tarball = tarball_path(curdir, version, cli_args)
+    manifest = manifest_path(tarball)
+    packages_dir = packages_path(curdir, cli_args)
 
-    os.chdir(blender_srcdir)
-    create_manifest(version, manifest)
-    create_tarball(version, tarball, manifest)
+    create_manifest(version, manifest, blender_srcdir, packages_dir)
+    create_tarball(version, tarball, manifest, blender_srcdir, packages_dir)
     create_checksum_file(tarball)
     cleanup(manifest)
     print("Done!")
@@ -84,43 +107,109 @@ def parse_blender_version(blender_srcdir: Path) -> BlenderVersion:
     )
 
 
+def tarball_path(output_dir: Path, version: BlenderVersion, cli_args: Any) -> Path:
+    extra = ""
+    if cli_args.include_packages:
+        extra = "-with-libraries"
+
+    return output_dir / f"blender{extra}-{version}.tar.xz"
+
+
+def manifest_path(tarball: Path) -> Path:
+    """Return the manifest path for the given tarball path.
+
+    >>> from pathlib import Path
+    >>> tarball = Path("/home/sybren/workspace/blender-git/blender-test.tar.gz")
+    >>> manifest_path(tarball).as_posix()
+    '/home/sybren/workspace/blender-git/blender-test-manifest.txt'
+    """
+    # ".tar.gz" is seen as two suffixes.
+    without_suffix = tarball.with_suffix("").with_suffix("")
+    name = without_suffix.name
+    return without_suffix.with_name(f"{name}-manifest.txt")
+
+
+def packages_path(current_directory: Path, cli_args: Any) -> Optional[Path]:
+    if not cli_args.include_packages:
+        return None
+
+    abspath = cli_args.include_packages.absolute()
+
+    # os.path.relpath() can return paths like "../../packages", where
+    # Path.relative_to() will not go up directories (so its return value never
+    # has "../" in there).
+    relpath = os.path.relpath(abspath, current_directory)
+
+    return Path(relpath)
+
+
 ### Manifest creation
 
 
-def create_manifest(version: BlenderVersion, outpath: Path) -> None:
+def create_manifest(
+    version: BlenderVersion,
+    outpath: Path,
+    blender_srcdir: Path,
+    packages_dir: Optional[Path],
+) -> None:
     print(f'Building manifest of files:  "{outpath}"...', end="", flush=True)
     with outpath.open("w", encoding="utf-8") as outfile:
-        main_files_to_manifest(outfile)
-        submodules_to_manifest(version, outfile)
+        main_files_to_manifest(blender_srcdir, outfile)
+        submodules_to_manifest(blender_srcdir, version, outfile)
+
+        if packages_dir:
+            packages_to_manifest(outfile, packages_dir)
     print("OK")
 
 
-def main_files_to_manifest(outfile: TextIO) -> None:
-    for path in git_ls_files():
+def main_files_to_manifest(blender_srcdir: Path, outfile: TextIO) -> None:
+    assert not blender_srcdir.is_absolute()
+    for path in git_ls_files(blender_srcdir):
         print(path, file=outfile)
 
 
-def submodules_to_manifest(version: BlenderVersion, outfile: TextIO) -> None:
+def submodules_to_manifest(
+    blender_srcdir: Path, version: BlenderVersion, outfile: TextIO
+) -> None:
     skip_addon_contrib = version.is_release
+    assert not blender_srcdir.is_absolute()
 
-    for line in git_command("submodule"):
+    for line in git_command("-C", blender_srcdir, "submodule"):
         submodule = line.split()[1]
 
         # Don't use native slashes as GIT for MS-Windows outputs forward slashes.
         if skip_addon_contrib and submodule == "release/scripts/addons_contrib":
             continue
 
-        for path in git_ls_files(Path(submodule)):
+        for path in git_ls_files(blender_srcdir / submodule):
             print(path, file=outfile)
 
 
-def create_tarball(version: BlenderVersion, tarball: Path, manifest: Path) -> None:
+def packages_to_manifest(outfile: TextIO, packages_dir: Path) -> None:
+    for path in packages_dir.glob("*"):
+        if not path.is_file():
+            continue
+        if path.name in SKIP_NAMES:
+            continue
+        print(path, file=outfile)
+
+
+### Higher-level functions
+
+
+def create_tarball(
+    version: BlenderVersion, tarball: Path, manifest: Path, blender_srcdir: Path, packages_dir: Optional[Path]
+) -> None:
     print(f'Creating archive:            "{tarball}" ...', end="", flush=True)
+    command = ["tar"]
+
     # Requires GNU `tar`, since `--transform` is used.
-    command = [
-        "tar",
+    if packages_dir:
+        command += ["--transform", f"s,{packages_dir}/,packages/,g"]
+
+    command += [
         "--transform",
-        f"s,^,blender-{version}/,g",
+        f"s,^{blender_srcdir.name}/,blender-{version}/,g",
         "--use-compress-program=xz -9",
         "--create",
         f"--file={tarball}",
@@ -130,7 +219,8 @@ def create_tarball(version: BlenderVersion, tarball: Path, manifest: Path) -> No
         "--owner=0",
         "--group=0",
     ]
-    subprocess.run(command, check=True, timeout=300)
+
+    subprocess.run(command, check=True, timeout=3600)
     print("OK")