Merge tag 'android-14.0.0_r50' of https://android.googlesource.com/platform/build/soong into u14.0

Android 14.0.0 Release 50 (AP2A.240605.024)

15 files changed, 136 insertions, 74 deletions

diff --git a/ui/build/Android.bp b/ui/build/Android.bp
index 21453ba95..ee286f68a 100644
--- a/ui/build/Android.bp
+++ b/ui/build/Android.bp
@@ -35,6 +35,7 @@ bootstrap_go_package {
         "blueprint",
         "blueprint-bootstrap",
         "blueprint-microfactory",
+        "soong-android",
         "soong-finder",
         "soong-remoteexec",
         "soong-shared",
@@ -46,6 +47,7 @@ bootstrap_go_package {
         "soong-ui-tracer",
     ],
     srcs: [
+        "androidmk_denylist.go",
         "build.go",
         "cleanbuild.go",
         "config.go",
diff --git a/ui/build/androidmk_denylist.go b/ui/build/androidmk_denylist.go
new file mode 100644
index 000000000..e7896ab29
--- /dev/null
+++ b/ui/build/androidmk_denylist.go
@@ -0,0 +1,46 @@
+// Copyright 2024 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"strings"
+)
+
+var androidmk_denylist []string = []string{
+	"chained_build_config/",
+	"cts/",
+	"dalvik/",
+	"developers/",
+	// Do not block other directories in kernel/, see b/319658303.
+	"kernel/configs/",
+	"kernel/prebuilts/",
+	"kernel/tests/",
+	"libcore/",
+	"libnativehelper/",
+	"pdk/",
+	// Add back toolchain/ once defensive Android.mk files are removed
+	//"toolchain/",
+}
+
+func blockAndroidMks(ctx Context, androidMks []string) {
+	for _, mkFile := range androidMks {
+		for _, d := range androidmk_denylist {
+			if strings.HasPrefix(mkFile, d) {
+				ctx.Fatalf("Found blocked Android.mk file: %s. "+
+					"Please see androidmk_denylist.go for the blocked directories and contact build system team if the file should not be blocked.", mkFile)
+			}
+		}
+	}
+}
diff --git a/ui/build/config.go b/ui/build/config.go
index 5085c6845..1a2539793 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -75,7 +75,6 @@ type configImpl struct {
 	queryview                bool
 	reportMkMetrics          bool // Collect and report mk2bp migration progress metrics.
 	soongDocs                bool
-	multitreeBuild           bool // This is a multitree build.
 	skipConfig               bool
 	skipKati                 bool
 	skipKatiNinja            bool
@@ -115,6 +114,11 @@ type configImpl struct {
 
 	// Data source to write ninja weight list
 	ninjaWeightListSource NinjaWeightListSource
+
+	// This file is a detailed dump of all soong-defined modules for debugging purposes.
+	// There's quite a bit of overlap with module-info.json and soong module graph. We
+	// could consider merging them.
+	moduleDebugFile string
 }
 
 type NinjaWeightListSource uint
@@ -273,6 +277,10 @@ func NewConfig(ctx Context, args ...string) Config {
 		ret.sandboxConfig.SetSrcDirIsRO(srcDirIsWritable == "false")
 	}
 
+	if os.Getenv("GENERATE_SOONG_DEBUG") == "true" {
+		ret.moduleDebugFile, _ = filepath.Abs(shared.JoinPath(ret.SoongOutDir(), "soong-debug-info.json"))
+	}
+
 	ret.environ.Unset(
 		// We're already using it
 		"USE_SOONG_UI",
@@ -325,6 +333,9 @@ func NewConfig(ctx Context, args ...string) Config {
 		"ANDROID_DEV_SCRIPTS",
 		"ANDROID_EMULATOR_PREBUILTS",
 		"ANDROID_PRE_BUILD_PATHS",
+
+		// We read it here already, don't let others share in the fun
+		"GENERATE_SOONG_DEBUG",
 	)
 
 	if ret.UseGoma() || ret.ForceUseGoma() {
@@ -412,10 +423,6 @@ func NewConfig(ctx Context, args ...string) Config {
 	// zip files produced by soong_zip.  Disable zipbomb detection.
 	ret.environ.Set("UNZIP_DISABLE_ZIPBOMB_DETECTION", "TRUE")
 
-	if ret.MultitreeBuild() {
-		ret.environ.Set("MULTITREE_BUILD", "true")
-	}
-
 	outDir := ret.OutDir()
 	buildDateTimeFile := filepath.Join(outDir, "build_date.txt")
 	if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" {
@@ -777,8 +784,6 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
 			c.skipMetricsUpload = true
 		} else if arg == "--mk-metrics" {
 			c.reportMkMetrics = true
-		} else if arg == "--multitree-build" {
-			c.multitreeBuild = true
 		} else if arg == "--search-api-dir" {
 			c.searchApiDir = true
 		} else if strings.HasPrefix(arg, "--ninja_weight_source=") {
@@ -1083,10 +1088,6 @@ func (c *configImpl) IsVerbose() bool {
 	return c.verbose
 }
 
-func (c *configImpl) MultitreeBuild() bool {
-	return c.multitreeBuild
-}
-
 func (c *configImpl) NinjaWeightListSource() NinjaWeightListSource {
 	return c.ninjaWeightListSource
 }
diff --git a/ui/build/config_test.go b/ui/build/config_test.go
index 5182b1226..b1222fe93 100644
--- a/ui/build/config_test.go
+++ b/ui/build/config_test.go
@@ -860,23 +860,24 @@ func TestGetConfigArgsBuildModules(t *testing.T) {
 }
 
 func TestGetConfigArgsBuildModulesInDirectory(t *testing.T) {
-	tests := []buildActionTestCase{{
-		description:  "normal execution in a directory",
-		dirsInTrees:  []string{"0/1/2"},
-		buildFiles:   []string{"0/1/2/Android.mk"},
-		args:         []string{"fake-module"},
-		curDir:       "0/1/2",
-		tidyOnly:     "",
-		expectedArgs: []string{"fake-module", "MODULES-IN-0-1-2"},
-	}, {
-		description:  "build file in parent directory",
-		dirsInTrees:  []string{"0/1/2"},
-		buildFiles:   []string{"0/1/Android.mk"},
-		args:         []string{},
-		curDir:       "0/1/2",
-		tidyOnly:     "",
-		expectedArgs: []string{"MODULES-IN-0-1"},
-	},
+	tests := []buildActionTestCase{
+		{
+			description:  "normal execution in a directory",
+			dirsInTrees:  []string{"0/1/2"},
+			buildFiles:   []string{"0/1/2/Android.mk"},
+			args:         []string{"fake-module"},
+			curDir:       "0/1/2",
+			tidyOnly:     "",
+			expectedArgs: []string{"fake-module", "MODULES-IN-0-1-2"},
+		}, {
+			description:  "build file in parent directory",
+			dirsInTrees:  []string{"0/1/2"},
+			buildFiles:   []string{"0/1/Android.mk"},
+			args:         []string{},
+			curDir:       "0/1/2",
+			tidyOnly:     "",
+			expectedArgs: []string{"MODULES-IN-0-1"},
+		},
 		{
 			description:  "build file in parent directory, multiple module names passed in",
 			dirsInTrees:  []string{"0/1/2"},
@@ -903,15 +904,6 @@ func TestGetConfigArgsBuildModulesInDirectory(t *testing.T) {
 			tidyOnly:     "",
 			expectedArgs: []string{},
 		}, {
-			description:  "multitree build action executed at root directory",
-			dirsInTrees:  []string{},
-			buildFiles:   []string{},
-			rootSymlink:  false,
-			args:         []string{"--multitree-build"},
-			curDir:       ".",
-			tidyOnly:     "",
-			expectedArgs: []string{"--multitree-build"},
-		}, {
 			description:  "build action executed at root directory in symlink",
 			dirsInTrees:  []string{},
 			buildFiles:   []string{},
@@ -953,7 +945,8 @@ func TestGetConfigArgsBuildModulesInDirectory(t *testing.T) {
 			curDir:       "",
 			tidyOnly:     "",
 			expectedArgs: []string{"-j", "-k", "fake_module"},
-		}}
+		},
+	}
 	for _, tt := range tests {
 		t.Run("build action BUILD_MODULES_IN_DIR, "+tt.description, func(t *testing.T) {
 			testGetConfigArgs(t, tt, BUILD_MODULES_IN_A_DIRECTORY)
diff --git a/ui/build/dumpvars.go b/ui/build/dumpvars.go
index c608cdcc8..7e5fca71d 100644
--- a/ui/build/dumpvars.go
+++ b/ui/build/dumpvars.go
@@ -245,7 +245,6 @@ func runMakeProductConfig(ctx Context, config Config) {
 		"BUILD_BROKEN_SRC_DIR_RW_ALLOWLIST",
 
 		// Not used, but useful to be in the soong.log
-		"BOARD_VNDK_VERSION",
 		"TARGET_BUILD_TYPE",
 		"HOST_ARCH",
 		"HOST_2ND_ARCH",
diff --git a/ui/build/finder.go b/ui/build/finder.go
index d0bcf4060..573df21d9 100644
--- a/ui/build/finder.go
+++ b/ui/build/finder.go
@@ -128,6 +128,7 @@ func FindSources(ctx Context, config Config, f *finder.Finder) {
 
 	// Stop searching a subdirectory recursively after finding an Android.mk.
 	androidMks := f.FindFirstNamedAt(".", "Android.mk")
+	blockAndroidMks(ctx, androidMks)
 	err := dumpListToFile(ctx, config, androidMks, filepath.Join(dumpDir, "Android.mk.list"))
 	if err != nil {
 		ctx.Fatalf("Could not export module list: %v", err)
diff --git a/ui/build/kati.go b/ui/build/kati.go
index 31e744029..d599c99a7 100644
--- a/ui/build/kati.go
+++ b/ui/build/kati.go
@@ -100,8 +100,6 @@ func runKati(ctx Context, config Config, extraSuffix string, args []string, envF
 		"--no_ninja_prelude",
 		// Support declaring phony outputs in AOSP Ninja.
 		"--use_ninja_phony_output",
-		// Support declaring symlink outputs in AOSP Ninja.
-		"--use_ninja_symlink_outputs",
 		// Regenerate the Ninja file if environment inputs have changed. e.g.
 		// CLI flags, .mk file timestamps, env vars, $(wildcard ..) and some
 		// $(shell ..) results.
diff --git a/ui/build/ninja.go b/ui/build/ninja.go
index b69e938f8..551b8ab41 100644
--- a/ui/build/ninja.go
+++ b/ui/build/ninja.go
@@ -271,11 +271,13 @@ func (c *ninjaStucknessChecker) check(ctx Context, config Config) {
 		// The Ninja file hasn't been modified since the last time it was
 		// checked, so Ninja could be stuck. Output some diagnostics.
 		ctx.Verbosef("ninja may be stuck; last update to %v was %v. dumping process tree...", c.logPath, newModTime)
+		ctx.Printf("ninja may be stuck, check %v for list of running processes.",
+			filepath.Join(config.LogsDir(), config.logsPrefix+"soong.log"))
 
 		// The "pstree" command doesn't exist on Mac, but "pstree" on Linux
 		// gives more convenient output than "ps" So, we try pstree first, and
 		// ps second
-		commandText := fmt.Sprintf("pstree -pal %v || ps -ef", os.Getpid())
+		commandText := fmt.Sprintf("pstree -palT %v || ps -ef", os.Getpid())
 
 		cmd := Command(ctx, config, "dump process tree", "bash", "-c", commandText)
 		output := cmd.CombinedOutputOrFatal()
diff --git a/ui/build/path.go b/ui/build/path.go
index 29128d81f..51ebff117 100644
--- a/ui/build/path.go
+++ b/ui/build/path.go
@@ -180,7 +180,7 @@ func SetupPath(ctx Context, config Config) {
 			// Compute the error message along with the process tree, including
 			// parents, for this log line.
 			procPrints := []string{
-				"See https://android.googlesource.com/platform/build/+/master/Changes.md#PATH_Tools for more information.",
+				"See https://android.googlesource.com/platform/build/+/main/Changes.md#PATH_Tools for more information.",
 			}
 			if len(log.Parents) > 0 {
 				procPrints = append(procPrints, "Process tree:")
diff --git a/ui/build/paths/config.go b/ui/build/paths/config.go
index d408f63f1..4095864d8 100644
--- a/ui/build/paths/config.go
+++ b/ui/build/paths/config.go
@@ -93,6 +93,7 @@ var Configuration = map[string]PathConfig{
 	"fuser":          Allowed,
 	"gcert":          Allowed,
 	"gcertstatus":    Allowed,
+	"gcloud":         Allowed,
 	"getopt":         Allowed,
 	"git":            Allowed,
 	"hexdump":        Allowed,
@@ -101,7 +102,6 @@ var Configuration = map[string]PathConfig{
 	"javap":          Allowed,
 	"lsof":           Allowed,
 	"openssl":        Allowed,
-	"prodcertstatus": Allowed,
 	"pstree":         Allowed,
 	"rsync":          Allowed,
 	"sh":             Allowed,
diff --git a/ui/build/soong.go b/ui/build/soong.go
index 0bf886205..79584c66f 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -36,6 +36,7 @@ import (
 	"github.com/google/blueprint"
 	"github.com/google/blueprint/bootstrap"
 	"github.com/google/blueprint/microfactory"
+	"github.com/google/blueprint/pathtools"
 
 	"google.golang.org/protobuf/proto"
 )
@@ -181,20 +182,30 @@ func getGlobPathName(config Config) string {
 	return globPathName
 }
 
-func (pb PrimaryBuilderFactory) primaryBuilderInvocation() bootstrap.PrimaryBuilderInvocation {
+func getGlobPathNameFromPrimaryBuilderFactory(config Config, pb PrimaryBuilderFactory) string {
+	if pb.name == soongBuildTag {
+		// Glob path for soong build would be separated per product target
+		return getGlobPathName(config)
+	}
+	return pb.name
+}
+
+func (pb PrimaryBuilderFactory) primaryBuilderInvocation(config Config) bootstrap.PrimaryBuilderInvocation {
 	commonArgs := make([]string, 0, 0)
 
 	if !pb.config.skipSoongTests {
 		commonArgs = append(commonArgs, "-t")
 	}
 
-	if pb.config.multitreeBuild {
-		commonArgs = append(commonArgs, "--multitree-build")
-	}
 	if pb.config.buildFromSourceStub {
 		commonArgs = append(commonArgs, "--build-from-source-stub")
 	}
 
+	if pb.config.moduleDebugFile != "" {
+		commonArgs = append(commonArgs, "--soong_module_debug")
+		commonArgs = append(commonArgs, pb.config.moduleDebugFile)
+	}
+
 	commonArgs = append(commonArgs, "-l", filepath.Join(pb.config.FileListDir(), "Android.bp.list"))
 	invocationEnv := make(map[string]string)
 	if pb.debugPort != "" {
@@ -215,11 +226,7 @@ func (pb PrimaryBuilderFactory) primaryBuilderInvocation() bootstrap.PrimaryBuil
 
 	var allArgs []string
 	allArgs = append(allArgs, pb.specificArgs...)
-	globPathName := pb.name
-	// Glob path for soong build would be separated per product target
-	if pb.name == soongBuildTag {
-		globPathName = getGlobPathName(pb.config)
-	}
+	globPathName := getGlobPathNameFromPrimaryBuilderFactory(config, pb)
 	allArgs = append(allArgs,
 		"--globListDir", globPathName,
 		"--globFile", pb.config.NamedGlobFile(globPathName))
@@ -234,8 +241,11 @@ func (pb PrimaryBuilderFactory) primaryBuilderInvocation() bootstrap.PrimaryBuil
 	}
 	allArgs = append(allArgs, "Android.bp")
 
+	globfiles := bootstrap.GlobFileListFiles(bootstrap.GlobDirectory(config.SoongOutDir(), globPathName))
+
 	return bootstrap.PrimaryBuilderInvocation{
 		Inputs:      []string{"Android.bp"},
+		Implicits:   globfiles,
 		Outputs:     []string{pb.output},
 		Args:        allArgs,
 		Description: pb.description,
@@ -292,9 +302,6 @@ func bootstrapBlueprint(ctx Context, config Config) {
 	if config.EmptyNinjaFile() {
 		mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--empty-ninja-file")
 	}
-	if config.MultitreeBuild() {
-		mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--multitree-build")
-	}
 	if config.buildFromSourceStub {
 		mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--build-from-source-stub")
 	}
@@ -376,17 +383,10 @@ func bootstrapBlueprint(ctx Context, config Config) {
 		if debuggedInvocations[pbf.name] {
 			pbf.debugPort = delvePort
 		}
-		pbi := pbf.primaryBuilderInvocation()
+		pbi := pbf.primaryBuilderInvocation(config)
 		invocations = append(invocations, pbi)
 	}
 
-	// The glob .ninja files are subninja'd. However, they are generated during
-	// the build itself so we write an empty file if the file does not exist yet
-	// so that the subninja doesn't fail on clean builds
-	for _, globFile := range bootstrapGlobFileList(config) {
-		writeEmptyFile(ctx, globFile)
-	}
-
 	blueprintArgs := bootstrap.Args{
 		ModuleListFile: filepath.Join(config.FileListDir(), "Android.bp.list"),
 		OutFile:        shared.JoinPath(config.SoongOutDir(), "bootstrap.ninja"),
@@ -408,6 +408,28 @@ func bootstrapBlueprint(ctx Context, config Config) {
 		primaryBuilderInvocations: invocations,
 	}
 
+	// The glob ninja files are generated during the main build phase. However, the
+	// primary buildifer invocation depends on all of its glob files, even before
+	// it's been run. Generate a "empty" glob ninja file on the first run,
+	// so that the files can be there to satisfy the dependency.
+	for _, pb := range pbfs {
+		globPathName := getGlobPathNameFromPrimaryBuilderFactory(config, pb)
+		globNinjaFile := config.NamedGlobFile(globPathName)
+		if _, err := os.Stat(globNinjaFile); os.IsNotExist(err) {
+			err := bootstrap.WriteBuildGlobsNinjaFile(&bootstrap.GlobSingleton{
+				GlobLister: func() pathtools.MultipleGlobResults { return nil },
+				GlobFile:   globNinjaFile,
+				GlobDir:    bootstrap.GlobDirectory(config.SoongOutDir(), globPathName),
+				SrcDir:     ".",
+			}, blueprintConfig)
+			if err != nil {
+				ctx.Fatal(err)
+			}
+		} else if err != nil {
+			ctx.Fatal(err)
+		}
+	}
+
 	// since `bootstrap.ninja` is regenerated unconditionally, we ignore the deps, i.e. little
 	// reason to write a `bootstrap.ninja.d` file
 	_, err := bootstrap.RunBlueprint(blueprintArgs, bootstrap.DoEverything, blueprintCtx, blueprintConfig)
diff --git a/ui/build/test_build.go b/ui/build/test_build.go
index c5dc4c53f..309513919 100644
--- a/ui/build/test_build.go
+++ b/ui/build/test_build.go
@@ -63,6 +63,7 @@ func testForDanglingRules(ctx Context, config Config) {
 
 	outDir := config.OutDir()
 	modulePathsDir := filepath.Join(outDir, ".module_paths")
+	rawFilesDir := filepath.Join(outDir, "soong", "raw")
 	variablesFilePath := filepath.Join(outDir, "soong", "soong.variables")
 
 	// dexpreopt.config is an input to the soong_docs action, which runs the
@@ -88,6 +89,7 @@ func testForDanglingRules(ctx Context, config Config) {
 			continue
 		}
 		if strings.HasPrefix(line, modulePathsDir) ||
+			strings.HasPrefix(line, rawFilesDir) ||
 			line == variablesFilePath ||
 			line == dexpreoptConfigFilePath ||
 			line == buildDatetimeFilePath ||
diff --git a/ui/status/ninja_frontend/README b/ui/status/ninja_frontend/README
index 8c4b4510f..767bbf17a 100644
--- a/ui/status/ninja_frontend/README
+++ b/ui/status/ninja_frontend/README
@@ -1,3 +1,3 @@
-This comes from https://android.googlesource.com/platform/external/ninja/+/master/src/frontend.proto
+This comes from https://android.googlesource.com/platform/external/ninja/+/main/src/frontend.proto
 
 The only difference is the specification of a go_package. To regenerate frontend.pb.go, run regen.sh.
diff --git a/ui/terminal/format.go b/ui/terminal/format.go
index 539102390..241a1ddf7 100644
--- a/ui/terminal/format.go
+++ b/ui/terminal/format.go
@@ -25,7 +25,6 @@ import (
 type formatter struct {
 	format string
 	quiet  bool
-	smart  bool
 	start  time.Time
 }
 
@@ -33,11 +32,10 @@ type formatter struct {
 // the terminal in a format similar to Ninja.
 // format takes nearly all the same options as NINJA_STATUS.
 // %c is currently unsupported.
-func newFormatter(format string, quiet bool, smart bool) formatter {
+func newFormatter(format string, quiet bool) formatter {
 	return formatter{
 		format: format,
 		quiet:  quiet,
-		smart:  smart,
 		start:  time.Now(),
 	}
 }
@@ -63,9 +61,8 @@ func remainingTimeString(t time.Time) string {
 func (s formatter) progress(counts status.Counts) string {
 	if s.format == "" {
 		output := fmt.Sprintf("[%3d%% %d/%d", 100*counts.FinishedActions/counts.TotalActions, counts.FinishedActions, counts.TotalActions)
-		// Not to break parsing logic in the build bot
-		// TODO(b/313981966): make buildbot more flexible for output format
-		if s.smart && !counts.EstimatedTime.IsZero() {
+
+		if !counts.EstimatedTime.IsZero() {
 			output += fmt.Sprintf(" %s remaining", remainingTimeString(counts.EstimatedTime))
 		}
 		output += "] "
diff --git a/ui/terminal/status.go b/ui/terminal/status.go
index 810e3c93d..2ad174fee 100644
--- a/ui/terminal/status.go
+++ b/ui/terminal/status.go
@@ -27,10 +27,9 @@ import (
 // statusFormat takes nearly all the same options as NINJA_STATUS.
 // %c is currently unsupported.
 func NewStatusOutput(w io.Writer, statusFormat string, forceSimpleOutput, quietBuild, forceKeepANSI bool) status.StatusOutput {
-	useSmartStatus := !forceSimpleOutput && isSmartTerminal(w)
-	formatter := newFormatter(statusFormat, quietBuild, useSmartStatus)
+	formatter := newFormatter(statusFormat, quietBuild)
 
-	if useSmartStatus {
+	if !forceSimpleOutput && isSmartTerminal(w) {
 		return NewSmartStatusOutput(w, formatter)
 	} else {
 		return NewSimpleStatusOutput(w, formatter, forceKeepANSI)