multiplex compiler progress messages into the build runner

Author: andrewrk

lib/build_runner.zig

@@ -571,7 +571,7 @@ fn workerMakeOneStep(
     // For example, CompileStep does some sus things with modifying the saved
     // *Build object in install header steps that might be able to be removed
     // by passing the *Build object through the make() functions.
-    const make_result = s.make();
+    const make_result = s.make(&sub_prog_node);
 
     // No matter the result, we want to display error/warning messages.
     if (s.result_error_msgs.items.len > 0) {

lib/std/Build.zig

@@ -718,7 +718,8 @@ pub fn getUninstallStep(self: *Build) *Step {
     return &self.uninstall_tls.step;
 }
 
-fn makeUninstall(uninstall_step: *Step) anyerror!void {
+fn makeUninstall(uninstall_step: *Step, prog_node: *std.Progress.Node) anyerror!void {
+    _ = prog_node;
     const uninstall_tls = @fieldParentPtr(TopLevelStep, "step", uninstall_step);
     const self = @fieldParentPtr(Build, "uninstall_tls", uninstall_tls);
 
@@ -1404,7 +1405,7 @@ pub fn execAllowFail(
 
 /// This function is used exclusively for spawning and communicating with the zig compiler.
 /// TODO: move to build_runner.zig
-pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step) ![]const u8 {
+pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step, prog_node: *std.Progress.Node) ![]const u8 {
     assert(argv.len != 0);
 
     if (b.verbose) {
@@ -1439,6 +1440,11 @@ pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step) ![]const u8 {
     const Header = std.zig.Server.Message.Header;
     var result: ?[]const u8 = null;
 
+    var node_name: std.ArrayListUnmanaged(u8) = .{};
+    defer node_name.deinit(b.allocator);
+    var sub_prog_node: ?std.Progress.Node = null;
+    defer if (sub_prog_node) |*n| n.end();
+
     while (try poller.poll()) {
         const stdout = poller.fifo(.stdout);
         const buf = stdout.readableSlice(0);
@@ -1478,7 +1484,11 @@ pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step) ![]const u8 {
                         };
                     },
                     .progress => {
-                        @panic("TODO handle progress message");
+                        if (sub_prog_node) |*n| n.end();
+                        node_name.clearRetainingCapacity();
+                        try node_name.appendSlice(b.allocator, body);
+                        sub_prog_node = prog_node.start(node_name.items, 0);
+                        sub_prog_node.?.activate();
                     },
                     .emit_bin_path => {
                         result = try b.allocator.dupe(u8, body);

lib/std/Build/CheckFileStep.zig

@@ -33,7 +33,8 @@ pub fn create(
     return self;
 }
 
-fn make(step: *Step) !void {
+fn make(step: *Step, prog_node: *std.Progress.Node) !void {
+    _ = prog_node;
     const self = @fieldParentPtr(CheckFileStep, "step", step);
 
     const src_path = self.source.getPath(self.builder);

lib/std/Build/CheckObjectStep.zig

@@ -300,7 +300,8 @@ pub fn checkComputeCompare(
     self.checks.append(new_check) catch @panic("OOM");
 }
 
-fn make(step: *Step) !void {
+fn make(step: *Step, prog_node: *std.Progress.Node) !void {
+    _ = prog_node;
     const self = @fieldParentPtr(CheckObjectStep, "step", step);
 
     const gpa = self.builder.allocator;

lib/std/Build/CompileStep.zig

@@ -1160,7 +1160,7 @@ fn constructDepString(
     }
 }
 
-fn make(step: *Step) !void {
+fn make(step: *Step, prog_node: *std.Progress.Node) !void {
     const self = @fieldParentPtr(CompileStep, "step", step);
     const builder = self.builder;
 
@@ -1718,7 +1718,7 @@ fn make(step: *Step) !void {
                 }
                 if (other.installed_headers.items.len > 0) {
                     for (other.installed_headers.items) |install_step| {
-                        try install_step.make();
+                        try install_step.make(prog_node);
                     }
                     try zig_args.append("-I");
                     try zig_args.append(builder.pathJoin(&.{
@@ -1894,7 +1894,7 @@ fn make(step: *Step) !void {
         try zig_args.append(resolved_args_file);
     }
 
-    const output_bin_path = try builder.execFromStep(zig_args.items, &self.step);
+    const output_bin_path = try builder.execFromStep(zig_args.items, &self.step, prog_node);
     const build_output_dir = fs.path.dirname(output_bin_path).?;
 
     if (self.output_dir) |output_dir| {

lib/std/Build/ConfigHeaderStep.zig

@@ -152,7 +152,8 @@ fn putValue(self: *ConfigHeaderStep, field_name: []const u8, comptime T: type, v
     }
 }
 
-fn make(step: *Step) !void {
+fn make(step: *Step, prog_node: *std.Progress.Node) !void {
+    _ = prog_node;
     const self = @fieldParentPtr(ConfigHeaderStep, "step", step);
     const gpa = self.builder.allocator;
 

lib/std/Build/EmulatableRunStep.zig

@@ -71,7 +71,8 @@ pub fn create(builder: *std.Build, name: []const u8, artifact: *CompileStep) *Em
     return self;
 }
 
-fn make(step: *Step) !void {
+fn make(step: *Step, prog_node: *std.Progress.Node) !void {
+    _ = prog_node;
     const self = @fieldParentPtr(EmulatableRunStep, "step", step);
     const host_info = self.builder.host;
 

lib/std/Build/FmtStep.zig

@@ -29,7 +29,8 @@ pub fn create(builder: *std.Build, paths: []const []const u8) *FmtStep {
     return self;
 }
 
-fn make(step: *Step) !void {
+fn make(step: *Step, prog_node: *std.Progress.Node) !void {
+    _ = prog_node;
     const self = @fieldParentPtr(FmtStep, "step", step);
 
     return self.builder.spawnChild(self.argv);

lib/std/Build/InstallArtifactStep.zig

@@ -64,7 +64,8 @@ pub fn create(builder: *std.Build, artifact: *CompileStep) *InstallArtifactStep
     return self;
 }
 
-fn make(step: *Step) !void {
+fn make(step: *Step, prog_node: *std.Progress.Node) !void {
+    _ = prog_node;
     const self = @fieldParentPtr(InstallArtifactStep, "step", step);
     const builder = self.builder;
 

lib/std/Build/InstallDirStep.zig

@@ -56,7 +56,8 @@ pub fn init(
     };
 }
 
-fn make(step: *Step) !void {
+fn make(step: *Step, prog_node: *std.Progress.Node) !void {
+    _ = prog_node;
     const self = @fieldParentPtr(InstallDirStep, "step", step);
     const dest_prefix = self.builder.getInstallPath(self.options.install_dir, self.options.install_subdir);
     const src_builder = self.override_source_builder orelse self.builder;

lib/std/Build/InstallFileStep.zig

@@ -35,7 +35,8 @@ pub fn init(
     };
 }
 
-fn make(step: *Step) !void {
+fn make(step: *Step, prog_node: *std.Progress.Node) !void {
+    _ = prog_node;
     const self = @fieldParentPtr(InstallFileStep, "step", step);
     const src_builder = self.override_source_builder orelse self.builder;
     const full_src_path = self.source.getPath2(src_builder, step);

lib/std/Build/LogStep.zig

@@ -21,7 +21,8 @@ pub fn init(builder: *std.Build, data: []const u8) LogStep {
     };
 }
 
-fn make(step: *Step) anyerror!void {
+fn make(step: *Step, prog_node: *std.Progress.Node) anyerror!void {
+    _ = prog_node;
     const self = @fieldParentPtr(LogStep, "step", step);
     log.info("{s}", .{self.data});
 }

lib/std/Build/ObjCopyStep.zig

@@ -66,7 +66,8 @@ pub fn getOutputSource(self: *const ObjCopyStep) std.Build.FileSource {
     return .{ .generated = &self.output_file };
 }
 
-fn make(step: *Step) !void {
+fn make(step: *Step, prog_node: *std.Progress.Node) !void {
+    _ = prog_node;
     const self = @fieldParentPtr(ObjCopyStep, "step", step);
     const b = self.builder;
 

lib/std/Build/OptionsStep.zig

@@ -219,7 +219,8 @@ pub fn getSource(self: *OptionsStep) FileSource {
     return .{ .generated = &self.generated_file };
 }
 
-fn make(step: *Step) !void {
+fn make(step: *Step, prog_node: *std.Progress.Node) !void {
+    _ = prog_node;
     const self = @fieldParentPtr(OptionsStep, "step", step);
 
     for (self.artifact_args.items) |item| {

lib/std/Build/RemoveDirStep.zig

@@ -22,7 +22,8 @@ pub fn init(builder: *std.Build, dir_path: []const u8) RemoveDirStep {
     };
 }
 
-fn make(step: *Step) !void {
+fn make(step: *Step, prog_node: *std.Progress.Node) !void {
+    _ = prog_node;
     const self = @fieldParentPtr(RemoveDirStep, "step", step);
 
     const full_path = self.builder.pathFromRoot(self.dir_path);

lib/std/Build/RunStep.zig

@@ -206,7 +206,8 @@ fn needOutputCheck(self: RunStep) bool {
     return false;
 }
 
-fn make(step: *Step) !void {
+fn make(step: *Step, prog_node: *std.Progress.Node) !void {
+    _ = prog_node;
     const self = @fieldParentPtr(RunStep, "step", step);
     const need_output_check = self.needOutputCheck();
 

lib/std/Build/Step.zig

@@ -1,6 +1,6 @@
 id: Id,
 name: []const u8,
-makeFn: *const fn (self: *Step) anyerror!void,
+makeFn: MakeFn,
 dependencies: std.ArrayList(*Step),
 /// This field is empty during execution of the user's build script, and
 /// then populated during dependency loop checking in the build runner.
@@ -13,6 +13,8 @@ debug_stack_trace: [n_debug_stack_frames]usize,
 result_error_msgs: std.ArrayListUnmanaged([]const u8),
 result_error_bundle: std.zig.ErrorBundle,
 
+pub const MakeFn = *const fn (self: *Step, prog_node: *std.Progress.Node) anyerror!void;
+
 const n_debug_stack_frames = 4;
 
 pub const State = enum {
@@ -72,7 +74,7 @@ pub const Id = enum {
 pub const Options = struct {
     id: Id,
     name: []const u8,
-    makeFn: *const fn (self: *Step) anyerror!void = makeNoOp,
+    makeFn: MakeFn = makeNoOp,
     first_ret_addr: ?usize = null,
 };
 
@@ -101,8 +103,8 @@ pub fn init(allocator: Allocator, options: Options) Step {
 /// If the Step's `make` function reports `error.MakeFailed`, it indicates they
 /// have already reported the error. Otherwise, we add a simple error report
 /// here.
-pub fn make(s: *Step) error{MakeFailed}!void {
-    return s.makeFn(s) catch |err| {
+pub fn make(s: *Step, prog_node: *std.Progress.Node) error{MakeFailed}!void {
+    return s.makeFn(s, prog_node) catch |err| {
         if (err != error.MakeFailed) {
             const gpa = s.dependencies.allocator;
             s.result_error_msgs.append(gpa, std.fmt.allocPrint(gpa, "{s} failed: {s}", .{
@@ -129,8 +131,9 @@ pub fn getStackTrace(s: *Step) std.builtin.StackTrace {
     };
 }
 
-fn makeNoOp(self: *Step) anyerror!void {
+fn makeNoOp(self: *Step, prog_node: *std.Progress.Node) anyerror!void {
     _ = self;
+    _ = prog_node;
 }
 
 pub fn cast(step: *Step, comptime T: type) ?*T {

lib/std/Build/TranslateCStep.zig

@@ -88,7 +88,7 @@ pub fn defineCMacroRaw(self: *TranslateCStep, name_and_value: []const u8) void {
     self.c_macros.append(self.builder.dupe(name_and_value)) catch @panic("OOM");
 }
 
-fn make(step: *Step) !void {
+fn make(step: *Step, prog_node: *std.Progress.Node) !void {
     const self = @fieldParentPtr(TranslateCStep, "step", step);
 
     var argv_list = std.ArrayList([]const u8).init(self.builder.allocator);
@@ -120,7 +120,7 @@ fn make(step: *Step) !void {
 
     try argv_list.append(self.source.getPath(self.builder));
 
-    const output_path_nl = try self.builder.execFromStep(argv_list.items, &self.step);
+    const output_path_nl = try self.builder.execFromStep(argv_list.items, &self.step, prog_node);
     const output_path = mem.trimRight(u8, output_path_nl, "\r\n");
 
     self.out_basename = fs.path.basename(output_path);

lib/std/Build/WriteFileStep.zig

@@ -99,7 +99,8 @@ pub fn getFileSource(wf: *WriteFileStep, sub_path: []const u8) ?std.Build.FileSo
     return null;
 }
 
-fn make(step: *Step) !void {
+fn make(step: *Step, prog_node: *std.Progress.Node) !void {
+    _ = prog_node;
     const wf = @fieldParentPtr(WriteFileStep, "step", step);
 
     // Writing to source files is kind of an extra capability of this

src/main.zig

@@ -3573,7 +3573,21 @@ fn serve(
                 if (comp.bin_file.options.output_mode == .Exe) {
                     try comp.makeBinFileWritable();
                 }
-                try comp.update(main_progress_node);
+
+                {
+                    var reset: std.Thread.ResetEvent = .{};
+
+                    var progress_thread = try std.Thread.spawn(.{}, progressThread, .{
+                        &progress, out, &reset,
+                    });
+                    defer {
+                        reset.set();
+                        progress_thread.join();
+                    }
+
+                    try comp.update(main_progress_node);
+                }
+
                 try comp.makeBinFileExecutable();
                 try serveUpdateResults(out, comp);
             },
@@ -3629,6 +3643,63 @@ fn serve(
     }
 }
 
+fn progressThread(progress: *std.Progress, out: fs.File, reset: *std.Thread.ResetEvent) void {
+    while (true) {
+        if (reset.timedWait(500 * std.time.ns_per_ms)) |_| {
+            // The Compilation update has completed.
+            return;
+        } else |err| switch (err) {
+            error.Timeout => {},
+        }
+
+        var buf: std.BoundedArray(u8, 160) = .{};
+
+        {
+            progress.update_mutex.lock();
+            defer progress.update_mutex.unlock();
+
+            var need_ellipse = false;
+            var maybe_node: ?*std.Progress.Node = &progress.root;
+            while (maybe_node) |node| {
+                if (need_ellipse) {
+                    buf.appendSlice("... ") catch {};
+                }
+                need_ellipse = false;
+                const eti = @atomicLoad(usize, &node.unprotected_estimated_total_items, .Monotonic);
+                const completed_items = @atomicLoad(usize, &node.unprotected_completed_items, .Monotonic);
+                const current_item = completed_items + 1;
+                if (node.name.len != 0 or eti > 0) {
+                    if (node.name.len != 0) {
+                        buf.appendSlice(node.name) catch {};
+                        need_ellipse = true;
+                    }
+                    if (eti > 0) {
+                        if (need_ellipse) buf.appendSlice(" ") catch {};
+                        buf.writer().print("[{d}/{d}] ", .{ current_item, eti }) catch {};
+                        need_ellipse = false;
+                    } else if (completed_items != 0) {
+                        if (need_ellipse) buf.appendSlice(" ") catch {};
+                        buf.writer().print("[{d}] ", .{current_item}) catch {};
+                        need_ellipse = false;
+                    }
+                }
+                maybe_node = @atomicLoad(?*std.Progress.Node, &node.recently_updated_child, .Acquire);
+            }
+        }
+
+        const progress_string = buf.slice();
+
+        serveMessage(out, .{
+            .tag = .progress,
+            .bytes_len = @intCast(u32, progress_string.len),
+        }, &.{
+            progress_string,
+        }) catch |err| {
+            fatal("unable to write to client: {s}", .{@errorName(err)});
+        };
+    }
+}
+
 fn serveMessage(
     out: fs.File,
     header: std.zig.Server.Message.Header,

test/tests.zig

@@ -875,7 +875,8 @@ pub const StackTracesContext = struct {
             return ptr;
         }
 
-        fn make(step: *Step) !void {
+        fn make(step: *Step, prog_node: *std.Progress.Node) !void {
+            _ = prog_node;
             const self = @fieldParentPtr(RunAndCompareStep, "step", step);
             const b = self.context.b;
 
@@ -1218,7 +1219,8 @@ pub const GenHContext = struct {
             return ptr;
         }
 
-        fn make(step: *Step) !void {
+        fn make(step: *Step, prog_node: *std.Progress.Node) !void {
+            _ = prog_node;
             const self = @fieldParentPtr(GenHCmpOutputStep, "step", step);
             const b = self.context.b;