Json Progress File (#391)

* Json Progress File

* Improve JSON progress file handling and error reporting

* Refactor progress file parsing and improve test validation

Author: roulpriya

Documentation/SwiftlyDocs.docc/swiftly-cli-reference.md

@@ -23,7 +23,7 @@ swiftly [--version] [--help]
 Install a new toolchain.
 
 ```
-swiftly install [<version>] [--use] [--verify|no-verify] [--post-install-file=<post-install-file>] [--assume-yes] [--verbose] [--version] [--help]
+swiftly install [<version>] [--use] [--verify|no-verify] [--post-install-file=<post-install-file>] [--progress-file=<progress-file>] [--assume-yes] [--verbose] [--version] [--help]
 ```
 
 **version:**
@@ -80,6 +80,15 @@ If the toolchain that is installed has extra post installation steps, they will
 written to this file as commands that can be run after the installation.
 
 
+**--progress-file=\<progress-file\>:**
+
+*A file path where progress information will be written in JSONL format*
+
+Progress information will be appended to this file as JSON objects, one per line.
+Each progress entry contains timestamp, progress percentage, and a descriptive message.
+The file must be writable, else an error will be thrown.
+
+
 **--assume-yes:**
 
 *Disable confirmation prompts by assuming 'yes'*

Sources/Swiftly/Install.swift

@@ -71,10 +71,21 @@ struct Install: SwiftlyCommand {
         ))
     var postInstallFile: FilePath?
 
+    @Option(
+        help: ArgumentHelp(
+            "A file path where progress information will be written in JSONL format",
+            discussion: """
+            Progress information will be appended to this file as JSON objects, one per line.
+            Each progress entry contains timestamp, progress percentage, and a descriptive message.
+            The file must be writable, else an error will be thrown.
+            """
+        ))
+    var progressFile: FilePath?
+
     @OptionGroup var root: GlobalOptions
 
     private enum CodingKeys: String, CodingKey {
-        case version, use, verify, postInstallFile, root
+        case version, use, verify, postInstallFile, root, progressFile
     }
 
     mutating func run() async throws {
@@ -93,7 +104,9 @@ struct Install: SwiftlyCommand {
         try await validateLinked(ctx)
 
         var config = try await Config.load(ctx)
-        let toolchainVersion = try await Self.determineToolchainVersion(ctx, version: self.version, config: &config)
+        let toolchainVersion = try await Self.determineToolchainVersion(
+            ctx, version: self.version, config: &config
+        )
 
         let (postInstallScript, pathChanged) = try await Self.execute(
             ctx,
@@ -102,7 +115,8 @@ struct Install: SwiftlyCommand {
             useInstalledToolchain: self.use,
             verifySignature: self.verify,
             verbose: self.root.verbose,
-            assumeYes: self.root.assumeYes
+            assumeYes: self.root.assumeYes,
+            progressFile: self.progressFile
         )
 
         let shell =
@@ -192,8 +206,9 @@ struct Install: SwiftlyCommand {
                 await ctx.message("Setting up toolchain proxies...")
             }
 
-            let proxiesToCreate = Set(toolchainBinDirContents).subtracting(swiftlyBinDirContents).union(
-                overwrite)
+            let proxiesToCreate = Set(toolchainBinDirContents).subtracting(swiftlyBinDirContents)
+                .union(
+                    overwrite)
 
             for p in proxiesToCreate {
                 let proxy = Swiftly.currentPlatform.swiftlyBinDir(ctx) / p
@@ -248,7 +263,8 @@ struct Install: SwiftlyCommand {
         useInstalledToolchain: Bool,
         verifySignature: Bool,
         verbose: Bool,
-        assumeYes: Bool
+        assumeYes: Bool,
+        progressFile: FilePath? = nil
     ) async throws -> (postInstall: String?, pathChanged: Bool) {
         guard !config.installedToolchains.contains(version) else {
             await ctx.message("\(version) is already installed.")
@@ -258,10 +274,11 @@ struct Install: SwiftlyCommand {
         // Ensure the system is set up correctly before downloading it. Problems that prevent installation
         //  will throw, while problems that prevent use of the toolchain will be written out as a post install
         //  script for the user to run afterwards.
-        let postInstallScript = try await Swiftly.currentPlatform.verifySystemPrerequisitesForInstall(
-            ctx, platformName: config.platform.name, version: version,
-            requireSignatureValidation: verifySignature
-        )
+        let postInstallScript = try await Swiftly.currentPlatform
+            .verifySystemPrerequisitesForInstall(
+                ctx, platformName: config.platform.name, version: version,
+                requireSignatureValidation: verifySignature
+            )
 
         await ctx.message("Installing \(version)")
 
@@ -296,10 +313,17 @@ struct Install: SwiftlyCommand {
                 }
             }
 
-            let animation = PercentProgressAnimation(
-                stream: stdoutStream,
-                header: "Downloading \(version)"
-            )
+            let animation: ProgressAnimationProtocol =
+                if let progressFile
+            {
+                try JsonFileProgressReporter(ctx, filePath: progressFile)
+            } else {
+                PercentProgressAnimation(stream: stdoutStream, header: "Downloading \(version)")
+            }
+
+            defer {
+                try? (animation as? JsonFileProgressReporter)?.close()
+            }
 
             var lastUpdate = Date()
 
@@ -315,7 +339,9 @@ struct Install: SwiftlyCommand {
                     reportProgress: { progress in
                         let now = Date()
 
-                        guard lastUpdate.distance(to: now) > 0.25 || progress.receivedBytes == progress.totalBytes
+                        guard
+                            lastUpdate.distance(to: now) > 0.25
+                            || progress.receivedBytes == progress.totalBytes
                         else {
                             return
                         }
@@ -334,7 +360,8 @@ struct Install: SwiftlyCommand {
                     }
                 )
             } catch let notFound as DownloadNotFoundError {
-                throw SwiftlyError(message: "\(version) does not exist at URL \(notFound.url), exiting")
+                throw SwiftlyError(
+                    message: "\(version) does not exist at URL \(notFound.url), exiting")
             } catch {
                 animation.complete(success: false)
                 throw error
@@ -401,7 +428,9 @@ struct Install: SwiftlyCommand {
     }
 
     /// Utilize the swift.org API along with the provided selector to select a toolchain for install.
-    public static func resolve(_ ctx: SwiftlyCoreContext, config: Config, selector: ToolchainSelector)
+    public static func resolve(
+        _ ctx: SwiftlyCoreContext, config: Config, selector: ToolchainSelector
+    )
         async throws -> ToolchainVersion
     {
         switch selector {
@@ -426,7 +455,8 @@ struct Install: SwiftlyCommand {
             }
 
             if let patch {
-                return .stable(ToolchainVersion.StableRelease(major: major, minor: minor, patch: patch))
+                return .stable(
+                    ToolchainVersion.StableRelease(major: major, minor: minor, patch: patch))
             }
 
             await ctx.message("Fetching the latest stable Swift \(major).\(minor) release...")

Sources/Swiftly/JsonFileProgressReporter.swift

@@ -0,0 +1,62 @@
+import Foundation
+import SwiftlyCore
+import SystemPackage
+import TSCUtility
+
+enum ProgressInfo: Codable {
+    case step(timestamp: Date, percent: Int, text: String)
+    case complete(success: Bool)
+}
+
+struct JsonFileProgressReporter: ProgressAnimationProtocol {
+    let filePath: FilePath
+    private let encoder: JSONEncoder
+    private let ctx: SwiftlyCoreContext
+    private let fileHandle: FileHandle
+
+    init(_ ctx: SwiftlyCoreContext, filePath: FilePath, encoder: JSONEncoder = JSONEncoder()) throws
+    {
+        self.ctx = ctx
+        self.filePath = filePath
+        self.encoder = encoder
+        self.fileHandle = try FileHandle(forWritingTo: URL(fileURLWithPath: filePath.string))
+    }
+
+    private func writeProgress(_ progress: ProgressInfo) {
+        let jsonData = try? self.encoder.encode(progress)
+        guard let jsonData = jsonData else {
+            Task { [ctx = self.ctx] in
+                await ctx.message("Failed to encode progress entry to JSON")
+            }
+            return
+        }
+
+        self.fileHandle.write(jsonData)
+        self.fileHandle.write("\n".data(using: .utf8) ?? Data())
+        try? self.fileHandle.synchronize()
+    }
+
+    func update(step: Int, total: Int, text: String) {
+        guard total > 0 && step <= total else {
+            return
+        }
+        self.writeProgress(
+            ProgressInfo.step(
+                timestamp: Date(),
+                percent: Int(Double(step) / Double(total) * 100),
+                text: text
+            ))
+    }
+
+    func complete(success: Bool) {
+        self.writeProgress(ProgressInfo.complete(success: success))
+    }
+
+    func clear() {
+        // not implemented for JSON file reporter
+    }
+
+    func close() throws {
+        try self.fileHandle.close()
+    }
+}

Sources/SwiftlyCore/FileManager+FilePath.swift

@@ -60,7 +60,7 @@ public enum FileSystem {
         case mode(Int)
     }
 
-    public static func create(_ options: CreateOptions..., file: FilePath, contents: Data?) async throws {
+    public static func create(_ options: CreateOptions..., file: FilePath, contents: Data? = nil) async throws {
         try await Self.create(options, file: file, contents: contents)
     }
 

Tests/SwiftlyTests/InstallTests.swift

@@ -1,6 +1,7 @@
 import Foundation
 @testable import Swiftly
 @testable import SwiftlyCore
+import SystemPackage
 import Testing
 
 @Suite struct InstallTests {
@@ -262,4 +263,130 @@ import Testing
         try await SwiftlyTests.installMockedToolchain(selector: ToolchainVersion.newStable.name, args: ["--use"])
         try await SwiftlyTests.validateInUse(expected: .newStable)
     }
+
+    /// Verify that progress information is written to the progress file when specified.
+    @Test(.testHomeMockedToolchain()) func installProgressFile() async throws {
+        let progressFile = fs.mktemp(ext: ".json")
+        try await fs.create(.mode(0o644), file: progressFile)
+
+        try await SwiftlyTests.runCommand(Install.self, [
+            "install", "5.7.0",
+            "--post-install-file=\(fs.mktemp())",
+            "--progress-file=\(progressFile.string)",
+        ])
+
+        #expect(try await fs.exists(atPath: progressFile))
+
+        let decoder = JSONDecoder()
+        let progressContent = try String(contentsOfFile: progressFile.string)
+        let progressInfo = try progressContent.split(separator: "\n")
+            .filter { !$0.isEmpty }
+            .map { line in
+                try decoder.decode(ProgressInfo.self, from: Data(line.utf8))
+            }
+
+        #expect(!progressInfo.isEmpty, "Progress file should contain progress entries")
+
+        // Verify that at least one step progress entry exists
+        let hasStepEntry = progressInfo.contains { info in
+            if case .step = info { return true }
+            return false
+        }
+        #expect(hasStepEntry, "Progress file should contain step progress entries")
+
+        // Verify that a completion entry exists
+        let hasCompletionEntry = progressInfo.contains { info in
+            if case .complete = info { return true }
+            return false
+        }
+        #expect(hasCompletionEntry, "Progress file should contain completion entry")
+
+        // Clean up
+        try FileManager.default.removeItem(atPath: progressFile.string)
+    }
+
+#if os(Linux) || os(macOS)
+    @Test(.testHomeMockedToolchain())
+    func installProgressFileNamedPipe() async throws {
+        let tempDir = NSTemporaryDirectory()
+        let pipePath = tempDir + "swiftly_install_progress_pipe"
+
+        let result = mkfifo(pipePath, 0o644)
+        guard result == 0 else {
+            return // Skip test if mkfifo syscall failed
+        }
+
+        defer {
+            try? FileManager.default.removeItem(atPath: pipePath)
+        }
+
+        var receivedMessages: [ProgressInfo] = []
+        let decoder = JSONDecoder()
+        var installCompleted = false
+
+        let readerTask = Task {
+            guard let fileHandle = FileHandle(forReadingAtPath: pipePath) else { return }
+            defer { fileHandle.closeFile() }
+
+            var buffer = Data()
+
+            while !installCompleted {
+                let data = fileHandle.availableData
+                if data.isEmpty {
+                    try await Task.sleep(nanoseconds: 100_000_000)
+                    continue
+                }
+
+                buffer.append(data)
+
+                while let newlineRange = buffer.range(of: "\n".data(using: .utf8)!) {
+                    let lineData = buffer.subdata(in: 0..<newlineRange.lowerBound)
+                    buffer.removeSubrange(0..<newlineRange.upperBound)
+
+                    if !lineData.isEmpty {
+                        if let progress = try? decoder.decode(ProgressInfo.self, from: lineData) {
+                            receivedMessages.append(progress)
+                            if case .complete = progress {
+                                installCompleted = true
+                                return
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        let installTask = Task {
+            try await SwiftlyTests.runCommand(Install.self, [
+                "install", "5.7.0",
+                "--post-install-file=\(fs.mktemp())",
+                "--progress-file=\(pipePath)",
+            ])
+        }
+
+        await withTaskGroup(of: Void.self) { group in
+            group.addTask { try? await readerTask.value }
+            group.addTask { try? await installTask.value }
+        }
+
+        #expect(!receivedMessages.isEmpty, "Named pipe should receive progress entries")
+
+        let hasCompletionEntry = receivedMessages.contains { info in
+            if case .complete = info { return true }
+            return false
+        }
+        #expect(hasCompletionEntry, "Named pipe should receive completion entry")
+
+        for message in receivedMessages {
+            switch message {
+            case let .step(timestamp, percent, text):
+                #expect(timestamp.timeIntervalSince1970 > 0)
+                #expect(percent >= 0 && percent <= 100)
+                #expect(!text.isEmpty)
+            case let .complete(success):
+                #expect(success == true)
+            }
+        }
+    }
+#endif
 }