Merge pull request #204 from rabc/handle_json_format

Update to handle the new JSON format introduced in Xcode 15.3

Sources/XCLogParser/activityparser/ActivityParser.swift

@@ -110,6 +110,7 @@ public class ActivityParser {
                                      uniqueIdentifier: try parseAsString(token: iterator.next()),
                                      localizedResultString: try parseAsString(token: iterator.next()),
                                      xcbuildSignature: try parseAsString(token: iterator.next()),
+                                     attachments: try parseIDEActivityLogSectionAttachments(iterator: &iterator),
                                      unknown: isCommandLineLog ? Int(try parseAsInt(token: iterator.next())) : 0)
     }
 
@@ -133,6 +134,7 @@ public class ActivityParser {
                                          uniqueIdentifier: try parseAsString(token: iterator.next()),
                                          localizedResultString: try parseAsString(token: iterator.next()),
                                          xcbuildSignature: try parseAsString(token: iterator.next()),
+                                         attachments: try parseIDEActivityLogSectionAttachments(iterator: &iterator),
                                          unknown: isCommandLineLog ? Int(try parseAsInt(token: iterator.next())) : 0,
                                          testsPassedString: try parseAsString(token: iterator.next()),
                                          durationString: try parseAsString(token: iterator.next()),
@@ -163,6 +165,8 @@ public class ActivityParser {
                                                  localizedResultString: try parseAsString(token: iterator.next()),
                                                  xcbuildSignature: try parseAsString(token: iterator.next()),
                                                  // swiftlint:disable:next line_length
+                                                 attachments: try parseIDEActivityLogSectionAttachments(iterator: &iterator),
+                                                 // swiftlint:disable:next line_length
                                                  unknown: isCommandLineLog ? Int(try parseAsInt(token: iterator.next())) : 0,
                                                  logConsoleItems: try parseIDEConsoleItems(iterator: &iterator)
                                                  )
@@ -361,6 +365,25 @@ public class ActivityParser {
         throw XCLogParserError.parseError("Unexpected className found parsing IDEActivityLogMessage \(className)")
     }
 
+    private func parseLogSectionAttachment(iterator: inout IndexingIterator<[Token]>)
+        throws -> IDEActivityLogSectionAttachment {
+            let classRefToken = try getClassRefToken(iterator: &iterator)
+            guard case Token.classNameRef(let className) = classRefToken else {
+                throw XCLogParserError.parseError("Unexpected token found parsing " +
+                                                  "IDEActivityLogSectionAttachment \(classRefToken)")
+            }
+
+            if className == "IDEFoundation.\(String(describing: IDEActivityLogSectionAttachment.self))" {
+                let jsonType = IDEActivityLogSectionAttachment.BuildOperationTaskMetrics.self
+                return try IDEActivityLogSectionAttachment(identifier: try parseAsString(token: iterator.next()),
+                                                           majorVersion: try parseAsInt(token: iterator.next()),
+                                                           minorVersion: try parseAsInt(token: iterator.next()),
+                                                           metrics: try parseAsJson(token: iterator.next(),
+                                                                                    type: jsonType))
+            }
+            throw XCLogParserError.parseError("Unexpected className found parsing IDEConsoleItem \(className)")
+    }
+
     private func parseLogSection(iterator: inout IndexingIterator<[Token]>)
         throws -> IDEActivityLogSection {
         var classRefToken = try getClassRefToken(iterator: &iterator)
@@ -433,6 +456,27 @@ public class ActivityParser {
             }
     }
 
+    private func parseIDEActivityLogSectionAttachments(iterator: inout IndexingIterator<[Token]>)
+        throws -> [IDEActivityLogSectionAttachment] {
+            guard let listToken = iterator.next() else {
+                throw XCLogParserError.parseError("Unexpected EOF parsing array of IDEActivityLogSectionAttachment")
+            }
+            switch listToken {
+            case .null:
+                return []
+            case .list(let count):
+                var sections = [IDEActivityLogSectionAttachment]()
+                for _ in 0..<count {
+                    let section = try parseLogSectionAttachment(iterator: &iterator)
+                    sections.append(section)
+                }
+                return sections
+            default:
+                throw XCLogParserError.parseError("Unexpected token parsing array of " +
+                                                  "IDEActivityLogSectionAttachment: \(listToken)")
+            }
+    }
+
     private func parseIDEConsoleItem(iterator: inout IndexingIterator<[Token]>)
         throws -> IDEConsoleItem? {
             let classRefToken = try getClassRefToken(iterator: &iterator)
@@ -555,6 +599,23 @@ public class ActivityParser {
         }
     }
 
+    private func parseAsJson<T: Decodable>(token: Token?, type: T.Type) throws -> T? {
+        guard let token = token else {
+            throw XCLogParserError.parseError("Unexpected EOF parsing JSON String")
+        }
+        switch token {
+        case .json(let string):
+            guard let data = string.data(using: .utf8) else {
+                throw XCLogParserError.parseError("Unexpected JSON string \(string)")
+            }
+            return try JSONDecoder().decode(type, from: data)
+        case .null:
+            return nil
+        default:
+            throw XCLogParserError.parseError("Unexpected token parsing JSON String: \(token)")
+        }
+    }
+
     private func parseAsInt(token: Token?) throws -> UInt64 {
         guard let token = token else {
             throw XCLogParserError.parseError("Unexpected EOF parsing Int")

Sources/XCLogParser/activityparser/IDEActivityModel.swift

@@ -49,6 +49,7 @@ public class IDEActivityLogSection: Encodable {
     public let uniqueIdentifier: String
     public let localizedResultString: String
     public let xcbuildSignature: String
+    public let attachments: [IDEActivityLogSectionAttachment]
     public let unknown: Int
 
     public init(sectionType: Int8,
@@ -69,6 +70,7 @@ public class IDEActivityLogSection: Encodable {
                 uniqueIdentifier: String,
                 localizedResultString: String,
                 xcbuildSignature: String,
+                attachments: [IDEActivityLogSectionAttachment],
                 unknown: Int) {
         self.sectionType = sectionType
         self.domainType = domainType
@@ -88,6 +90,7 @@ public class IDEActivityLogSection: Encodable {
         self.uniqueIdentifier = uniqueIdentifier
         self.localizedResultString = localizedResultString
         self.xcbuildSignature = xcbuildSignature
+        self.attachments = attachments
         self.unknown = unknown
     }
 
@@ -119,6 +122,7 @@ public class IDEActivityLogUnitTestSection: IDEActivityLogSection {
                 uniqueIdentifier: String,
                 localizedResultString: String,
                 xcbuildSignature: String,
+                attachments: [IDEActivityLogSectionAttachment],
                 unknown: Int,
                 testsPassedString: String,
                 durationString: String,
@@ -151,6 +155,7 @@ public class IDEActivityLogUnitTestSection: IDEActivityLogSection {
                    uniqueIdentifier: uniqueIdentifier,
                    localizedResultString: localizedResultString,
                    xcbuildSignature: xcbuildSignature,
+                   attachments: attachments,
                    unknown: unknown)
     }
 
@@ -421,6 +426,7 @@ public class DBGConsoleLog: IDEActivityLogSection {
                 uniqueIdentifier: String,
                 localizedResultString: String,
                 xcbuildSignature: String,
+                attachments: [IDEActivityLogSectionAttachment],
                 unknown: Int,
                 logConsoleItems: [IDEConsoleItem]) {
         self.logConsoleItems = logConsoleItems
@@ -442,6 +448,7 @@ public class DBGConsoleLog: IDEActivityLogSection {
                    uniqueIdentifier: uniqueIdentifier,
                    localizedResultString: localizedResultString,
                    xcbuildSignature: xcbuildSignature,
+                   attachments: attachments,
                    unknown: unknown)
     }
 
@@ -635,3 +642,32 @@ public class DVTMemberDocumentLocation: DVTDocumentLocation, Equatable {
     }
 
 }
+
+// MARK: Added in Xcode 15.3
+
+public class IDEActivityLogSectionAttachment: Encodable {
+    public struct BuildOperationTaskMetrics: Codable {
+        public let utime: UInt64
+        public let stime: UInt64
+        public let maxRSS: UInt64
+        public let wcStartTime: UInt64
+        public let wcDuration: UInt64
+    }
+
+    public let identifier: String
+    public let majorVersion: UInt64
+    public let minorVersion: UInt64
+    public let metrics: BuildOperationTaskMetrics?
+
+    public init(
+        identifier: String,
+        majorVersion: UInt64,
+        minorVersion: UInt64,
+        metrics: BuildOperationTaskMetrics?
+    ) throws {
+        self.identifier = identifier
+        self.majorVersion = majorVersion
+        self.minorVersion = minorVersion
+        self.metrics = metrics
+    }
+}

Sources/XCLogParser/lexer/Lexer.swift

@@ -163,6 +163,11 @@ public final class Lexer {
             return .null
         case .list:
             return handleListTokenTypeCase(payload: payload)
+        case .json:
+            return handleJSONTokenTypeCase(scanner: scanner,
+                                           payload: payload,
+                                           redacted: redacted,
+                                           withoutBuildSpecificInformation: withoutBuildSpecificInformation)
         }
     }
 
@@ -213,6 +218,20 @@ public final class Lexer {
         return .string(content)
     }
 
+    private func handleJSONTokenTypeCase(scanner: Scanner,
+                                         payload: String,
+                                         redacted: Bool,
+                                         withoutBuildSpecificInformation: Bool) -> Token? {
+        guard let content = scanString(length: payload,
+                                       scanner: scanner,
+                                       redacted: redacted,
+                                       withoutBuildSpecificInformation: withoutBuildSpecificInformation) else {
+                                        print("error parsing string")
+                                        return nil
+        }
+        return .json(content)
+    }
+
     private func handleDoubleTokenTypeCase(payload: String) -> Token? {
         guard let double = hexToInt(payload) else {
             print("error parsing double")

Sources/XCLogParser/lexer/LexerModel.swift

@@ -27,6 +27,7 @@ public enum TokenType: String, CaseIterable {
     case double = "^"
     case null = "-"
     case list = "("
+    case json = "*"
 
     static func all() -> String {
         return TokenType.allCases.reduce(String()) {
@@ -43,6 +44,7 @@ public enum Token: CustomDebugStringConvertible, Equatable {
     case double(Double)
     case null
     case list(Int)
+    case json(String)
 }
 
 extension Token {
@@ -62,6 +64,8 @@ extension Token {
             return "[type: nil]"
         case .list(let count):
             return "[type: list, count: \(count)]"
+        case .json(let json):
+            return "[type: json, value: \(json)]"
         }
     }
 }

Sources/XCLogParser/parser/IDEActivityLogSection+Builders.swift

@@ -40,6 +40,7 @@ extension IDEActivityLogSection {
                                      uniqueIdentifier: self.uniqueIdentifier,
                                      localizedResultString: self.localizedResultString,
                                      xcbuildSignature: self.xcbuildSignature,
+                                     attachments: self.attachments,
                                      unknown: self.unknown)
     }
 

Sources/XCLogParser/parser/IDEActivityLogSection+Parsing.swift

@@ -156,6 +156,7 @@ extension IDEActivityLogSection {
             uniqueIdentifier: "",
             localizedResultString: "",
             xcbuildSignature: "",
+            attachments: section.attachments,
             unknown: 0)
     }
 

Tests/XCLogParserTests/ActivityParserTests.swift

@@ -97,6 +97,12 @@ class ActivityParserTests: XCTestCase {
                          Token.string("501796C4-6BE4-4F80-9F9D-3269617ECC17"),
                          Token.string("localizedResultString"),
                          Token.string("xcbuildSignature"),
+                         Token.list(1),
+                         Token.classNameRef("IDEFoundation.IDEActivityLogSectionAttachment"),
+                         Token.string("com.apple.dt.ActivityLogSectionAttachment.TaskMetrics"),
+                         Token.int(1),
+                         Token.int(0),
+                         Token.json("{\"wcStartTime\":1,\"maxRSS\":1,\"utime\":1,\"wcDuration\":1,\"stime\":1}"),
                          Token.int(0)
         ]
         return startTokens + logMessageTokens + endTokens
@@ -129,7 +135,13 @@ class ActivityParserTests: XCTestCase {
                            Token.string("79D9C1DE-F736-4743-A7C6-B08ED42A1DFE"),
                            Token.null,
                            Token.null,
-                           Token.list(1)
+                           Token.list(1),
+                           Token.classNameRef("IDEFoundation.IDEActivityLogSectionAttachment"),
+                           Token.string("com.apple.dt.ActivityLogSectionAttachment.TaskMetrics"),
+                           Token.int(1),
+                           Token.int(0),
+                           Token.json("{\"wcStartTime\":1,\"maxRSS\":1,\"utime\":1,\"wcDuration\":1,\"stime\":1}"),
+                           Token.list(1),
         ]
         return startTokens + IDEConsoleItemTokens
     }()

Tests/XCLogParserTests/ClangCompilerParserTests.swift

@@ -95,6 +95,7 @@ class ClangCompilerParserTests: XCTestCase {
                                      uniqueIdentifier: "",
                                      localizedResultString: "",
                                      xcbuildSignature: "",
+                                     attachments: [],
                                      unknown: 0)
     }
 }

Tests/XCLogParserTests/LogManifestTests.swift

@@ -28,7 +28,7 @@ class LogManifestTests: XCTestCase {
 <plist version="1.0">
 <dict>
 <key>logFormatVersion</key>
-<integer>10</integer>
+<integer>11</integer>
 <key>logs</key>
 <dict>
 <key>E8557234-04E4-40E7-A6D6-920AC64BCF21</key>

Tests/XCLogParserTests/ParserTests.swift

@@ -64,6 +64,7 @@ class ParserTests: XCTestCase {
                                                     uniqueIdentifier: uniqueIdentifier,
                                                     localizedResultString: "",
                                                     xcbuildSignature: "",
+                                                    attachments: [],
                                                     unknown: 0)
         let fakeActivityLog = IDEActivityLog(version: 10, mainSection: fakeMainSection)
         let buildStep = try parser.parse(activityLog: fakeActivityLog)
@@ -260,6 +261,7 @@ note: use 'updatedDoSomething' instead\r doSomething()\r        ^~~~~~~~~~~\r
                                                 uniqueIdentifier: "ABC",
                                                 localizedResultString: "",
                                                 xcbuildSignature: "",
+                                                attachments: [],
                                                 unknown: 0)
 
         let parsedTarget = fakeSection.getTargetFromCommand()
@@ -310,6 +312,7 @@ note: use 'updatedDoSomething' instead\r doSomething()\r        ^~~~~~~~~~~\r
                                                  uniqueIdentifier: "uniqueIdentifier",
                                                  localizedResultString: "",
                                                  xcbuildSignature: "",
+                                                 attachments: [],
                                                  unknown: 0)
         return IDEActivityLog(version: 10, mainSection: fakeMainStep)
     }
@@ -555,6 +558,7 @@ CompileSwift normal x86_64 (in target 'Alamofire' from project 'Pods')
                                      uniqueIdentifier: "uniqueIdentifier",
                                      localizedResultString: "",
                                      xcbuildSignature: "",
+                                     attachments: [],
                                      unknown: 0)
     }()
 }

Tests/XCLogParserTests/SwiftCompilerParserTests.swift

@@ -127,6 +127,7 @@ class SwiftCompilerParserTests: XCTestCase {
                                      uniqueIdentifier: "",
                                      localizedResultString: "",
                                      xcbuildSignature: "",
+                                     attachments: [],
                                      unknown: 0)
     }
 

docs/Xcactivitylog Format.md

@@ -15,6 +15,7 @@ A `SLF` document starts with the header `SLF0`. After the header, the document h
 - Class names
 - Class instances
 - Null
+- JSON
 
 A value encoded is formed by 3 parts:
 
@@ -81,6 +82,14 @@ In this case, there are three encoded values:
 
 The elements of an `Array` are `Class instances`
 
+### JSON
+
+- Character type delimiter: `*`
+- Example: `"{\"wcStartTime\":732791618924407,\"maxRSS\":0,\"utime\":798,\"wcDuration\":852,\"stime\":798}"`
+- Left hand side value: An `Integer` with the number of characters that are part of the `JSON` string.
+
+The JSON is of the type `IDEFoundation.IDEActivityLogSectionAttachment`
+
 ### Class name
 
 - Character type delimiter: `%`
@@ -142,6 +151,7 @@ Inside the logs you can find these classes:
 - `DVTTextDocumentLocation`
 - `IDEActivityLogCommandInvocationSection`
 - `IDEActivityLogMajorGroupSection`
+- `IDEFoundation.IDEActivityLogSectionAttachment`
 
 If you search for them, you will find that they belong to the IDEFoundation.framework. A private framework part of Xcode. You can class dump it to get the headers of those classes. Once you have the headers, you will have the name and type of the properties that belong to the class. Now, you can match them to the tokens you got from the log. Some of them are in the same order than in the headers, but for others it will be about trial and error.