diff --git a/Sources/SwiftDocC/Converter/DocumentationContextConverter.swift b/Sources/SwiftDocC/Converter/DocumentationContextConverter.swift index 9f4242e7bc..444d07c29d 100644 --- a/Sources/SwiftDocC/Converter/DocumentationContextConverter.swift +++ b/Sources/SwiftDocC/Converter/DocumentationContextConverter.swift @@ -84,7 +84,7 @@ public class DocumentationContextConverter { /// - Parameters: /// - node: The documentation node to convert. /// - Returns: The render node representation of the documentation node. - public func renderNode(for node: DocumentationNode) throws -> RenderNode? { + public func renderNode(for node: DocumentationNode) -> RenderNode? { guard !node.isVirtual else { return nil } @@ -104,6 +104,6 @@ public class DocumentationContextConverter { @available(*, deprecated, renamed: "renderNode(for:)", message: "Use 'renderNode(for:)' instead. This deprecated API will be removed after 6.1 is released") public func renderNode(for node: DocumentationNode, at source: URL?) throws -> RenderNode? { - return try self.renderNode(for: node) + self.renderNode(for: node) } } diff --git a/Sources/SwiftDocC/DocumentationService/Convert/ConvertService+DataProvider.swift b/Sources/SwiftDocC/DocumentationService/Convert/ConvertService+DataProvider.swift index 91897a13c3..e53352f09d 100644 --- a/Sources/SwiftDocC/DocumentationService/Convert/ConvertService+DataProvider.swift +++ b/Sources/SwiftDocC/DocumentationService/Convert/ConvertService+DataProvider.swift @@ -1,7 +1,7 @@ /* This source file is part of the Swift.org open source project - Copyright (c) 2021 Apple Inc. and the Swift project authors + Copyright (c) 2021-2024 Apple Inc. and the Swift project authors Licensed under Apache License v2.0 with Runtime Library Exception See https://swift.org/LICENSE.txt for license information @@ -11,91 +11,46 @@ import Foundation extension ConvertService { - /// Data provider for a conversion service. - /// - /// This data provider accepts in-memory documentation and assigns unique URLs for each document. - struct InMemoryContentDataProvider: DocumentationWorkspaceDataProvider { - var identifier: String = UUID().uuidString - var bundles: [DocumentationBundle] = [] - + /// Creates a bundle and an associated in-memory data provider from the information of a given convert request + static func makeBundleAndInMemoryDataProvider(_ request: ConvertRequest) -> (bundle: DocumentationBundle, provider: InMemoryDataProvider) { var files: [URL: Data] = [:] - - mutating func registerBundle( - info: DocumentationBundle.Info, - symbolGraphs: [Data], - markupFiles: [Data], - tutorialFiles: [Data], - miscResourceURLs: [URL] - ) { - let symbolGraphURLs = symbolGraphs.map { registerFile(contents: $0, pathExtension: nil) } - let markupFileURLs = markupFiles.map { markupFile in - registerFile( - contents: markupFile, - pathExtension: - DocumentationBundleFileTypes.referenceFileExtension - ) - } + tutorialFiles.map { tutorialFile in - registerFile( - contents: tutorialFile, - pathExtension: - DocumentationBundleFileTypes.tutorialFileExtension - ) - } - - bundles.append( - DocumentationBundle( - info: info, - symbolGraphURLs: symbolGraphURLs, - markupURLs: markupFileURLs, - miscResourceURLs: miscResourceURLs - ) - ) + files.reserveCapacity( + request.symbolGraphs.count + + request.markupFiles.count + + request.tutorialFiles.count + + request.miscResourceURLs.count + ) + for markupFile in request.markupFiles { + files[makeURL().appendingPathExtension(DocumentationBundleFileTypes.referenceFileExtension)] = markupFile } - - private mutating func registerFile(contents: Data, pathExtension: String?) -> URL { - let url = Self.createURL(pathExtension: pathExtension) - files[url] = contents - return url - } - - /// Creates a unique URL for a resource. - /// - /// The URL this function generates for a resource is not derived from the resource itself, because it doesn't need to be. The - /// ``DocumentationWorkspaceDataProvider`` model revolves around retrieving resources by their URL. In our use - /// case, our resources are not file URLs so we generate a URL for each resource. - static private func createURL(pathExtension: String? = nil) -> URL { - var url = URL(string: "docc-service:/\(UUID().uuidString)")! - - if let pathExtension { - url.appendPathExtension(pathExtension) - } - - return url + for tutorialFile in request.tutorialFiles { + files[makeURL().appendingPathExtension(DocumentationBundleFileTypes.tutorialFileExtension)] = tutorialFile } + let markupFileURL = Array(files.keys) - func contentsOfURL(_ url: URL) throws -> Data { - guard let contents = files[url] else { - throw Error.unknownURL(url: url) - } - return contents + var symbolGraphURLs: [URL] = [] + symbolGraphURLs.reserveCapacity(request.symbolGraphs.count) + for symbolGraph in request.symbolGraphs { + let url = makeURL() + symbolGraphURLs.append(url) + files[url] = symbolGraph } - func bundles(options: BundleDiscoveryOptions) throws -> [DocumentationBundle] { - return bundles - } - - enum Error: DescribedError { - case unknownURL(url: URL) - - var errorDescription: String { - switch self { - case .unknownURL(let url): - return """ - Unable to retrieve contents of file at \(url.absoluteString.singleQuoted). - """ - } - } - } + return ( + DocumentationBundle( + info: request.bundleInfo, + symbolGraphURLs: symbolGraphURLs, + markupURLs: markupFileURL, + miscResourceURLs: request.miscResourceURLs + ), + InMemoryDataProvider( + files: files, + fallbackFileManager: FileManager.default + ) + ) } + private static func makeURL() -> URL { + URL(string: "docc-service:/\(UUID().uuidString)")! + } } diff --git a/Sources/SwiftDocC/DocumentationService/Convert/ConvertService+OutputConsumer.swift b/Sources/SwiftDocC/DocumentationService/Convert/ConvertService+OutputConsumer.swift deleted file mode 100644 index aa7d2918b5..0000000000 --- a/Sources/SwiftDocC/DocumentationService/Convert/ConvertService+OutputConsumer.swift +++ /dev/null @@ -1,43 +0,0 @@ -/* - This source file is part of the Swift.org open source project - - Copyright (c) 2021 Apple Inc. and the Swift project authors - Licensed under Apache License v2.0 with Runtime Library Exception - - See https://swift.org/LICENSE.txt for license information - See https://swift.org/CONTRIBUTORS.txt for Swift project authors -*/ - -import Foundation - -extension ConvertService { - /// Output consumer for a conversion service. - /// - /// This consumer stores render nodes so that they can be retrieved by the service after conversion. - class OutputConsumer: ConvertOutputConsumer { - var renderNodes = Synchronized<[RenderNode]>([]) - var renderReferenceStore: RenderReferenceStore? - - func consume(problems: [Problem]) throws {} - - func consume(renderNode: RenderNode) throws { - renderNodes.sync { $0.append(renderNode) } - } - - func consume(assetsInBundle bundle: DocumentationBundle) throws {} - - func consume(linkableElementSummaries: [LinkDestinationSummary]) throws {} - - func consume(indexingRecords: [IndexingRecord]) throws {} - - func consume(assets: [RenderReferenceType : [RenderReference]]) throws {} - - func consume(benchmarks: Benchmark) throws {} - - func consume(documentationCoverageInfo: [CoverageDataEntry]) throws {} - - func consume(renderReferenceStore: RenderReferenceStore) throws { - self.renderReferenceStore = renderReferenceStore - } - } -} diff --git a/Sources/SwiftDocC/DocumentationService/Convert/ConvertService.swift b/Sources/SwiftDocC/DocumentationService/Convert/ConvertService.swift index f859419918..6b93e1f9a9 100644 --- a/Sources/SwiftDocC/DocumentationService/Convert/ConvertService.swift +++ b/Sources/SwiftDocC/DocumentationService/Convert/ConvertService.swift @@ -29,34 +29,21 @@ public struct ConvertService: DocumentationService { public static var handlingTypes = [convertMessageType] - /// Converter that can be injected from a test. - var converter: DocumentationConverterProtocol? - /// A peer server that can be used for resolving links. var linkResolvingServer: DocumentationServer? private let allowArbitraryCatalogDirectories: Bool /// Creates a conversion service, which converts in-memory documentation data. - public init(linkResolvingServer: DocumentationServer? = nil, allowArbitraryCatalogDirectories: Bool) { + public init(linkResolvingServer: DocumentationServer? = nil, allowArbitraryCatalogDirectories: Bool = false) { self.linkResolvingServer = linkResolvingServer self.allowArbitraryCatalogDirectories = allowArbitraryCatalogDirectories } - init( - converter: DocumentationConverterProtocol?, - linkResolvingServer: DocumentationServer? - ) { - self.converter = converter - self.linkResolvingServer = linkResolvingServer - self.allowArbitraryCatalogDirectories = false - } - public func process( _ message: DocumentationServer.Message, completion: @escaping (DocumentationServer.Message) -> () ) { - let conversionResult = retrievePayload(message) .flatMap(decodeRequest) .flatMap(convert) @@ -123,33 +110,6 @@ public struct ConvertService: DocumentationService { // in the request. FeatureFlags.current = request.featureFlags - // Set up the documentation context. - - let workspace = DocumentationWorkspace() - - let provider: DocumentationWorkspaceDataProvider - if let bundleLocation = request.bundleLocation { - // If an on-disk bundle is provided, convert it. - // Additional symbol graphs and markup are ignored for now. - provider = try LocalFileSystemDataProvider( - rootURL: bundleLocation, - allowArbitraryCatalogDirectories: allowArbitraryCatalogDirectories - ) - } else { - // Otherwise, convert the in-memory content. - var inMemoryProvider = InMemoryContentDataProvider() - - inMemoryProvider.registerBundle( - info: request.bundleInfo, - symbolGraphs: request.symbolGraphs, - markupFiles: request.markupFiles, - tutorialFiles: request.tutorialFiles, - miscResourceURLs: request.miscResourceURLs - ) - - provider = inMemoryProvider - } - var configuration = DocumentationContext.Configuration() configuration.convertServiceConfiguration.knownDisambiguatedSymbolPathComponents = request.knownDisambiguatedSymbolPathComponents @@ -177,62 +137,102 @@ public struct ConvertService: DocumentationService { configuration.externalDocumentationConfiguration.globalSymbolResolver = resolver } - let context = try DocumentationContext(dataProvider: workspace, configuration: configuration) + let bundle: DocumentationBundle + let dataProvider: DocumentationBundleDataProvider - var converter = try self.converter ?? DocumentationConverter( - documentationBundleURL: request.bundleLocation ?? URL(fileURLWithPath: "/"), - emitDigest: false, - documentationCoverageOptions: .noCoverage, - currentPlatforms: nil, - workspace: workspace, - context: context, - dataProvider: provider, - externalIDsToConvert: request.externalIDsToConvert, - documentPathsToConvert: request.documentPathsToConvert, - bundleDiscoveryOptions: BundleDiscoveryOptions( + let inputProvider = DocumentationContext.InputsProvider() + if let bundleLocation = request.bundleLocation, + let catalogURL = try inputProvider.findCatalog(startingPoint: bundleLocation, allowArbitraryCatalogDirectories: allowArbitraryCatalogDirectories) + { + let bundleDiscoveryOptions = try BundleDiscoveryOptions( fallbackInfo: request.bundleInfo, additionalSymbolGraphFiles: [] - ), + ) + + bundle = try inputProvider.makeInputs(contentOf: catalogURL, options: bundleDiscoveryOptions) + dataProvider = FileManager.default + } else { + (bundle, dataProvider) = Self.makeBundleAndInMemoryDataProvider(request) + } + + let context = try DocumentationContext(bundle: bundle, dataProvider: dataProvider, configuration: configuration) + + // Precompute the render context + let renderContext = RenderContext(documentationContext: context, bundle: bundle) + + let symbolIdentifiersMeetingRequirementsForExpandedDocumentation: [String]? = request.symbolIdentifiersWithExpandedDocumentation?.compactMap { identifier, expandedDocsRequirement in + guard let documentationNode = context.documentationCache[identifier] else { + return nil + } + + return documentationNode.meetsExpandedDocumentationRequirements(expandedDocsRequirement) ? identifier : nil + } + let converter = DocumentationContextConverter( + bundle: bundle, + context: context, + renderContext: renderContext, emitSymbolSourceFileURIs: request.emitSymbolSourceFileURIs, emitSymbolAccessLevels: true, - symbolIdentifiersWithExpandedDocumentation: request.symbolIdentifiersWithExpandedDocumentation + sourceRepository: nil, + symbolIdentifiersWithExpandedDocumentation: symbolIdentifiersMeetingRequirementsForExpandedDocumentation ) - - // Run the conversion. - - let outputConsumer = OutputConsumer() - let (_, conversionProblems) = try converter.convert(outputConsumer: outputConsumer) - - guard conversionProblems.isEmpty else { - throw ConvertServiceError.conversionError( - underlyingError: DiagnosticConsoleWriter.formattedDescription(for: conversionProblems)) + + let referencesToConvert: [ResolvedTopicReference] + if request.documentPathsToConvert == nil && request.externalIDsToConvert == nil { + // Should build all symbols + referencesToConvert = context.knownPages + } + else { + let symbolReferencesToConvert = Set( + (request.externalIDsToConvert ?? []).compactMap { context.documentationCache.reference(symbolID: $0) } + ) + let documentPathsToConvert = request.documentPathsToConvert ?? [] + + referencesToConvert = context.knownPages.filter { + symbolReferencesToConvert.contains($0) || documentPathsToConvert.contains($0.path) + } + } + + // Accumulate the render nodes + let renderNodes: [RenderNode] = referencesToConvert.concurrentPerform { reference, results in + // Wrap JSON encoding in an autorelease pool to avoid retaining the autoreleased ObjC objects returned by `JSONSerialization` + autoreleasepool { + guard let entity = try? context.entity(with: reference) else { + assertionFailure("The context should always have an entity for each of its `knownPages`") + return + } + + guard let renderNode = converter.renderNode(for: entity) else { + assertionFailure("A non-virtual documentation node should always convert to a render node and the context's `knownPages` already filters out all virtual nodes.") + return + } + + results.append(renderNode) + } } - let references: RenderReferenceStore? + let referenceStore: RenderReferenceStore? if request.includeRenderReferenceStore == true { // Create a reference store and filter non-linkable references. - references = outputConsumer.renderReferenceStore - .map { - var store = referenceStore(for: context, baseReferenceStore: $0) - store.topics = store.topics.filter({ pair in - // Filter non-linkable nodes that do belong to the topic graph. - guard let node = context.topicGraph.nodeWithReference(pair.key) else { - return true - } - return context.topicGraph.isLinkable(node.reference) - }) - return store + var store = self.referenceStore(for: context, baseReferenceStore: renderContext.store) + store.topics = store.topics.filter({ pair in + // Filter non-linkable nodes that do belong to the topic graph. + guard let node = context.topicGraph.nodeWithReference(pair.key) else { + return true } + return context.topicGraph.isLinkable(node.reference) + }) + referenceStore = store } else { - references = nil + referenceStore = nil } - return (outputConsumer.renderNodes.sync({ $0 }), references) + return (renderNodes, referenceStore) }.mapErrorToConvertServiceError { .conversionError(underlyingError: $0.localizedDescription) } } - + /// Encodes a conversion response to send to the client. /// /// - Parameter renderNodes: The render nodes that were produced as part of the conversion. diff --git a/Sources/SwiftDocC/Infrastructure/Bundle Assets/DataAssetManager.swift b/Sources/SwiftDocC/Infrastructure/Bundle Assets/DataAssetManager.swift index 541b658bff..6d78864fcd 100644 --- a/Sources/SwiftDocC/Infrastructure/Bundle Assets/DataAssetManager.swift +++ b/Sources/SwiftDocC/Infrastructure/Bundle Assets/DataAssetManager.swift @@ -60,7 +60,7 @@ struct DataAssetManager { try! NSRegularExpression(pattern: "(?!^)(?<=@)[1|2|3]x(?=\\.\\w*$)") }() - private mutating func referenceMetaInformationForDataURL(_ dataURL: URL, dataProvider: DocumentationContextDataProvider? = nil, bundle documentationBundle: DocumentationBundle? = nil) throws -> (reference: String, traits: DataTraitCollection, metadata: DataAsset.Metadata) { + private mutating func referenceMetaInformationForDataURL(_ dataURL: URL) throws -> (reference: String, traits: DataTraitCollection, metadata: DataAsset.Metadata) { var dataReference = dataURL.path var traitCollection = DataTraitCollection() @@ -106,9 +106,9 @@ struct DataAssetManager { /// Registers a collection of data and determines their trait collection. /// /// Data objects which have a file name ending with '~dark' are associated to their light variant. - mutating func register(data datas: some Collection, dataProvider: DocumentationContextDataProvider? = nil, bundle documentationBundle: DocumentationBundle? = nil) throws { + mutating func register(data datas: some Collection) throws { for dataURL in datas { - let meta = try referenceMetaInformationForDataURL(dataURL, dataProvider: dataProvider, bundle: documentationBundle) + let meta = try referenceMetaInformationForDataURL(dataURL) let referenceURL = URL(fileURLWithPath: meta.reference, isDirectory: false) @@ -129,7 +129,7 @@ struct DataAssetManager { } /// Replaces an existing asset with a new one. - mutating func update(name: String, asset: DataAsset, dataProvider: DocumentationContextDataProvider? = nil, bundle documentationBundle: DocumentationBundle? = nil) { + mutating func update(name: String, asset: DataAsset) { bestKey(forAssetName: name).flatMap({ storage[$0] = asset }) } } diff --git a/Sources/SwiftDocC/Infrastructure/ConvertActionConverter.swift b/Sources/SwiftDocC/Infrastructure/ConvertActionConverter.swift new file mode 100644 index 0000000000..166b0e684c --- /dev/null +++ b/Sources/SwiftDocC/Infrastructure/ConvertActionConverter.swift @@ -0,0 +1,203 @@ +/* + This source file is part of the Swift.org open source project + + Copyright (c) 2024 Apple Inc. and the Swift project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See https://swift.org/LICENSE.txt for license information + See https://swift.org/CONTRIBUTORS.txt for Swift project authors +*/ + +import Foundation + +package enum ConvertActionConverter { + + /// Converts the documentation bundle in the given context and passes its output to a given consumer. + /// + /// - Parameters: + /// - bundle: The documentation bundle to convert. + /// - context: The context that the bundle is a part of. + /// - outputConsumer: The consumer that the conversion passes outputs of the conversion to. + /// - sourceRepository: The source repository where the documentation's sources are hosted. + /// - emitDigest: Whether the conversion should pass additional metadata output––such as linkable entities information, indexing information, or asset references by asset type––to the consumer. + /// - documentationCoverageOptions: The level of experimental documentation coverage information that the conversion should pass to the consumer. + /// - Returns: A list of problems that occurred during the conversion (excluding the problems that the context already encountered). + package static func convert( + bundle: DocumentationBundle, + context: DocumentationContext, + outputConsumer: some ConvertOutputConsumer, + sourceRepository: SourceRepository?, + emitDigest: Bool, + documentationCoverageOptions: DocumentationCoverageOptions + ) throws -> [Problem] { + defer { + context.diagnosticEngine.flush() + } + + let processingDurationMetric = benchmark(begin: Benchmark.Duration(id: "documentation-processing")) + defer { + benchmark(end: processingDurationMetric) + } + + guard !context.problems.containsErrors else { + if emitDigest { + try outputConsumer.consume(problems: context.problems) + } + return [] + } + + // Precompute the render context + let renderContext = RenderContext(documentationContext: context, bundle: bundle) + try outputConsumer.consume(renderReferenceStore: renderContext.store) + + // Copy images, sample files, and other static assets. + try outputConsumer.consume(assetsInBundle: bundle) + + let converter = DocumentationContextConverter( + bundle: bundle, + context: context, + renderContext: renderContext, + sourceRepository: sourceRepository + ) + + // Arrays to gather additional metadata if `emitDigest` is `true`. + var indexingRecords = [IndexingRecord]() + var linkSummaries = [LinkDestinationSummary]() + var assets = [RenderReferenceType : [RenderReference]]() + var coverageInfo = [CoverageDataEntry]() + let coverageFilterClosure = documentationCoverageOptions.generateFilterClosure() + + // An inner function to gather problems for errors encountered during the conversion. + // + // These problems only represent unexpected thrown errors and aren't particularly user-facing but because + // `DocumentationConverter.convert(outputConsumer:)` emits them as diagnostics we do the same here. + func recordProblem(from error: Swift.Error, in problems: inout [Problem], withIdentifier identifier: String) { + let problem = Problem(diagnostic: Diagnostic( + severity: .error, + identifier: "org.swift.docc.documentation-converter.\(identifier)", + summary: error.localizedDescription + ), possibleSolutions: []) + + context.diagnosticEngine.emit(problem) + problems.append(problem) + } + + let resultsSyncQueue = DispatchQueue(label: "Convert Serial Queue", qos: .unspecified, attributes: []) + let resultsGroup = DispatchGroup() + + var conversionProblems: [Problem] = context.knownPages.concurrentPerform { identifier, results in + // If cancelled skip all concurrent conversion work in this block. + guard !Task.isCancelled else { return } + + // Wrap JSON encoding in an autorelease pool to avoid retaining the autoreleased ObjC objects returned by `JSONSerialization` + autoreleasepool { + do { + let entity = try context.entity(with: identifier) + + guard let renderNode = converter.renderNode(for: entity) else { + // No render node was produced for this entity, so just skip it. + return + } + + try outputConsumer.consume(renderNode: renderNode) + + switch documentationCoverageOptions.level { + case .detailed, .brief: + let coverageEntry = try CoverageDataEntry( + documentationNode: entity, + renderNode: renderNode, + context: context + ) + if coverageFilterClosure(coverageEntry) { + resultsGroup.async(queue: resultsSyncQueue) { + coverageInfo.append(coverageEntry) + } + } + case .none: + break + } + + if emitDigest { + let nodeLinkSummaries = entity.externallyLinkableElementSummaries(context: context, renderNode: renderNode, includeTaskGroups: true) + let nodeIndexingRecords = try renderNode.indexingRecords(onPage: identifier) + + resultsGroup.async(queue: resultsSyncQueue) { + assets.merge(renderNode.assetReferences, uniquingKeysWith: +) + linkSummaries.append(contentsOf: nodeLinkSummaries) + indexingRecords.append(contentsOf: nodeIndexingRecords) + } + } else if FeatureFlags.current.isExperimentalLinkHierarchySerializationEnabled { + let nodeLinkSummaries = entity.externallyLinkableElementSummaries(context: context, renderNode: renderNode, includeTaskGroups: false) + + resultsGroup.async(queue: resultsSyncQueue) { + linkSummaries.append(contentsOf: nodeLinkSummaries) + } + } + } catch { + recordProblem(from: error, in: &results, withIdentifier: "render-node") + } + } + } + + // Wait for any concurrent updates to complete. + resultsGroup.wait() + + guard !Task.isCancelled else { return [] } + + // Write various metadata + if emitDigest { + do { + try outputConsumer.consume(linkableElementSummaries: linkSummaries) + try outputConsumer.consume(indexingRecords: indexingRecords) + try outputConsumer.consume(assets: assets) + } catch { + recordProblem(from: error, in: &conversionProblems, withIdentifier: "metadata") + } + } + + if FeatureFlags.current.isExperimentalLinkHierarchySerializationEnabled { + do { + let serializableLinkInformation = try context.linkResolver.localResolver.prepareForSerialization(bundleID: bundle.identifier) + try outputConsumer.consume(linkResolutionInformation: serializableLinkInformation) + + if !emitDigest { + try outputConsumer.consume(linkableElementSummaries: linkSummaries) + } + } catch { + recordProblem(from: error, in: &conversionProblems, withIdentifier: "link-resolver") + } + } + + if emitDigest { + do { + try outputConsumer.consume(problems: context.problems + conversionProblems) + } catch { + recordProblem(from: error, in: &conversionProblems, withIdentifier: "problems") + } + } + + switch documentationCoverageOptions.level { + case .detailed, .brief: + do { + try outputConsumer.consume(documentationCoverageInfo: coverageInfo) + } catch { + recordProblem(from: error, in: &conversionProblems, withIdentifier: "coverage") + } + case .none: + break + } + + try outputConsumer.consume(buildMetadata: BuildMetadata(bundleDisplayName: bundle.displayName, bundleIdentifier: bundle.identifier)) + + // Log the finalized topic graph checksum. + benchmark(add: Benchmark.TopicGraphHash(context: context)) + // Log the finalized list of topic anchor sections. + benchmark(add: Benchmark.TopicAnchorHash(context: context)) + // Log the finalized external topics checksum. + benchmark(add: Benchmark.ExternalTopicsHash(context: context)) + // Log the peak memory. + benchmark(add: Benchmark.PeakMemory()) + + return conversionProblems + } +} diff --git a/Sources/SwiftDocC/Infrastructure/DocumentationContext.swift b/Sources/SwiftDocC/Infrastructure/DocumentationContext.swift index e23fc3cd35..42f77f1714 100644 --- a/Sources/SwiftDocC/Infrastructure/DocumentationContext.swift +++ b/Sources/SwiftDocC/Infrastructure/DocumentationContext.swift @@ -113,9 +113,39 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate { /// A class that resolves documentation links by orchestrating calls to other link resolver implementations. public var linkResolver = LinkResolver() + private enum _Provider { + case legacy(DocumentationContextDataProvider) + case new(DocumentationBundleDataProvider) + } + private var dataProvider: _Provider + /// The provider of documentation bundles for this context. - var dataProvider: DocumentationContextDataProvider + var _legacyDataProvider: DocumentationContextDataProvider! { + get { + switch dataProvider { + case .legacy(let legacyDataProvider): + legacyDataProvider + case .new: + nil + } + } + set { + dataProvider = .legacy(newValue) + } + } + func contentsOfURL(_ url: URL, in bundle: DocumentationBundle) throws -> Data { + switch dataProvider { + case .legacy(let legacyDataProvider): + return try legacyDataProvider.contentsOfURL(url, in: bundle) + case .new(let dataProvider): + assert(self.bundle?.identifier == bundle.identifier, "New code shouldn't pass unknown bundle identifiers to 'DocumentationContext.bundle(identifier:)'.") + return try dataProvider.contents(of: url) + } + } + + var bundle: DocumentationBundle? + /// A collection of configuration for this context. public package(set) var configuration: Configuration { get { _configuration } @@ -260,17 +290,40 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate { diagnosticEngine: DiagnosticEngine = .init(), configuration: Configuration = .init() ) throws { - self.dataProvider = dataProvider + self.dataProvider = .legacy(dataProvider) self.diagnosticEngine = diagnosticEngine self._configuration = configuration - self.dataProvider.delegate = self + _legacyDataProvider.delegate = self for bundle in dataProvider.bundles.values { try register(bundle) } } - + + /// Initializes a documentation context with a given `bundle`. + /// + /// - Parameters: + /// - bundle: The bundle to register with the context. + /// - fileManager: The file manager that the context uses to read files from the bundle. + /// - diagnosticEngine: The pre-configured engine that will collect problems encountered during compilation. + /// - configuration: A collection of configuration for the created context. + /// - Throws: If an error is encountered while registering a documentation bundle. + package init( + bundle: DocumentationBundle, + dataProvider: DocumentationBundleDataProvider, + diagnosticEngine: DiagnosticEngine = .init(), + configuration: Configuration = .init() + ) throws { + self.bundle = bundle + self.dataProvider = .new(dataProvider) + self.diagnosticEngine = diagnosticEngine + self._configuration = configuration + + ResolvedTopicReference.enableReferenceCaching(for: bundle.identifier) + try register(bundle) + } + /// Respond to a new `bundle` being added to the `dataProvider` by registering it. /// /// - Parameters: @@ -302,12 +355,23 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate { /// The documentation bundles that are currently registered with the context. public var registeredBundles: some Collection { - return dataProvider.bundles.values + switch dataProvider { + case .legacy(let legacyDataProvider): + Array(legacyDataProvider.bundles.values) + case .new: + bundle.map { [$0] } ?? [] + } } /// Returns the `DocumentationBundle` with the given `identifier` if it's registered with the context, otherwise `nil`. public func bundle(identifier: String) -> DocumentationBundle? { - return dataProvider.bundles[identifier] + switch dataProvider { + case .legacy(let legacyDataProvider): + return legacyDataProvider.bundles[identifier] + case .new: + assert(bundle?.identifier == identifier, "New code shouldn't pass unknown bundle identifiers to 'DocumentationContext.bundle(identifier:)'.") + return bundle?.identifier == identifier ? bundle : nil + } } /// Perform semantic analysis on a given `document` at a given `source` location and append any problems found to `problems`. @@ -789,7 +853,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate { guard decodeError.sync({ $0 == nil }) else { return } do { - let data = try dataProvider.contentsOfURL(url, in: bundle) + let data = try contentsOfURL(url, in: bundle) let source = String(decoding: data, as: UTF8.self) let document = Document(parsing: source, source: url, options: [.parseBlockDirectives, .parseSymbolLinks]) @@ -1690,8 +1754,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate { private func registerMiscResources(from bundle: DocumentationBundle) throws { let miscResources = Set(bundle.miscResourceURLs) - try assetManagers[bundle.identifier, default: DataAssetManager()] - .register(data: miscResources, dataProvider: dataProvider, bundle: bundle) + try assetManagers[bundle.identifier, default: DataAssetManager()].register(data: miscResources) } private func registeredAssets(withExtensions extensions: Set? = nil, inContexts contexts: [DataAsset.Context] = DataAsset.Context.allCases, forBundleID bundleIdentifier: BundleIdentifier) -> [DataAsset] { @@ -2071,7 +2134,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate { discoveryGroup.async(queue: discoveryQueue) { [unowned self] in symbolGraphLoader = SymbolGraphLoader( bundle: bundle, - dataProvider: self.dataProvider, + dataLoader: { try self.contentsOfURL($0, in: $1) }, symbolGraphTransformer: configuration.convertServiceConfiguration.symbolGraphTransformer ) @@ -2617,7 +2680,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate { let resource = asset.data(bestMatching: trait) - return try dataProvider.contentsOfURL(resource.url, in: bundle) + return try contentsOfURL(resource.url, in: bundle) } /// Returns true if a resource with the given identifier exists in the registered bundle. @@ -2880,7 +2943,12 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate { /// - parent: The topic the code listing reference appears in. @available(*, deprecated, message: "This deprecated API will be removed after 6.1 is released") public func resolveCodeListing(_ unresolvedCodeListingReference: UnresolvedCodeListingReference, in parent: ResolvedTopicReference) -> AttributedCodeListing? { - return dataProvider.bundles[parent.bundleIdentifier]?.attributedCodeListings[unresolvedCodeListingReference.identifier] + switch dataProvider { + case .legacy(let legacyDataProvider): + legacyDataProvider.bundles[parent.bundleIdentifier]?.attributedCodeListings[unresolvedCodeListingReference.identifier] + case .new: + nil + } } /// The references of all nodes in the topic graph. diff --git a/Sources/SwiftDocC/Infrastructure/DocumentationConverter.swift b/Sources/SwiftDocC/Infrastructure/DocumentationConverter.swift index 4d5724314a..761ef4614b 100644 --- a/Sources/SwiftDocC/Infrastructure/DocumentationConverter.swift +++ b/Sources/SwiftDocC/Infrastructure/DocumentationConverter.swift @@ -288,7 +288,7 @@ public struct DocumentationConverter: DocumentationConverterProtocol { return } - guard let renderNode = try converter.renderNode(for: entity) else { + guard let renderNode = converter.renderNode(for: entity) else { // No render node was produced for this entity, so just skip it. return } diff --git a/Sources/SwiftDocC/Infrastructure/Input Discovery/DocumentationBundle+DataProvider.swift b/Sources/SwiftDocC/Infrastructure/Input Discovery/DocumentationBundle+DataProvider.swift new file mode 100644 index 0000000000..946f2cf835 --- /dev/null +++ b/Sources/SwiftDocC/Infrastructure/Input Discovery/DocumentationBundle+DataProvider.swift @@ -0,0 +1,50 @@ +/* + This source file is part of the Swift.org open source project + + Copyright (c) 2024 Apple Inc. and the Swift project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See https://swift.org/LICENSE.txt for license information + See https://swift.org/CONTRIBUTORS.txt for Swift project authors +*/ + +import Foundation + +/// A type that provides data for files in a documentation bundle. +package protocol DocumentationBundleDataProvider { + /// Returns the contents of the file at the specified location. + /// + /// - Parameter url: The url of the file to read. + /// - Throws: If the provider failed to read the file. + func contents(of url: URL) throws -> Data +} + +/// A type that provides in-memory data for files in a bundle. +struct InMemoryDataProvider: DocumentationBundleDataProvider { + private let files: [URL: Data] + private let fileManager: FileManagerProtocol? + + /// Creates a data provider with a collection of in-memory files. + /// + /// If the provider doesn't have in-memory data for a given file it will use the file manager as a fallback. + /// This allows the in-memory provider to be used for a mix of in-memory and on-disk content. + /// + /// - Parameters: + /// - files: The in-memory data for the files that provider can provide + /// - fallbackFileManager: The file manager that the provider uses as a fallback for any file it doesn't have in-memory data for. + init(files: [URL: Data], fallbackFileManager: FileManagerProtocol?) { + self.files = files + self.fileManager = fallbackFileManager + } + + func contents(of url: URL) throws -> Data { + if let inMemoryResult = files[url] { + return inMemoryResult + } + if let onDiskResult = try fileManager?.contents(of: url) { + return onDiskResult + } + + throw CocoaError(.fileReadNoSuchFile, userInfo: [NSFilePathErrorKey: url.path]) + } +} diff --git a/Sources/SwiftDocC/Infrastructure/Input Discovery/DocumentationInputsProvider.swift b/Sources/SwiftDocC/Infrastructure/Input Discovery/DocumentationInputsProvider.swift index c0bc78c89e..cddbde7bf3 100644 --- a/Sources/SwiftDocC/Infrastructure/Input Discovery/DocumentationInputsProvider.swift +++ b/Sources/SwiftDocC/Infrastructure/Input Discovery/DocumentationInputsProvider.swift @@ -38,7 +38,7 @@ extension DocumentationContext.InputsProvider { private typealias FileTypes = DocumentationBundleFileTypes /// A discovered documentation catalog. - package struct CatalogURL { + struct CatalogURL { let url: URL } @@ -60,7 +60,7 @@ extension DocumentationContext.InputsProvider { /// - allowArbitraryCatalogDirectories: Whether to treat the starting point as a documentation catalog if the provider doesn't find an actual catalog on the file system. /// - Returns: The found documentation catalog. /// - Throws: If the directory hierarchy contains more than one documentation catalog. - package func findCatalog( + func findCatalog( startingPoint: URL, allowArbitraryCatalogDirectories: Bool = false ) throws -> CatalogURL? { @@ -91,7 +91,7 @@ extension DocumentationContext.InputsProvider { } } -// MARK: Inputs creation +// MARK: Create from catalog extension DocumentationContext { package typealias Inputs = DocumentationBundle @@ -107,7 +107,7 @@ extension DocumentationContext.InputsProvider { /// - catalogURL: The location of a discovered documentation catalog. /// - options: Options to configure how the provider creates the documentation inputs. /// - Returns: Inputs that categorize the files of the given catalog. - package func makeInputs(contentOf catalogURL: CatalogURL, options: Options) throws -> DocumentationContext.Inputs { + func makeInputs(contentOf catalogURL: CatalogURL, options: Options) throws -> DocumentationContext.Inputs { let url = catalogURL.url let shallowContent = try fileManager.contentsOfDirectory(at: url, options: [.skipsHiddenFiles]).files let infoPlistData = try shallowContent @@ -168,7 +168,7 @@ extension DocumentationContext.InputsProvider { /// /// - Parameter options: Options to configure how the provider creates the documentation inputs. /// - Returns: Inputs that categorize the files of the given catalog. - package func makeInputsFromSymbolGraphs(options: Options) throws -> DocumentationContext.Inputs? { + func makeInputsFromSymbolGraphs(options: Options) throws -> InputsAndDataProvider? { guard !options.additionalSymbolGraphFiles.isEmpty else { return nil } @@ -183,32 +183,36 @@ extension DocumentationContext.InputsProvider { let derivedDisplayName = moduleNames.count == 1 ? moduleNames.first : nil let info = try DocumentationContext.Inputs.Info(bundleDiscoveryOptions: options, derivedDisplayName: derivedDisplayName) - - var topLevelPages: [URL] = [] - if moduleNames.count == 1, let moduleName = moduleNames.first, moduleName != info.displayName { - let tempURL = fileManager.uniqueTemporaryDirectory() - try? fileManager.createDirectory(at: tempURL, withIntermediateDirectories: true, attributes: nil) - - let url = tempURL.appendingPathComponent("\(moduleName).md") - topLevelPages.append(url) - try fileManager.createFile( - at: url, - contents: Data(""" + + let topLevelPages: [URL] + let provider: DocumentationBundleDataProvider + if moduleNames.count == 1, let moduleName = moduleNames.first, moduleName != info.displayName, let url = URL(string: "in-memory-data://\(moduleName).md") { + let synthesizedExtensionFileData = Data(""" # ``\(moduleName)`` @Metadata { @DisplayName("\(info.displayName)") } - """.utf8), - options: .atomic + """.utf8) + + topLevelPages = [url] + provider = InMemoryDataProvider( + files: [url: synthesizedExtensionFileData], + fallbackFileManager: fileManager ) + } else { + topLevelPages = [] + provider = fileManager } - return DocumentationBundle( - info: info, - symbolGraphURLs: options.additionalSymbolGraphFiles, - markupURLs: topLevelPages, - miscResourceURLs: [] + return ( + inputs: DocumentationBundle( + info: info, + symbolGraphURLs: options.additionalSymbolGraphFiles, + markupURLs: topLevelPages, + miscResourceURLs: [] + ), + dataProvider: provider ) } } @@ -230,25 +234,91 @@ private struct SymbolGraphModuleContainer: Decodable { // MARK: Discover and create extension DocumentationContext.InputsProvider { + /// A pair of documentation inputs and a corresponding data provider for those input files. + package typealias InputsAndDataProvider = (inputs: DocumentationContext.Inputs, dataProvider: DocumentationBundleDataProvider) + /// Traverses the file system from the given starting point to find a documentation catalog and creates a collection of documentation inputs from that catalog. /// /// If the provider can't find a catalog, it will try to create documentation inputs from the option's symbol graph files. /// + /// If the provider can't create documentation inputs it will raise an error with high level suggestions on how the caller can provide the missing information. + /// /// - Parameters: /// - startingPoint: The top of the directory hierarchy that the provider traverses to find a documentation catalog. /// - allowArbitraryCatalogDirectories: Whether to treat the starting point as a documentation catalog if the provider doesn't find an actual catalog on the file system. /// - options: Options to configure how the provider creates the documentation inputs. - /// - Returns: The documentation inputs for the found documentation catalog, or `nil` if the directory hierarchy doesn't contain a catalog. - /// - Throws: If the directory hierarchy contains more than one documentation catalog. - package func inputs( - startingPoint: URL, + /// - Returns: A pair of documentation inputs and a corresponding data provider for those input files. + package func inputsAndDataProvider( + startingPoint: URL?, allowArbitraryCatalogDirectories: Bool = false, options: Options - ) throws -> DocumentationContext.Inputs? { - if let catalogURL = try findCatalog(startingPoint: startingPoint, allowArbitraryCatalogDirectories: allowArbitraryCatalogDirectories) { - try makeInputs(contentOf: catalogURL, options: options) - } else { - try makeInputsFromSymbolGraphs(options: options) + ) throws -> InputsAndDataProvider { + if let startingPoint, let catalogURL = try findCatalog(startingPoint: startingPoint, allowArbitraryCatalogDirectories: allowArbitraryCatalogDirectories) { + return (inputs: try makeInputs(contentOf: catalogURL, options: options), dataProvider: fileManager) + } + + do { + if let generated = try makeInputsFromSymbolGraphs(options: options) { + return generated + } + } catch { + throw InputsFromSymbolGraphError(underlyingError: error) + } + + throw NotEnoughInformationError(startingPoint: startingPoint, additionalSymbolGraphFiles: options.additionalSymbolGraphFiles, allowArbitraryCatalogDirectories: allowArbitraryCatalogDirectories) + } + + private static let insufficientInputsErrorMessageBase = "The information provided as command line arguments isn't enough to generate documentation.\n" + + struct InputsFromSymbolGraphError: DescribedError { + var underlyingError: Error + + var errorDescription: String { + "\(DocumentationContext.InputsProvider.insufficientInputsErrorMessageBase)\n\(underlyingError.localizedDescription)" + } + } + + struct NotEnoughInformationError: DescribedError { + var startingPoint: URL? + var additionalSymbolGraphFiles: [URL] + var allowArbitraryCatalogDirectories: Bool + + var errorDescription: String { + var message = DocumentationContext.InputsProvider.insufficientInputsErrorMessageBase + if let startingPoint { + message.append(""" + + The `` positional argument \(startingPoint.path.singleQuoted) isn't a documentation catalog (`.docc` directory) \ + and its directory sub-hierarchy doesn't contain a documentation catalog (`.docc` directory). + + """) + if !allowArbitraryCatalogDirectories { + message.append(""" + + To build documentation for the files in \(startingPoint.path.singleQuoted), \ + either give it a `.docc` file extension to make it a documentation catalog \ + or pass the `--allow-arbitrary-catalog-directories` flag to treat it as a documentation catalog, \ + regardless of file extension. + + """) + } + } + if additionalSymbolGraphFiles.isEmpty { + if CommandLine.arguments.contains("--additional-symbol-graph-dir") { + message.append(""" + + The provided `--additional-symbol-graph-dir` directory doesn't contain any symbol graph files (with a `.symbols.json` file extension). + """) + } else { + message.append(""" + + To build documentation using only in-source documentation comments, \ + pass a directory of symbol graph files (with a `.symbols.json` file extension) for the `--additional-symbol-graph-dir` argument. + """) + } + } + + return message } } } diff --git a/Sources/SwiftDocC/Infrastructure/Symbol Graph/SymbolGraphLoader.swift b/Sources/SwiftDocC/Infrastructure/Symbol Graph/SymbolGraphLoader.swift index 8893a0e09c..89c7da0258 100644 --- a/Sources/SwiftDocC/Infrastructure/Symbol Graph/SymbolGraphLoader.swift +++ b/Sources/SwiftDocC/Infrastructure/Symbol Graph/SymbolGraphLoader.swift @@ -19,24 +19,43 @@ struct SymbolGraphLoader { private(set) var symbolGraphs: [URL: SymbolKit.SymbolGraph] = [:] private(set) var unifiedGraphs: [String: SymbolKit.UnifiedSymbolGraph] = [:] private(set) var graphLocations: [String: [SymbolKit.GraphCollector.GraphKind]] = [:] - private var dataProvider: DocumentationContextDataProvider + // FIXME: After 6.2, when we no longer have `DocumentationContextDataProvider` we can simply this code to not use a closure to read data. + private var dataLoader: (URL, DocumentationBundle) throws -> Data private var bundle: DocumentationBundle private var symbolGraphTransformer: ((inout SymbolGraph) -> ())? = nil + /// Creates a new symbol graph loader + /// - Parameters: + /// - bundle: The documentation bundle from which to load symbol graphs. + /// - dataLoader: A closure that the loader uses to read symbol graph data. + /// - symbolGraphTransformer: An optional closure that transforms the symbol graph after the loader decodes it. + init( + bundle: DocumentationBundle, + dataLoader: @escaping (URL, DocumentationBundle) throws -> Data, + symbolGraphTransformer: ((inout SymbolGraph) -> ())? = nil + ) { + self.bundle = bundle + self.dataLoader = dataLoader + self.symbolGraphTransformer = symbolGraphTransformer + } + /// Creates a new loader, initialized with the given bundle. /// - Parameters: /// - bundle: The documentation bundle from which to load symbol graphs. /// - dataProvider: A data provider in the bundle's context. + /// - symbolGraphTransformer: An optional closure that transforms the symbol graph after the loader decodes it. init( bundle: DocumentationBundle, dataProvider: DocumentationContextDataProvider, symbolGraphTransformer: ((inout SymbolGraph) -> ())? = nil ) { self.bundle = bundle - self.dataProvider = dataProvider + self.dataLoader = { url, bundle in + try dataProvider.contentsOfURL(url, in: bundle) + } self.symbolGraphTransformer = symbolGraphTransformer } - + /// A strategy to decode symbol graphs. enum DecodingConcurrencyStrategy { /// Decode all symbol graph files on separate threads concurrently. @@ -56,17 +75,15 @@ struct SymbolGraphLoader { var loadedGraphs = [URL: (usesExtensionSymbolFormat: Bool?, graph: SymbolKit.SymbolGraph)]() var loadError: Error? - let bundle = self.bundle - let dataProvider = self.dataProvider - - let loadGraphAtURL: (URL) -> Void = { symbolGraphURL in + + let loadGraphAtURL: (URL) -> Void = { [dataLoader, bundle] symbolGraphURL in // Bail out in case a symbol graph has already errored guard loadError == nil else { return } do { // Load and decode a single symbol graph file - let data = try dataProvider.contentsOfURL(symbolGraphURL, in: bundle) - + let data = try dataLoader(symbolGraphURL, bundle) + var symbolGraph: SymbolGraph switch decodingStrategy { diff --git a/Sources/SwiftDocC/Model/Rendering/RenderNodeTranslator.swift b/Sources/SwiftDocC/Model/Rendering/RenderNodeTranslator.swift index c6440c681c..f134c2bc75 100644 --- a/Sources/SwiftDocC/Model/Rendering/RenderNodeTranslator.swift +++ b/Sources/SwiftDocC/Model/Rendering/RenderNodeTranslator.swift @@ -1757,7 +1757,7 @@ public struct RenderNodeTranslator: SemanticVisitor { let downloadReference: DownloadReference do { let downloadURL = resolvedAssets.variants.first!.value - let downloadData = try context.dataProvider.contentsOfURL(downloadURL, in: bundle) + let downloadData = try context.contentsOfURL(downloadURL, in: bundle) downloadReference = DownloadReference(identifier: mediaReference, renderURL: downloadURL, checksum: Checksum.sha512(of: downloadData)) diff --git a/Sources/SwiftDocC/Utility/FileManagerProtocol.swift b/Sources/SwiftDocC/Utility/FileManagerProtocol.swift index 527f150bb5..7f42f5b554 100644 --- a/Sources/SwiftDocC/Utility/FileManagerProtocol.swift +++ b/Sources/SwiftDocC/Utility/FileManagerProtocol.swift @@ -22,7 +22,7 @@ import Foundation /// Should you need a file system with a different storage, create your own /// protocol implementations to manage files in memory, /// on a network, in a database, or elsewhere. -package protocol FileManagerProtocol { +package protocol FileManagerProtocol: DocumentationBundleDataProvider { /// Returns the data content of a file at the given path, if it exists. func contents(atPath: String) -> Data? diff --git a/Sources/SwiftDocC/Utility/FoundationExtensions/Dictionary+TypedValues.swift b/Sources/SwiftDocC/Utility/FoundationExtensions/Dictionary+TypedValues.swift index 5ea6fbbd41..6ca1858aca 100644 --- a/Sources/SwiftDocC/Utility/FoundationExtensions/Dictionary+TypedValues.swift +++ b/Sources/SwiftDocC/Utility/FoundationExtensions/Dictionary+TypedValues.swift @@ -62,27 +62,23 @@ enum TypedValueError: DescribedError { case let .wrongType(key, expected, actual): return "Type mismatch for key '\(key.singleQuoted)'. Expected '\(expected)', but found '\(actual)'." case .missingRequiredKeys(let keys): - var errorMessage = "" - - for key in keys { - errorMessage += """ - \n + return keys.map { key in + var errorMessage = """ Missing value for \(key.rawValue.singleQuoted). """ if let argumentName = key.argumentName { - errorMessage += """ - Use the \(argumentName.singleQuoted) argument or add \(key.rawValue.singleQuoted) to the bundle Info.plist. - """ + errorMessage.append(""" + Use the \(argumentName.singleQuoted) argument or add \(key.rawValue.singleQuoted) to the catalog's Info.plist. + """) } else { - errorMessage += """ - Add \(key.rawValue.singleQuoted) to the bundle Info.plist. - """ + errorMessage.append(""" + Add \(key.rawValue.singleQuoted) to the catalog's Info.plist. + """) } - } - - return errorMessage + return errorMessage + }.joined(separator: "\n\n") } } } diff --git a/Sources/SwiftDocCTestUtilities/TestFileSystem.swift b/Sources/SwiftDocCTestUtilities/TestFileSystem.swift index cd0ce4c339..c111141ca1 100644 --- a/Sources/SwiftDocCTestUtilities/TestFileSystem.swift +++ b/Sources/SwiftDocCTestUtilities/TestFileSystem.swift @@ -46,9 +46,8 @@ package class TestFileSystem: FileManagerProtocol, DocumentationWorkspaceDataPro package var identifier: String = UUID().uuidString package func bundles(options: BundleDiscoveryOptions) throws -> [DocumentationBundle] { - try DocumentationContext.InputsProvider(fileManager: self) - .inputs(startingPoint: URL(fileURLWithPath: currentDirectoryPath), options: options) - .map { [$0] } ?? [] + [try DocumentationContext.InputsProvider(fileManager: self) + .inputsAndDataProvider(startingPoint: URL(fileURLWithPath: currentDirectoryPath), options: options).inputs] } /// Thread safe access to the file system. @@ -100,11 +99,32 @@ package class TestFileSystem: FileManagerProtocol, DocumentationWorkspaceDataPro case let folder as Folder: result[at.appendingPathComponent(folder.name).path] = Self.folderFixtureData result.merge(try filesIn(folder: folder, at: at.appendingPathComponent(folder.name)), uniquingKeysWith: +) + case let file as File & DataRepresentable: result[at.appendingPathComponent(file.name).path] = try file.data() if let copy = file as? CopyOfFile { result[copy.original.path] = try file.data() } + + case let folder as CopyOfFolder: + // These are copies of real file and folders so we use `FileManager` here to read their content + let enumerator = FileManager.default.enumerator(at: folder.original, includingPropertiesForKeys: [.isDirectoryKey], options: .skipsHiddenFiles)! + + let contentBase = at.appendingPathComponent(folder.name) + result[contentBase.path] = Self.folderFixtureData + + let basePathString = folder.original.standardizedFileURL.deletingLastPathComponent().path + for case let url as URL in enumerator where folder.shouldCopyFile(url) { + let data = try url.resourceValues(forKeys: [.isDirectoryKey]).isDirectory == true + ? Self.folderFixtureData + : try Data(contentsOf: url) + + assert(url.standardizedFileURL.path.hasPrefix(basePathString)) + let relativePath = String(url.standardizedFileURL.path.dropFirst(basePathString.count)) + + result[at.appendingPathComponent(relativePath).path] = data + } + default: break } } diff --git a/Sources/SwiftDocCUtilities/Action/Actions/Convert/ConvertAction.swift b/Sources/SwiftDocCUtilities/Action/Actions/Convert/ConvertAction.swift index bb650ec591..35bb2280e3 100644 --- a/Sources/SwiftDocCUtilities/Action/Actions/Convert/ConvertAction.swift +++ b/Sources/SwiftDocCUtilities/Action/Actions/Convert/ConvertAction.swift @@ -15,53 +15,26 @@ import SwiftDocC /// An action that converts a source bundle into compiled documentation. public struct ConvertAction: AsyncAction { - enum Error: DescribedError { - case doesNotContainBundle(url: URL) - case cancelPending - var errorDescription: String { - switch self { - case .doesNotContainBundle(let url): - return """ - The directory at '\(url)' and its subdirectories do not contain at least one valid documentation \ - bundle. A documentation bundle is a directory ending in `.docc`. Pass \ - `--allow-arbitrary-catalog-directories` flag to convert a directory without a `.docc` extension. - """ - case .cancelPending: - return "The action is already in the process of being cancelled." - } - } - } - let rootURL: URL? - let outOfProcessResolver: OutOfProcessReferenceResolver? - let analyze: Bool let targetDirectory: URL let htmlTemplateDirectory: URL? - let emitDigest: Bool - let inheritDocs: Bool + + private let emitDigest: Bool let treatWarningsAsErrors: Bool let experimentalEnableCustomTemplates: Bool - let experimentalModifyCatalogWithGeneratedCuration: Bool + private let experimentalModifyCatalogWithGeneratedCuration: Bool let buildLMDBIndex: Bool - let documentationCoverageOptions: DocumentationCoverageOptions - let diagnosticLevel: DiagnosticSeverity + private let documentationCoverageOptions: DocumentationCoverageOptions let diagnosticEngine: DiagnosticEngine - let transformForStaticHosting: Bool - let hostingBasePath: String? + private let transformForStaticHosting: Bool + private let hostingBasePath: String? let sourceRepository: SourceRepository? - private(set) var context: DocumentationContext - private let workspace: DocumentationWorkspace - private var currentDataProvider: DocumentationWorkspaceDataProvider? - private var injectedDataProvider: DocumentationWorkspaceDataProvider? private var fileManager: FileManagerProtocol private let temporaryDirectory: URL - var converter: DocumentationConverter - - private var durationMetric: Benchmark.Duration? private let diagnosticWriterOptions: (formatting: DiagnosticFormattingOptions, baseURL: URL) /// Initializes the action with the given validated options, creates or uses the given action workspace & context. @@ -106,9 +79,9 @@ public struct ConvertAction: AsyncAction { emitDigest: Bool, currentPlatforms: [String : PlatformVersion]?, buildIndex: Bool = false, - workspace: DocumentationWorkspace = DocumentationWorkspace(), - context: DocumentationContext? = nil, - dataProvider: DocumentationWorkspaceDataProvider? = nil, + workspace _: DocumentationWorkspace = DocumentationWorkspace(), + context _: DocumentationContext? = nil, + dataProvider _: DocumentationWorkspaceDataProvider? = nil, fileManager: FileManagerProtocol = FileManager.default, temporaryDirectory: URL, documentationCoverageOptions: DocumentationCoverageOptions = .noCoverage, @@ -128,14 +101,10 @@ public struct ConvertAction: AsyncAction { dependencies: [URL] = [] ) throws { self.rootURL = documentationBundleURL - self.outOfProcessResolver = outOfProcessResolver - self.analyze = analyze self.targetDirectory = targetDirectory self.htmlTemplateDirectory = htmlTemplateDirectory self.emitDigest = emitDigest self.buildLMDBIndex = buildIndex - self.workspace = workspace - self.injectedDataProvider = dataProvider self.fileManager = fileManager self.temporaryDirectory = temporaryDirectory self.documentationCoverageOptions = documentationCoverageOptions @@ -161,7 +130,6 @@ public struct ConvertAction: AsyncAction { documentationBundleURL ?? URL(fileURLWithPath: fileManager.currentDirectoryPath) ) - self.inheritDocs = inheritDocs self.treatWarningsAsErrors = treatWarningsAsErrors self.experimentalEnableCustomTemplates = experimentalEnableCustomTemplates @@ -174,9 +142,8 @@ public struct ConvertAction: AsyncAction { } self.diagnosticEngine = engine - self.diagnosticLevel = filterLevel - var configuration = context?.configuration ?? DocumentationContext.Configuration() + var configuration = DocumentationContext.Configuration() configuration.externalMetadata.diagnosticLevel = filterLevel // Inject current platform versions if provided @@ -198,52 +165,38 @@ public struct ConvertAction: AsyncAction { break } - let dataProvider: DocumentationWorkspaceDataProvider - if let injectedDataProvider { - dataProvider = injectedDataProvider - } else if let rootURL { - dataProvider = try LocalFileSystemDataProvider( - rootURL: rootURL, - allowArbitraryCatalogDirectories: allowArbitraryCatalogDirectories - ) - } else { - configuration.externalMetadata.isGeneratedBundle = true - dataProvider = GeneratedDataProvider(symbolGraphDataLoader: { url in - fileManager.contents(atPath: url.path) - }) - } - if let outOfProcessResolver { configuration.externalDocumentationConfiguration.sources[outOfProcessResolver.bundleIdentifier] = outOfProcessResolver configuration.externalDocumentationConfiguration.globalSymbolResolver = outOfProcessResolver } configuration.externalDocumentationConfiguration.dependencyArchives = dependencies - (context as _DeprecatedConfigurationSetAccess?)?.configuration = configuration - - self.context = try context ?? DocumentationContext(dataProvider: workspace, diagnosticEngine: engine, configuration: configuration) - - self.converter = DocumentationConverter( - documentationBundleURL: documentationBundleURL, - emitDigest: emitDigest, - documentationCoverageOptions: documentationCoverageOptions, - currentPlatforms: currentPlatforms, - workspace: workspace, - context: self.context, - dataProvider: dataProvider, - bundleDiscoveryOptions: bundleDiscoveryOptions, - sourceRepository: sourceRepository, - diagnosticEngine: self.diagnosticEngine, - experimentalModifyCatalogWithGeneratedCuration: experimentalModifyCatalogWithGeneratedCuration + let inputProvider = DocumentationContext.InputsProvider(fileManager: fileManager) + let (bundle, dataProvider) = try inputProvider.inputsAndDataProvider( + startingPoint: documentationBundleURL, + allowArbitraryCatalogDirectories: allowArbitraryCatalogDirectories, + options: bundleDiscoveryOptions ) + self.configuration = configuration + + self.bundle = bundle + self.dataProvider = dataProvider } + let configuration: DocumentationContext.Configuration + private let bundle: DocumentationBundle + private let dataProvider: DocumentationBundleDataProvider + /// A block of extra work that tests perform to affect the time it takes to convert documentation var _extraTestWork: (() async -> Void)? /// Converts each eligible file from the source documentation bundle, /// saves the results in the given output alongside the template files. - public mutating func perform(logHandle: inout LogHandle) async throws -> ActionResult { + public func perform(logHandle: inout LogHandle) async throws -> ActionResult { + try await perform(logHandle: &logHandle).0 + } + + func perform(logHandle: inout LogHandle) async throws -> (ActionResult, DocumentationContext) { // FIXME: Use `defer` again when the asynchronous defer-statement miscompilation (rdar://137774949) is fixed. let temporaryFolder = try createTempFolder(with: htmlTemplateDirectory) do { @@ -258,7 +211,7 @@ public struct ConvertAction: AsyncAction { } } - private mutating func _perform(logHandle: inout LogHandle, temporaryFolder: URL) async throws -> ActionResult { + private func _perform(logHandle: inout LogHandle, temporaryFolder: URL) async throws -> (ActionResult, DocumentationContext) { // Add the default diagnostic console writer now that we know what log handle it should write to. if !diagnosticEngine.hasConsumer(matching: { $0 is DiagnosticConsoleWriter }) { diagnosticEngine.add( @@ -329,15 +282,10 @@ public struct ConvertAction: AsyncAction { workingDirectory: temporaryFolder, fileManager: fileManager) - // An optional indexer, if indexing while converting is enabled. - var indexer: Indexer? = nil - - let bundleIdentifier = converter.firstAvailableBundle()?.identifier - if let bundleIdentifier { - // Create an index builder and prepare it to receive nodes. - indexer = try Indexer(outputURL: temporaryFolder, bundleIdentifier: bundleIdentifier) - } + let indexer = try Indexer(outputURL: temporaryFolder, bundleIdentifier: bundle.identifier) + let context = try DocumentationContext(bundle: bundle, dataProvider: dataProvider, diagnosticEngine: diagnosticEngine, configuration: configuration) + let outputConsumer = ConvertFileWritingConsumer( targetFolder: temporaryFolder, bundleRootFolder: rootURL, @@ -346,13 +294,31 @@ public struct ConvertAction: AsyncAction { indexer: indexer, enableCustomTemplates: experimentalEnableCustomTemplates, transformForStaticHostingIndexHTML: transformForStaticHosting ? indexHTML : nil, - bundleIdentifier: bundleIdentifier + bundleIdentifier: bundle.identifier ) + if experimentalModifyCatalogWithGeneratedCuration, let catalogURL = rootURL { + let writer = GeneratedCurationWriter(context: context, catalogURL: catalogURL, outputURL: catalogURL) + let curation = try writer.generateDefaultCurationContents() + for (url, updatedContent) in curation { + guard let data = updatedContent.data(using: .utf8) else { continue } + try? FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true, attributes: nil) + try? data.write(to: url, options: .atomic) + } + } + let analysisProblems: [Problem] let conversionProblems: [Problem] do { - (analysisProblems, conversionProblems) = try converter.convert(outputConsumer: outputConsumer) + conversionProblems = try ConvertActionConverter.convert( + bundle: bundle, + context: context, + outputConsumer: outputConsumer, + sourceRepository: sourceRepository, + emitDigest: emitDigest, + documentationCoverageOptions: documentationCoverageOptions + ) + analysisProblems = context.problems } catch { if emitDigest { let problem = Problem(description: (error as? DescribedError)?.errorDescription ?? error.localizedDescription, source: nil) @@ -369,14 +335,11 @@ public struct ConvertAction: AsyncAction { }) // Warn the user if the catalog is a tutorial but does not contains a table of contents // and provide template content to fix this problem. - if ( - context.tutorialTableOfContentsReferences.isEmpty && - hasTutorial - ) { + if context.tutorialTableOfContentsReferences.isEmpty, hasTutorial { let tableOfContentsFilename = CatalogTemplateKind.tutorialTopLevelFilename let source = rootURL?.appendingPathComponent(tableOfContentsFilename) var replacements = [Replacement]() - if let tableOfContentsTemplate = CatalogTemplateKind.tutorialTemplateFiles(converter.firstAvailableBundle()?.displayName ?? "Tutorial Name")[tableOfContentsFilename] { + if let tableOfContentsTemplate = CatalogTemplateKind.tutorialTemplateFiles(bundle.displayName)[tableOfContentsFilename] { replacements.append( Replacement( range: .init(line: 1, column: 1, source: source) ..< .init(line: 1, column: 1, source: source), @@ -404,7 +367,7 @@ public struct ConvertAction: AsyncAction { } // If we're building a navigation index, finalize the process and collect encountered problems. - if let indexer { + do { let finalizeNavigationIndexMetric = benchmark(begin: Benchmark.Duration(id: "finalize-navigation-index")) // Always emit a JSON representation of the index but only emit the LMDB @@ -465,13 +428,13 @@ public struct ConvertAction: AsyncAction { context: context, indexer: nil, transformForStaticHostingIndexHTML: nil, - bundleIdentifier: bundleIdentifier + bundleIdentifier: bundle.identifier ) try outputConsumer.consume(benchmarks: Benchmark.main) } - return ActionResult(didEncounterError: didEncounterError, outputs: [targetDirectory]) + return (ActionResult(didEncounterError: didEncounterError, outputs: [targetDirectory]), context) } func createTempFolder(with templateURL: URL?) throws -> URL { @@ -482,8 +445,3 @@ public struct ConvertAction: AsyncAction { return try Self.moveOutput(from: from, to: to, fileManager: fileManager) } } - -private protocol _DeprecatedConfigurationSetAccess: AnyObject { - var configuration: DocumentationContext.Configuration { get set } -} -extension DocumentationContext: _DeprecatedConfigurationSetAccess {} diff --git a/Sources/SwiftDocCUtilities/Action/Actions/EmitGeneratedCurationAction.swift b/Sources/SwiftDocCUtilities/Action/Actions/EmitGeneratedCurationAction.swift index 12b07721d7..5e58f93740 100644 --- a/Sources/SwiftDocCUtilities/Action/Actions/EmitGeneratedCurationAction.swift +++ b/Sources/SwiftDocCUtilities/Action/Actions/EmitGeneratedCurationAction.swift @@ -42,22 +42,15 @@ struct EmitGeneratedCurationAction: AsyncAction { } mutating func perform(logHandle: inout LogHandle) async throws -> ActionResult { - let workspace = DocumentationWorkspace() - let context = try DocumentationContext(dataProvider: workspace) - - let dataProvider: DocumentationWorkspaceDataProvider - if let catalogURL { - dataProvider = try LocalFileSystemDataProvider(rootURL: catalogURL) - } else { - dataProvider = GeneratedDataProvider(symbolGraphDataLoader: { [fileManager] url in - fileManager.contents(atPath: url.path) - }) - } - let bundleDiscoveryOptions = BundleDiscoveryOptions( - infoPlistFallbacks: [:], - additionalSymbolGraphFiles: symbolGraphFiles(in: additionalSymbolGraphDirectory) + let inputProvider = DocumentationContext.InputsProvider(fileManager: fileManager) + let (bundle, dataProvider) = try inputProvider.inputsAndDataProvider( + startingPoint: catalogURL, + options: BundleDiscoveryOptions( + infoPlistFallbacks: [:], + additionalSymbolGraphFiles: symbolGraphFiles(in: additionalSymbolGraphDirectory) + ) ) - try workspace.registerProvider(dataProvider, options: bundleDiscoveryOptions) + let context = try DocumentationContext(bundle: bundle, dataProvider: dataProvider) let writer = GeneratedCurationWriter(context: context, catalogURL: catalogURL, outputURL: outputURL) let curation = try writer.generateDefaultCurationContents(fromSymbol: startingPointSymbolLink, depthLimit: depthLimit) diff --git a/Sources/SwiftDocCUtilities/Action/Actions/PreviewAction.swift b/Sources/SwiftDocCUtilities/Action/Actions/PreviewAction.swift index d218472464..f943f6f6e5 100644 --- a/Sources/SwiftDocCUtilities/Action/Actions/PreviewAction.swift +++ b/Sources/SwiftDocCUtilities/Action/Actions/PreviewAction.swift @@ -37,8 +37,6 @@ public final class PreviewAction: AsyncAction { /// A test configuration allowing running multiple previews for concurrent testing. static var allowConcurrentPreviews = false - private let context: DocumentationContext - private let workspace: DocumentationWorkspace private let printHTMLTemplatePath: Bool var logHandle = LogHandle.standardOutput @@ -73,8 +71,8 @@ public final class PreviewAction: AsyncAction { public init( port: Int, createConvertAction: @escaping () throws -> ConvertAction, - workspace: DocumentationWorkspace = DocumentationWorkspace(), - context: DocumentationContext? = nil, + workspace _: DocumentationWorkspace = DocumentationWorkspace(), + context _: DocumentationContext? = nil, printTemplatePath: Bool = true) throws { if !Self.allowConcurrentPreviews && !servers.isEmpty { @@ -85,8 +83,6 @@ public final class PreviewAction: AsyncAction { self.port = port self.createConvertAction = createConvertAction self.convertAction = try createConvertAction() - self.workspace = workspace - self.context = try context ?? DocumentationContext(dataProvider: workspace, diagnosticEngine: self.convertAction.diagnosticEngine) self.printHTMLTemplatePath = printTemplatePath } @@ -166,9 +162,9 @@ public final class PreviewAction: AsyncAction { func convert() async throws -> ActionResult { convertAction = try createConvertAction() - let result = try await convertAction.perform(logHandle: &logHandle) + let (result, context) = try await convertAction.perform(logHandle: &logHandle) - previewPaths = try convertAction.context.previewPaths() + previewPaths = try context.previewPaths() return result } @@ -219,10 +215,6 @@ extension PreviewAction { throw ErrorsEncountered() } print("Done.", to: &self.logHandle) - } catch ConvertAction.Error.cancelPending { - // `monitor.restart()` is already queueing a new convert action which will start when the previous one completes. - // We can safely ignore the current action and just log to the console. - print("\nConversion already in progress...", to: &self.logHandle) } catch DocumentationContext.ContextError.registrationDisabled { // The context cancelled loading the bundles and threw to yield execution early. print("\nConversion cancelled...", to: &self.logHandle) diff --git a/Sources/SwiftDocCUtilities/ArgumentParsing/ActionExtensions/ConvertAction+CommandInitialization.swift b/Sources/SwiftDocCUtilities/ArgumentParsing/ActionExtensions/ConvertAction+CommandInitialization.swift index a465255c95..8115032d96 100644 --- a/Sources/SwiftDocCUtilities/ArgumentParsing/ActionExtensions/ConvertAction+CommandInitialization.swift +++ b/Sources/SwiftDocCUtilities/ArgumentParsing/ActionExtensions/ConvertAction+CommandInitialization.swift @@ -44,15 +44,7 @@ extension ConvertAction { // into a dictionary. This will throw with a descriptive error upon failure. let parsedPlatforms = try PlatformArgumentParser.parse(convert.platforms) - let additionalSymbolGraphFiles = symbolGraphFiles(in: convert.additionalSymbolGraphDirectory) - - let bundleDiscoveryOptions = BundleDiscoveryOptions( - fallbackDisplayName: convert.fallbackBundleDisplayName, - fallbackIdentifier: convert.fallbackBundleIdentifier, - fallbackDefaultCodeListingLanguage: convert.defaultCodeListingLanguage, - fallbackDefaultModuleKind: convert.fallbackDefaultModuleKind, - additionalSymbolGraphFiles: additionalSymbolGraphFiles - ) + let bundleDiscoveryOptions = convert.bundleDiscoveryOptions // The `preview` and `convert` action defaulting to the current working directory is only supported // when running `docc preview` and `docc convert` without any of the fallback options. @@ -94,6 +86,20 @@ extension ConvertAction { } } +package extension Docc.Convert { + var bundleDiscoveryOptions: BundleDiscoveryOptions { + let additionalSymbolGraphFiles = symbolGraphFiles(in: additionalSymbolGraphDirectory) + + return BundleDiscoveryOptions( + fallbackDisplayName: fallbackBundleDisplayName, + fallbackIdentifier: fallbackBundleIdentifier, + fallbackDefaultCodeListingLanguage: defaultCodeListingLanguage, + fallbackDefaultModuleKind: fallbackDefaultModuleKind, + additionalSymbolGraphFiles: additionalSymbolGraphFiles + ) + } +} + private func symbolGraphFiles(in directory: URL?) -> [URL] { guard let directory else { return [] } diff --git a/Tests/SwiftDocCTests/Converter/DocumentationContextConverterTests.swift b/Tests/SwiftDocCTests/Converter/DocumentationContextConverterTests.swift index 375d497c29..345e0841ac 100644 --- a/Tests/SwiftDocCTests/Converter/DocumentationContextConverterTests.swift +++ b/Tests/SwiftDocCTests/Converter/DocumentationContextConverterTests.swift @@ -28,7 +28,7 @@ class DocumentationContextConverterTests: XCTestCase { let documentationNode = try XCTUnwrap(try context.entity(with: identifier)) let renderNode1 = try perNodeConverter.convert(documentationNode) - let renderNode2 = try bulkNodeConverter.renderNode(for: documentationNode) + let renderNode2 = bulkNodeConverter.renderNode(for: documentationNode) // Compare the two nodes are identical let data1 = try encoder.encode(renderNode1) diff --git a/Tests/SwiftDocCTests/DocumentationService/ConvertService/ConvertServiceTests.swift b/Tests/SwiftDocCTests/DocumentationService/ConvertService/ConvertServiceTests.swift index 7c3669a437..dbc2e01de0 100644 --- a/Tests/SwiftDocCTests/DocumentationService/ConvertService/ConvertServiceTests.swift +++ b/Tests/SwiftDocCTests/DocumentationService/ConvertService/ConvertServiceTests.swift @@ -2299,15 +2299,12 @@ class ConvertServiceTests: XCTestCase { let request = ConvertRequest( bundleInfo: testBundleInfo, externalIDsToConvert: nil, - symbolGraphs: [], + symbolGraphs: [Data()], markupFiles: [], miscResourceURLs: [] ) - try processAndAssert( - request: request, - converter: TestConverter { throw TestError.testError } - ) { message in + try processAndAssert(request: request) { message in XCTAssertEqual(message.type, "convert-response-error") XCTAssertEqual(message.identifier, "test-identifier-response-error") @@ -2356,42 +2353,8 @@ class ConvertServiceTests: XCTestCase { XCTAssert(linkResolutionRequests.allSatisfy { $0.hasSuffix("/SymbolName") }, "Should have made some link resolution requests to try to match the extension file") } - func testReturnsErrorWhenConversionHasProblems() throws { - let request = ConvertRequest( - bundleInfo: testBundleInfo, - externalIDsToConvert: nil, - symbolGraphs: [], - markupFiles: [], - miscResourceURLs: [] - ) - - let testProblem = Problem( - diagnostic: Diagnostic( - source: nil, - severity: .error, - range: nil, - identifier: "", - summary: "" - ), - possibleSolutions: [] - ) - - try processAndAssert( - request: request, - converter: TestConverter { ([], [testProblem]) } - ) { message in - XCTAssertEqual(message.type, "convert-response-error") - XCTAssertEqual(message.identifier, "test-identifier-response-error") - - let error = try JSONDecoder().decode( - ConvertServiceError.self, from: XCTUnwrap(message.payload)) - XCTAssertEqual(error.identifier, "conversion-error") - } - } - func processAndAssert( request: ConvertRequest, - converter: DocumentationConverterProtocol? = nil, linkResolvingServer: DocumentationServer? = nil, assertion: @escaping (DocumentationServer.Message) throws -> () ) throws { @@ -2399,8 +2362,8 @@ class ConvertServiceTests: XCTestCase { message: DocumentationServer.Message( type: "convert", identifier: "test-identifier", - payload: try JSONEncoder().encode(request)), - converter: converter, + payload: try JSONEncoder().encode(request) + ), linkResolvingServer: linkResolvingServer, assertion: assertion ) @@ -2408,16 +2371,12 @@ class ConvertServiceTests: XCTestCase { func processAndAssert( message: DocumentationServer.Message, - converter: DocumentationConverterProtocol? = nil, linkResolvingServer: DocumentationServer? = nil, assertion: @escaping (DocumentationServer.Message) throws -> () ) throws { let expectation = XCTestExpectation(description: "Sends a response") - ConvertService( - converter: converter, - linkResolvingServer: linkResolvingServer - ).process(message) { message in + ConvertService(linkResolvingServer: linkResolvingServer).process(message) { message in do { try assertion(message) } catch { @@ -2459,21 +2418,6 @@ class ConvertServiceTests: XCTestCase { XCTAssertEqual(topicRenderReference.title, title, file: file, line: line) } - struct TestConverter: DocumentationConverterProtocol { - var convertDelegate: () throws -> ([Problem], [Problem]) - - func convert( - outputConsumer: some ConvertOutputConsumer - ) throws -> (analysisProblems: [Problem], conversionProblems: [Problem]) - { - try convertDelegate() - } - } - - enum TestError: Error { - case testError - } - struct LinkResolvingService: DocumentationService { static var handlingTypes: [DocumentationServer.MessageType] = ["resolve-reference"] diff --git a/Tests/SwiftDocCTests/Infrastructure/DataAssetManagerTests.swift b/Tests/SwiftDocCTests/Infrastructure/DataAssetManagerTests.swift index 6d11a3f465..52b139c722 100644 --- a/Tests/SwiftDocCTests/Infrastructure/DataAssetManagerTests.swift +++ b/Tests/SwiftDocCTests/Infrastructure/DataAssetManagerTests.swift @@ -121,18 +121,15 @@ class DataAssetManagerTests: XCTestCase { // Create the manager let workspace = DocumentationWorkspace() - let bundle = try testBundleFromRootURL(named: "TestBundle") - let bundleURL = Bundle.module.url( - forResource: "TestBundle", withExtension: "docc", subdirectory: "Test Bundles")! - let dataProvider = try LocalFileSystemDataProvider(rootURL: bundleURL) + let catalogURL = try testCatalogURL(named: "TestBundle") + let dataProvider = try LocalFileSystemDataProvider(rootURL: catalogURL) try workspace.registerProvider(dataProvider) var manager = DataAssetManager() // Register an image asset - let imageFileURL = bundleURL.appendingPathComponent("figure1.png") - try manager - .register(data: [imageFileURL], dataProvider: workspace, bundle: bundle) + let imageFileURL = catalogURL.appendingPathComponent("figure1.png") + try manager.register(data: [imageFileURL]) // Check the asset is registered guard !manager.storage.values.isEmpty else { diff --git a/Tests/SwiftDocCTests/Infrastructure/Input Discovery/DocumentationInputsProviderTests.swift b/Tests/SwiftDocCTests/Infrastructure/Input Discovery/DocumentationInputsProviderTests.swift index 7bbaffb5cf..d394ace874 100644 --- a/Tests/SwiftDocCTests/Infrastructure/Input Discovery/DocumentationInputsProviderTests.swift +++ b/Tests/SwiftDocCTests/Infrastructure/Input Discovery/DocumentationInputsProviderTests.swift @@ -76,9 +76,9 @@ class DocumentationInputsProviderTests: XCTestCase { ]) let foundPrevImplBundle = try XCTUnwrap(LocalFileSystemDataProvider(rootURL: tempDirectory.appendingPathComponent("/one/two")).bundles(options: options).first) - let foundRealBundle = try XCTUnwrap(realProvider.inputs(startingPoint: tempDirectory.appendingPathComponent("/one/two"), options: options)) + let (foundRealBundle, _) = try XCTUnwrap(realProvider.inputsAndDataProvider(startingPoint: tempDirectory.appendingPathComponent("/one/two"), options: options)) - let foundTestBundle = try XCTUnwrap(testProvider.inputs(startingPoint: URL(fileURLWithPath: "/one/two"), options: .init( + let (foundTestBundle, _) = try XCTUnwrap(testProvider.inputsAndDataProvider(startingPoint: URL(fileURLWithPath: "/one/two"), options: .init( infoPlistFallbacks: options.infoPlistFallbacks, // The test file system has a default base URL and needs different URLs for the symbol graph files additionalSymbolGraphFiles: [ @@ -141,22 +141,36 @@ class DocumentationInputsProviderTests: XCTestCase { // Allow arbitrary directories as a fallback do { - let foundBundle = try provider.inputs( + let (foundInputs, _) = try provider.inputsAndDataProvider( startingPoint: startingPoint, allowArbitraryCatalogDirectories: true, options: .init() ) - XCTAssertEqual(foundBundle?.displayName, "two") - XCTAssertEqual(foundBundle?.identifier, "two") + XCTAssertEqual(foundInputs.displayName, "two") + XCTAssertEqual(foundInputs.identifier, "two") } // Without arbitrary directories as a fallback do { - XCTAssertNil(try provider.inputs( + XCTAssertThrowsError(try provider.inputsAndDataProvider( startingPoint: startingPoint, allowArbitraryCatalogDirectories: false, options: .init() - )) + )) { error in + XCTAssertEqual(error.localizedDescription, """ + The information provided as command line arguments isn't enough to generate documentation. + + The `` positional argument '/one/two' isn't a documentation catalog (`.docc` directory) \ + and its directory sub-hierarchy doesn't contain a documentation catalog (`.docc` directory). + + To build documentation for the files in '/one/two', either give it a `.docc` file extension to make \ + it a documentation catalog or pass the `--allow-arbitrary-catalog-directories` flag to treat it as \ + a documentation catalog, regardless of file extension. + + To build documentation using only in-source documentation comments, pass a directory of symbol graph \ + files (with a `.symbols.json` file extension) for the `--additional-symbol-graph-dir` argument. + """) + } } } @@ -177,7 +191,7 @@ class DocumentationInputsProviderTests: XCTestCase { let provider = DocumentationContext.InputsProvider(fileManager: fileSystem) XCTAssertThrowsError( - try provider.inputs( + try provider.inputsAndDataProvider( startingPoint: URL(fileURLWithPath: "/one/two"), options: .init() ) @@ -216,14 +230,15 @@ class DocumentationInputsProviderTests: XCTestCase { let provider = DocumentationContext.InputsProvider(fileManager: fileSystem) let startingPoint = URL(fileURLWithPath: "/one/two") - let foundBundle = try provider.inputs( + let (foundInputs, _) = try provider.inputsAndDataProvider( startingPoint: startingPoint, options: .init(additionalSymbolGraphFiles: [ - URL(fileURLWithPath: "/path/to/Something.symbols.json")]) + URL(fileURLWithPath: "/path/to/Something.symbols.json") + ]) ) - XCTAssertEqual(foundBundle?.displayName, "Something") - XCTAssertEqual(foundBundle?.identifier, "Something") - XCTAssertEqual(foundBundle?.symbolGraphURLs.map(\.path), [ + XCTAssertEqual(foundInputs.displayName, "Something") + XCTAssertEqual(foundInputs.identifier, "Something") + XCTAssertEqual(foundInputs.symbolGraphURLs.map(\.path), [ "/path/to/Something.symbols.json", ]) } diff --git a/Tests/SwiftDocCTests/Infrastructure/SymbolDisambiguationTests.swift b/Tests/SwiftDocCTests/Infrastructure/SymbolDisambiguationTests.swift index 3fcc147c99..d73c776ae2 100644 --- a/Tests/SwiftDocCTests/Infrastructure/SymbolDisambiguationTests.swift +++ b/Tests/SwiftDocCTests/Infrastructure/SymbolDisambiguationTests.swift @@ -191,7 +191,7 @@ class SymbolDisambiguationTests: XCTestCase { func testMixedLanguageFramework() throws { let (bundle, context) = try testBundleAndContext(named: "MixedLanguageFramework") - var loader = SymbolGraphLoader(bundle: bundle, dataProvider: context.dataProvider) + var loader = SymbolGraphLoader(bundle: bundle, dataProvider: context._legacyDataProvider!) try loader.loadAll() let references = context.linkResolver.localResolver.referencesForSymbols(in: loader.unifiedGraphs, bundle: bundle, context: context).mapValues(\.path) diff --git a/Tests/SwiftDocCTests/TestRenderNodeOutputConsumer.swift b/Tests/SwiftDocCTests/TestRenderNodeOutputConsumer.swift index c1f65de1eb..22f96fdcb4 100644 --- a/Tests/SwiftDocCTests/TestRenderNodeOutputConsumer.swift +++ b/Tests/SwiftDocCTests/TestRenderNodeOutputConsumer.swift @@ -99,7 +99,7 @@ extension XCTestCase { emitDigest: false, documentationCoverageOptions: .noCoverage, currentPlatforms: nil, - workspace: context.dataProvider as! DocumentationWorkspace, + workspace: context._legacyDataProvider as! DocumentationWorkspace, context: context, dataProvider: try LocalFileSystemDataProvider(rootURL: bundleURL), bundleDiscoveryOptions: BundleDiscoveryOptions() diff --git a/Tests/SwiftDocCTests/XCTestCase+LoadingTestData.swift b/Tests/SwiftDocCTests/XCTestCase+LoadingTestData.swift index 908eb9809e..907b935dbc 100644 --- a/Tests/SwiftDocCTests/XCTestCase+LoadingTestData.swift +++ b/Tests/SwiftDocCTests/XCTestCase+LoadingTestData.swift @@ -18,7 +18,7 @@ extension XCTestCase { /// Loads a documentation bundle from the given source URL and creates a documentation context. func loadBundle( - from bundleURL: URL, + from catalogURL: URL, externalResolvers: [String: ExternalDocumentationSource] = [:], externalSymbolResolver: GlobalExternalSymbolResolver? = nil, fallbackResolver: ConvertServiceFallbackResolver? = nil, @@ -37,12 +37,12 @@ extension XCTestCase { let context = try DocumentationContext(dataProvider: workspace, diagnosticEngine: DiagnosticEngine(filterLevel: diagnosticFilterLevel), configuration: configuration) try configureContext?(context) // Load the bundle using automatic discovery - let automaticDataProvider = try LocalFileSystemDataProvider(rootURL: bundleURL) + let automaticDataProvider = try LocalFileSystemDataProvider(rootURL: catalogURL) // Mutate the bundle to include the code listings, then apply to the workspace using a manual provider. let bundle = try XCTUnwrap(automaticDataProvider.bundles().first) let dataProvider = PrebuiltLocalFileSystemDataProvider(bundles: [bundle]) try workspace.registerProvider(dataProvider) - return (bundleURL, bundle, context) + return (catalogURL, bundle, context) } /// Loads a documentation catalog from an in-memory test file system. @@ -68,6 +68,13 @@ extension XCTestCase { return (bundle, context) } + func testCatalogURL(named name: String, file: StaticString = #file, line: UInt = #line) throws -> URL { + try XCTUnwrap( + Bundle.module.url(forResource: name, withExtension: "docc", subdirectory: "Test Bundles"), + file: file, line: line + ) + } + func testBundleAndContext( copying name: String, excludingPaths excludedPaths: [String] = [], @@ -77,8 +84,7 @@ extension XCTestCase { configuration: DocumentationContext.Configuration = .init(), configureBundle: ((URL) throws -> Void)? = nil ) throws -> (URL, DocumentationBundle, DocumentationContext) { - let sourceURL = try XCTUnwrap(Bundle.module.url( - forResource: name, withExtension: "docc", subdirectory: "Test Bundles")) + let sourceURL = try testCatalogURL(named: name) let sourceExists = FileManager.default.fileExists(atPath: sourceURL.path) let bundleURL = sourceExists @@ -110,9 +116,8 @@ extension XCTestCase { externalResolvers: [String: ExternalDocumentationSource] = [:], fallbackResolver: ConvertServiceFallbackResolver? = nil ) throws -> (URL, DocumentationBundle, DocumentationContext) { - let bundleURL = try XCTUnwrap(Bundle.module.url( - forResource: name, withExtension: "docc", subdirectory: "Test Bundles")) - return try loadBundle(from: bundleURL, externalResolvers: externalResolvers, fallbackResolver: fallbackResolver) + let catalogURL = try testCatalogURL(named: name) + return try loadBundle(from: catalogURL, externalResolvers: externalResolvers, fallbackResolver: fallbackResolver) } func testBundleAndContext(named name: String, externalResolvers: [String: ExternalDocumentationSource] = [:]) throws -> (DocumentationBundle, DocumentationContext) { @@ -133,12 +138,10 @@ extension XCTestCase { } func testBundleFromRootURL(named name: String) throws -> DocumentationBundle { - let bundleURL = try XCTUnwrap(Bundle.module.url( - forResource: name, withExtension: "docc", subdirectory: "Test Bundles")) - let dataProvider = try LocalFileSystemDataProvider(rootURL: bundleURL) - - let bundles = try dataProvider.bundles() - return bundles[0] + let rootURL = try testCatalogURL(named: name) + let inputProvider = DocumentationContext.InputsProvider() + let catalogURL = try XCTUnwrap(inputProvider.findCatalog(startingPoint: rootURL)) + return try inputProvider.makeInputs(contentOf: catalogURL, options: .init()) } func testBundleAndContext() throws -> (bundle: DocumentationBundle, context: DocumentationContext) { @@ -152,11 +155,8 @@ extension XCTestCase { markupURLs: [], miscResourceURLs: [] ) - let provider = PrebuiltLocalFileSystemDataProvider(bundles: [bundle]) - let workspace = DocumentationWorkspace() - try workspace.registerProvider(provider) - let context = try DocumentationContext(dataProvider: workspace) + let context = try DocumentationContext(bundle: bundle, dataProvider: FileManager.default) return (bundle, context) } diff --git a/Tests/SwiftDocCUtilitiesTests/ArgumentParsing/ConvertSubcommandTests.swift b/Tests/SwiftDocCUtilitiesTests/ArgumentParsing/ConvertSubcommandTests.swift index 073a238582..015a27e1b7 100644 --- a/Tests/SwiftDocCUtilitiesTests/ArgumentParsing/ConvertSubcommandTests.swift +++ b/Tests/SwiftDocCUtilitiesTests/ArgumentParsing/ConvertSubcommandTests.swift @@ -269,9 +269,8 @@ class ConvertSubcommandTests: XCTestCase { "--additional-symbol-graph-dir", testBundleURL.path, ]) - - let action = try ConvertAction(fromConvertCommand: convertOptions) - XCTAssertEqual(action.converter.bundleDiscoveryOptions.additionalSymbolGraphFiles.map { $0.lastPathComponent }.sorted(), [ + + XCTAssertEqual(convertOptions.bundleDiscoveryOptions.additionalSymbolGraphFiles.map { $0.lastPathComponent }.sorted(), [ "FillIntroduced.symbols.json", "MyKit@SideKit.symbols.json", "mykit-iOS.symbols.json", @@ -318,9 +317,8 @@ class ConvertSubcommandTests: XCTestCase { let action = try ConvertAction(fromConvertCommand: convertOptions) XCTAssertNil(action.rootURL) - XCTAssertNil(action.converter.rootURL) - XCTAssertEqual(action.converter.bundleDiscoveryOptions.additionalSymbolGraphFiles.map { $0.lastPathComponent }.sorted(), [ + XCTAssertEqual(convertOptions.bundleDiscoveryOptions.additionalSymbolGraphFiles.map { $0.lastPathComponent }.sorted(), [ "FillIntroduced.symbols.json", "MyKit@SideKit.symbols.json", "mykit-iOS.symbols.json", @@ -549,21 +547,24 @@ class ConvertSubcommandTests: XCTestCase { func testTreatWarningAsError() throws { do { // Passing no argument should default to the current working directory. - let convert = try Docc.Convert.parse([]) + let convert = try Docc.Convert.parse([ + testBundleURL.path + ]) let convertAction = try ConvertAction(fromConvertCommand: convert) XCTAssertEqual(convertAction.treatWarningsAsErrors, false) } catch { - XCTFail("Failed to run docc convert without arguments.") + XCTFail("Failed to run docc convert with minimal arguments.") } do { // Passing no argument should default to the current working directory. let convert = try Docc.Convert.parse([ + testBundleURL.path, "--warnings-as-errors" ]) let convertAction = try ConvertAction(fromConvertCommand: convert) XCTAssertEqual(convertAction.treatWarningsAsErrors, true) } catch { - XCTFail("Failed to run docc convert without arguments.") + XCTFail("Failed to run docc convert with minimal arguments.") } } diff --git a/Tests/SwiftDocCUtilitiesTests/ConvertActionTests.swift b/Tests/SwiftDocCUtilitiesTests/ConvertActionTests.swift index cc2f2db692..abac91cfb8 100644 --- a/Tests/SwiftDocCUtilitiesTests/ConvertActionTests.swift +++ b/Tests/SwiftDocCUtilitiesTests/ConvertActionTests.swift @@ -312,9 +312,7 @@ class ConvertActionTests: XCTestCase { func testMoveOutputCreatesTargetFolderParent() throws { // Source folder to test moving - let source = Folder(name: "source", content: [ - TextFile(name: "index.html", utf8Content: ""), - ]) + let source = Folder(name: "source.docc", content: []) // The target location to test moving to let target = Folder(name: "target", content: [ @@ -347,9 +345,7 @@ class ConvertActionTests: XCTestCase { func testMoveOutputDoesNotCreateIntermediateTargetFolderParents() throws { // Source folder to test moving - let source = Folder(name: "source", content: [ - TextFile(name: "index.html", utf8Content: ""), - ]) + let source = Folder(name: "source.docc", content: []) // The target location to test moving to let target = Folder(name: "intermediate", content: [ @@ -915,7 +911,7 @@ class ConvertActionTests: XCTestCase { return try? JSONDecoder().decode(Result.self, from: data) } - var action = try ConvertAction( + let action = try ConvertAction( documentationBundleURL: bundle.absoluteURL, outOfProcessResolver: nil, analyze: false, @@ -926,10 +922,10 @@ class ConvertActionTests: XCTestCase { dataProvider: testDataProvider, fileManager: testDataProvider, temporaryDirectory: testDataProvider.uniqueTemporaryDirectory()) - let result = try await action.perform(logHandle: .none) + let (result, context) = try await action.perform(logHandle: .none) // Because the page order isn't deterministic, we create the indexing records and linkable entities in the same order as the pages. - let indexingRecords: [IndexingRecord] = action.context.knownPages.compactMap { reference in + let indexingRecords: [IndexingRecord] = context.knownPages.compactMap { reference in switch reference.path { case "/documentation/TestBed": return IndexingRecord( @@ -972,7 +968,7 @@ class ConvertActionTests: XCTestCase { return nil } } - let linkableEntities = action.context.knownPages.flatMap { (reference: ResolvedTopicReference) -> [LinkDestinationSummary] in + let linkableEntities = context.knownPages.flatMap { (reference: ResolvedTopicReference) -> [LinkDestinationSummary] in switch reference.path { case "/documentation/TestBed": return [ @@ -1053,7 +1049,7 @@ class ConvertActionTests: XCTestCase { return [] } } - let images: [ImageReference] = action.context.knownPages.flatMap { + let images: [ImageReference] = context.knownPages.flatMap { reference -> [ImageReference] in switch reference.path { case "/documentation/TestBundle/Article": @@ -1335,7 +1331,7 @@ class ConvertActionTests: XCTestCase { return try? JSONDecoder().decode(Result.self, from: data) } - var action = try ConvertAction( + let action = try ConvertAction( documentationBundleURL: bundle.absoluteURL, outOfProcessResolver: nil, analyze: false, @@ -1347,10 +1343,10 @@ class ConvertActionTests: XCTestCase { fileManager: testDataProvider, temporaryDirectory: testDataProvider.uniqueTemporaryDirectory() ) - let result = try await action.perform(logHandle: .none) + let (result, context) = try await action.perform(logHandle: .none) // Because the page order isn't deterministic, we create the indexing records and linkable entities in the same order as the pages. - let indexingRecords: [IndexingRecord] = action.context.knownPages.compactMap { reference in + let indexingRecords: [IndexingRecord] = context.knownPages.compactMap { reference in switch reference.path { case "/tutorials/TestBundle/Article": return IndexingRecord( @@ -1376,7 +1372,7 @@ class ConvertActionTests: XCTestCase { return nil } } - let linkableEntities = action.context.knownPages.flatMap { (reference: ResolvedTopicReference) -> [LinkDestinationSummary] in + let linkableEntities = context.knownPages.flatMap { (reference: ResolvedTopicReference) -> [LinkDestinationSummary] in switch reference.path { case "/tutorials/TestBundle/Article": return [ @@ -1431,7 +1427,7 @@ class ConvertActionTests: XCTestCase { return [] } } - let images: [ImageReference] = action.context.knownPages.flatMap { + let images: [ImageReference] = context.knownPages.flatMap { reference -> [ImageReference] in switch reference.path { case "/tutorials/TestBundle/Article": @@ -1545,50 +1541,24 @@ class ConvertActionTests: XCTestCase { } } - /// An empty implementation of `ConvertOutputConsumer` that purposefully does nothing except - /// to pass the number of documentation coverage info structs received to the given handler - struct TestDocumentationCoverageConsumer: ConvertOutputConsumer { - - let coverageConsumeHandler: (Int) -> Void - - init(coverageConsumeHandler: @escaping (Int) -> Void) { - self.coverageConsumeHandler = coverageConsumeHandler - } - - func consume(renderNode: RenderNode) throws { } - func consume(problems: [Problem]) throws { } - func consume(assetsInBundle bundle: DocumentationBundle) throws {} - func consume(linkableElementSummaries: [LinkDestinationSummary]) throws {} - func consume(indexingRecords: [IndexingRecord]) throws {} - func consume(assets: [RenderReferenceType: [RenderReference]]) throws {} - func consume(benchmarks: Benchmark) throws {} - - // Call the handler with the number of coverage items consumed here - func consume(documentationCoverageInfo: [CoverageDataEntry]) throws { - coverageConsumeHandler(documentationCoverageInfo.count) - } - } - func testMetadataIsOnlyWrittenToOutputFolderWhenDocumentationCoverage() async throws { - - // An empty documentation bundle, except for a single symbol graph file - // containing 8 symbols. + // An empty documentation bundle, except for a single symbol graph file containing 8 symbols. let bundle = Folder(name: "unit-test.docc", content: [ InfoPlist(displayName: "TestBundle", identifier: "com.test.example"), CopyOfFile(original: symbolGraphFile, newName: "MyKit.symbols.json"), ]) - // Count the number of coverage info structs consumed by each test below, - // using TestDocumentationCoverageConsumer and this handler. - var coverageInfoCount = 0 - let coverageInfoHandler = { count in coverageInfoCount += count } - - // Check that they're nothing is written for `.noCoverage` - do { - let testDataProvider = try TestFileSystem(folders: [bundle, Folder.emptyHTMLTemplateDirectory]) - let targetDirectory = URL(fileURLWithPath: testDataProvider.currentDirectoryPath) - .appendingPathComponent("target", isDirectory: true) - + func assertCollectedCoverageCount( + expectedCoverageInfoCount: Int, + expectedCoverageFileExist: Bool, + coverageOptions: DocumentationCoverageOptions, + file: StaticString = #file, + line: UInt = #line + ) async throws { + let fileSystem = try TestFileSystem(folders: [bundle, Folder.emptyHTMLTemplateDirectory]) + let currentDirectory = URL(fileURLWithPath: fileSystem.currentDirectoryPath) + let targetDirectory = currentDirectory.appendingPathComponent("target", isDirectory: true) + var action = try ConvertAction( documentationBundleURL: bundle.absoluteURL, outOfProcessResolver: nil, @@ -1597,74 +1567,30 @@ class ConvertActionTests: XCTestCase { htmlTemplateDirectory: Folder.emptyHTMLTemplateDirectory.absoluteURL, emitDigest: false, currentPlatforms: nil, - dataProvider: testDataProvider, - fileManager: testDataProvider, - temporaryDirectory: testDataProvider.uniqueTemporaryDirectory(), - documentationCoverageOptions: .noCoverage) + dataProvider: fileSystem, + fileManager: fileSystem, + temporaryDirectory: fileSystem.uniqueTemporaryDirectory(), + documentationCoverageOptions: coverageOptions + ) let result = try await action.perform(logHandle: .none) + + let coverageFile = result.outputs[0].appendingPathComponent("documentation-coverage.json") + XCTAssertEqual(expectedCoverageFileExist, fileSystem.fileExists(atPath: coverageFile.path), file: file, line: line) - XCTAssertFalse(testDataProvider.fileExists(atPath: result.outputs[0].appendingPathComponent("documentation-coverage.json").path)) - - // Rerun the convert and test no coverage info structs were consumed - _ = try action.converter.convert(outputConsumer: TestDocumentationCoverageConsumer(coverageConsumeHandler: coverageInfoHandler)) - XCTAssertEqual(coverageInfoCount, 0) + if expectedCoverageFileExist { + let coverageInfo = try JSONDecoder().decode([CoverageDataEntry].self, from: fileSystem.contents(of: coverageFile)) + XCTAssertEqual(coverageInfo.count, expectedCoverageInfoCount, file: file, line: line) + } } + + // Check that they're nothing is written for `.noCoverage` + try await assertCollectedCoverageCount(expectedCoverageInfoCount: 0, expectedCoverageFileExist: false, coverageOptions: .noCoverage) // Check that JSON is written for `.brief` - do { - let testDataProvider = try TestFileSystem(folders: [bundle, Folder.emptyHTMLTemplateDirectory]) - let targetDirectory = URL(fileURLWithPath: testDataProvider.currentDirectoryPath) - .appendingPathComponent("target", isDirectory: true) - - var action = try ConvertAction( - documentationBundleURL: bundle.absoluteURL, - outOfProcessResolver: nil, - analyze: false, - targetDirectory: targetDirectory, - htmlTemplateDirectory: Folder.emptyHTMLTemplateDirectory.absoluteURL, - emitDigest: false, - currentPlatforms: nil, - dataProvider: testDataProvider, - fileManager: testDataProvider, - temporaryDirectory: testDataProvider.uniqueTemporaryDirectory(), - documentationCoverageOptions: DocumentationCoverageOptions(level: .brief)) - let result = try await action.perform(logHandle: .none) - - XCTAssertTrue(testDataProvider.fileExists(atPath: result.outputs[0].appendingPathComponent("documentation-coverage.json").path)) - - // Rerun the convert and test one coverage info structs was consumed for each symbol page (8) - coverageInfoCount = 0 - _ = try action.converter.convert(outputConsumer: TestDocumentationCoverageConsumer(coverageConsumeHandler: coverageInfoHandler)) - XCTAssertEqual(coverageInfoCount, 8) - } - + try await assertCollectedCoverageCount(expectedCoverageInfoCount: 8, expectedCoverageFileExist: true, coverageOptions: .init(level: .brief)) + // Check that JSON is written for `.detailed` - do { - let testDataProvider = try TestFileSystem(folders: [bundle, Folder.emptyHTMLTemplateDirectory]) - let targetDirectory = URL(fileURLWithPath: testDataProvider.currentDirectoryPath) - .appendingPathComponent("target", isDirectory: true) - - var action = try ConvertAction( - documentationBundleURL: bundle.absoluteURL, - outOfProcessResolver: nil, - analyze: false, - targetDirectory: targetDirectory, - htmlTemplateDirectory: Folder.emptyHTMLTemplateDirectory.absoluteURL, - emitDigest: false, - currentPlatforms: nil, - dataProvider: testDataProvider, - fileManager: testDataProvider, - temporaryDirectory: testDataProvider.uniqueTemporaryDirectory(), - documentationCoverageOptions: DocumentationCoverageOptions(level: .detailed)) - let result = try await action.perform(logHandle: .none) - - XCTAssertTrue(testDataProvider.fileExists(atPath: result.outputs[0].appendingPathComponent("documentation-coverage.json").path)) - - // Rerun the convert and test one coverage info structs was consumed for each symbol page (8) - coverageInfoCount = 0 - _ = try action.converter.convert(outputConsumer: TestDocumentationCoverageConsumer(coverageConsumeHandler: coverageInfoHandler)) - XCTAssertEqual(coverageInfoCount, 8) - } + try await assertCollectedCoverageCount(expectedCoverageInfoCount: 8, expectedCoverageFileExist: true, coverageOptions: .init(level: .detailed)) } /// Test context gets the current platforms provided by command line. @@ -1698,16 +1624,26 @@ class ConvertActionTests: XCTestCase { temporaryDirectory: testDataProvider.uniqueTemporaryDirectory() ) - XCTAssertEqual(action.context.configuration.externalMetadata.currentPlatforms, [ + XCTAssertEqual(action.configuration.externalMetadata.currentPlatforms, [ "platform1" : PlatformVersion(.init(10, 11, 12), beta: false), "platform2" : PlatformVersion(.init(11, 12, 13), beta: false), ]) } func testBetaInAvailabilityFallbackPlatforms() throws { - - func generateConvertAction(currentPlatforms: [String : PlatformVersion]) throws -> ConvertAction { - try ConvertAction( + func makeConvertAction(currentPlatforms: [String : PlatformVersion]) throws -> ConvertAction { + let bundle = Folder(name: "nested", content: [ + Folder(name: "folders", content: [ + Folder(name: "unit-test.docc", content: [ + InfoPlist(displayName: "TestBundle", identifier: "com.test.example"), + ]), + ]) + ]) + let testDataProvider = try TestFileSystem(folders: [bundle, Folder.emptyHTMLTemplateDirectory]) + let targetDirectory = URL(fileURLWithPath: testDataProvider.currentDirectoryPath) + .appendingPathComponent("target", isDirectory: true) + + return try ConvertAction( documentationBundleURL: bundle.absoluteURL, outOfProcessResolver: nil, analyze: false, @@ -1720,51 +1656,41 @@ class ConvertActionTests: XCTestCase { temporaryDirectory: testDataProvider.uniqueTemporaryDirectory() ) } - let bundle = Folder(name: "nested", content: [ - Folder(name: "folders", content: [ - Folder(name: "unit-test.docc", content: [ - InfoPlist(displayName: "TestBundle", identifier: "com.test.example"), - ]), - ]) - ]) - let testDataProvider = try TestFileSystem(folders: [bundle, Folder.emptyHTMLTemplateDirectory]) - let targetDirectory = URL(fileURLWithPath: testDataProvider.currentDirectoryPath) - .appendingPathComponent("target", isDirectory: true) // Test whether the missing platforms copy the availability information from the fallback platform. - var action = try generateConvertAction(currentPlatforms: ["iOS": PlatformVersion(.init(10, 0, 0), beta: true)]) - XCTAssertEqual(action.context.configuration.externalMetadata.currentPlatforms, [ + var action = try makeConvertAction(currentPlatforms: ["iOS": PlatformVersion(.init(10, 0, 0), beta: true)]) + XCTAssertEqual(action.configuration.externalMetadata.currentPlatforms, [ "iOS" : PlatformVersion(.init(10, 0, 0), beta: true), "Mac Catalyst" : PlatformVersion(.init(10, 0, 0), beta: true), "iPadOS" : PlatformVersion(.init(10, 0, 0), beta: true), ]) // Test whether the non-missing platforms don't copy the availability information from the fallback platform. - action = try generateConvertAction(currentPlatforms: [ + action = try makeConvertAction(currentPlatforms: [ "iOS": PlatformVersion(.init(10, 0, 0), beta: true), "Mac Catalyst": PlatformVersion(.init(11, 0, 0), beta: false) ]) - XCTAssertEqual(action.context.configuration.externalMetadata.currentPlatforms, [ + XCTAssertEqual(action.configuration.externalMetadata.currentPlatforms, [ "iOS" : PlatformVersion(.init(10, 0, 0), beta: true), "Mac Catalyst" : PlatformVersion(.init(11, 0, 0), beta: false), "iPadOS" : PlatformVersion(.init(10, 0, 0), beta: true) ]) - action = try generateConvertAction(currentPlatforms: [ + action = try makeConvertAction(currentPlatforms: [ "iOS": PlatformVersion(.init(10, 0, 0), beta: true), "Mac Catalyst" : PlatformVersion(.init(11, 0, 0), beta: true), "iPadOS": PlatformVersion(.init(12, 0, 0), beta: false), ]) - XCTAssertEqual(action.context.configuration.externalMetadata.currentPlatforms, [ + XCTAssertEqual(action.configuration.externalMetadata.currentPlatforms, [ "iOS" : PlatformVersion(.init(10, 0, 0), beta: true), "Mac Catalyst" : PlatformVersion(.init(11, 0, 0), beta: true), "iPadOS" : PlatformVersion(.init(12, 0, 0), beta: false), ]) // Test whether the non-missing platforms don't copy the availability information from the non-fallback platform. - action = try generateConvertAction(currentPlatforms: [ + action = try makeConvertAction(currentPlatforms: [ "tvOS": PlatformVersion(.init(13, 0, 0), beta: true) ]) - XCTAssertEqual(action.context.configuration.externalMetadata.currentPlatforms, [ + XCTAssertEqual(action.configuration.externalMetadata.currentPlatforms, [ "tvOS": PlatformVersion(.init(13, 0, 0), beta: true) ]) } @@ -1797,7 +1723,7 @@ class ConvertActionTests: XCTestCase { _ = try await action.perform(logHandle: .none) - XCTAssertEqual(ResolvedTopicReference._numberOfCachedReferences(bundleID: #function), 13) + XCTAssertEqual(ResolvedTopicReference._numberOfCachedReferences(bundleID: #function), 8) } func testIgnoresAnalyzerHintsByDefault() async throws { @@ -1885,7 +1811,9 @@ class ConvertActionTests: XCTestCase { temporaryDirectory: testDataProvider.uniqueTemporaryDirectory() ) - action.converter.batchNodeCount = batchSize + // FIXME: This test has never used different batch sizes. (rdar://137885335) + // All the way since the initial commit, `DocumentationConverter.convert(outputConsumer:)` has called + // `Collection.concurrentPerform(batches:block:)` without passing a custom number of `batches`. return try await action.perform(logHandle: .none) } @@ -2418,12 +2346,8 @@ class ConvertActionTests: XCTestCase { XCTAssertTrue(FileManager.default.fileExists(atPath: diagnosticFile.path), "Diagnostic file exist after") } - // Verifies setting convert inherit docs flag func testConvertInheritDocsOption() throws { - // Empty documentation bundle - let bundle = Folder(name: "unit-test.documentation", content: [ - InfoPlist(displayName: "TestBundle", identifier: "com.test.example"), - ]) + let bundle = Folder(name: "unit-test.docc", content: []) let testDataProvider = try TestFileSystem(folders: [bundle, Folder.emptyHTMLTemplateDirectory]) let targetDirectory = URL(fileURLWithPath: testDataProvider.currentDirectoryPath) @@ -2442,8 +2366,10 @@ class ConvertActionTests: XCTestCase { dataProvider: testDataProvider, fileManager: testDataProvider, temporaryDirectory: testDataProvider.uniqueTemporaryDirectory(), - inheritDocs: flag) - XCTAssertEqual(action.context.configuration.externalMetadata.inheritDocs, flag) + inheritDocs: flag + ) + + XCTAssertEqual(action.configuration.externalMetadata.inheritDocs, flag) } // Verify implicit value @@ -2457,8 +2383,9 @@ class ConvertActionTests: XCTestCase { currentPlatforms: nil, dataProvider: testDataProvider, fileManager: testDataProvider, - temporaryDirectory: testDataProvider.uniqueTemporaryDirectory()) - XCTAssertEqual(action.context.configuration.externalMetadata.inheritDocs, false) + temporaryDirectory: testDataProvider.uniqueTemporaryDirectory() + ) + XCTAssertEqual(action.configuration.externalMetadata.inheritDocs, false) } func testEmitsDigest() async throws { @@ -2507,7 +2434,7 @@ class ConvertActionTests: XCTestCase { let dataProvider = try LocalFileSystemDataProvider(rootURL: catalogURL) var action = try ConvertAction( - documentationBundleURL: catalog.absoluteURL, + documentationBundleURL: catalogURL, outOfProcessResolver: nil, analyze: false, targetDirectory: targetDirectory, @@ -2519,7 +2446,6 @@ class ConvertActionTests: XCTestCase { temporaryDirectory: createTemporaryDirectory() ) - try await action.performAndHandleResult(logHandle: .none) let indexDirectory = targetDirectory.appendingPathComponent("index", isDirectory: true) let renderIndexJSON = indexDirectory.appendingPathComponent("index.json", isDirectory: false) @@ -2966,7 +2892,7 @@ class ConvertActionTests: XCTestCase { let testDataProvider = try TestFileSystem(folders: [Folder.emptyHTMLTemplateDirectory, symbolGraphFiles, outputLocation]) - var action = try ConvertAction( + let action = try ConvertAction( documentationBundleURL: nil, outOfProcessResolver: nil, analyze: false, @@ -2980,12 +2906,10 @@ class ConvertActionTests: XCTestCase { additionalSymbolGraphFiles: [URL(fileURLWithPath: "/Not-a-doc-bundle/MyKit.symbols.json")] ) ) - - XCTAssert(action.context.registeredBundles.isEmpty) - _ = try await action.perform(logHandle: .none) + let (_, context) = try await action.perform(logHandle: .none) - XCTAssertEqual(action.context.registeredBundles.count, 1) - let bundle = try XCTUnwrap(action.context.registeredBundles.first, "Should have registered the generated test bundle.") + XCTAssertEqual(context.registeredBundles.count, 1) + let bundle = try XCTUnwrap(context.registeredBundles.first, "Should have registered the generated test bundle.") XCTAssertEqual(bundle.displayName, "MyKit") XCTAssertEqual(bundle.identifier, "MyKit") } @@ -3006,55 +2930,49 @@ class ConvertActionTests: XCTestCase { let outputLocation = Folder(name: "output", content: []) - let testDataProvider = try TestFileSystem( + let fileSystem = try TestFileSystem( folders: [Folder.emptyHTMLTemplateDirectory, symbolGraphFiles, outputLocation] ) - - var infoPlistFallbacks = [String: Any]() - infoPlistFallbacks["CFBundleIdentifier"] = "com.example.test" - - var action = try ConvertAction( - documentationBundleURL: nil, - outOfProcessResolver: nil, - analyze: false, - targetDirectory: outputLocation.absoluteURL, - htmlTemplateDirectory: Folder.emptyHTMLTemplateDirectory.absoluteURL, - emitDigest: false, - currentPlatforms: nil, - fileManager: testDataProvider, - temporaryDirectory: testDataProvider.uniqueTemporaryDirectory(), - bundleDiscoveryOptions: BundleDiscoveryOptions( - infoPlistFallbacks: infoPlistFallbacks, - additionalSymbolGraphFiles: [ - URL(fileURLWithPath: "/Not-a-doc-bundle/MyKit.symbols.json"), - URL(fileURLWithPath: "/Not-a-doc-bundle/SideKit.symbols.json") - ] - ) - ) - do { + var action = try ConvertAction( + documentationBundleURL: nil, + outOfProcessResolver: nil, + analyze: false, + targetDirectory: outputLocation.absoluteURL, + htmlTemplateDirectory: Folder.emptyHTMLTemplateDirectory.absoluteURL, + emitDigest: false, + currentPlatforms: nil, + fileManager: fileSystem, + temporaryDirectory: fileSystem.uniqueTemporaryDirectory(), + bundleDiscoveryOptions: BundleDiscoveryOptions( + infoPlistFallbacks: ["CFBundleIdentifier": "com.example.test"], + additionalSymbolGraphFiles: [ + URL(fileURLWithPath: "/Not-a-doc-bundle/MyKit.symbols.json"), + URL(fileURLWithPath: "/Not-a-doc-bundle/SideKit.symbols.json") + ] + ) + ) _ = try await action.perform(logHandle: .none) XCTFail("The action didn't raise an error") } catch { XCTAssertEqual(error.localizedDescription, """ - The information provided as command line arguments is not enough to generate a documentation bundle: - + The information provided as command line arguments isn't enough to generate documentation. + Missing value for 'CFBundleDisplayName'. - Use the '--fallback-display-name' argument or add 'CFBundleDisplayName' to the bundle Info.plist. - + Use the '--fallback-display-name' argument or add 'CFBundleDisplayName' to the catalog's Info.plist. """) } } func testConvertWithBundleDerivesDisplayNameFromBundle() async throws { - let emptyDoccCatalog = try createTemporaryDirectory(named: "Something.docc") + let emptyCatalog = try createTemporaryDirectory(named: "Something.docc") let outputLocation = try createTemporaryDirectory(named: "output") var infoPlistFallbacks = [String: Any]() infoPlistFallbacks["CFBundleIdentifier"] = "com.example.test" - var action = try ConvertAction( - documentationBundleURL: emptyDoccCatalog, + let action = try ConvertAction( + documentationBundleURL: emptyCatalog, outOfProcessResolver: nil, analyze: false, targetDirectory: outputLocation.absoluteURL, @@ -3067,11 +2985,10 @@ class ConvertActionTests: XCTestCase { additionalSymbolGraphFiles: [] ) ) - XCTAssert(action.context.registeredBundles.isEmpty) - _ = try await action.perform(logHandle: .none) + let (_, context) = try await action.perform(logHandle: .none) - XCTAssertEqual(action.context.registeredBundles.count, 1) - let bundle = try XCTUnwrap(action.context.registeredBundles.first, "Should have registered the generated test bundle.") + XCTAssertEqual(context.registeredBundles.count, 1) + let bundle = try XCTUnwrap(context.registeredBundles.first, "Should have registered the generated test bundle.") XCTAssertEqual(bundle.displayName, "Something") XCTAssertEqual(bundle.identifier, "com.example.test") } @@ -3370,3 +3287,11 @@ extension Folder { return Folder(name: url.lastPathComponent, content: content) } } + +private extension ConvertAction { + @_disfavoredOverload + func perform(logHandle: LogHandle) async throws -> (ActionResult, DocumentationContext) { + var logHandle = logHandle + return try await perform(logHandle: &logHandle) + } +} diff --git a/Tests/SwiftDocCUtilitiesTests/EmitGeneratedCurationsActionTests.swift b/Tests/SwiftDocCUtilitiesTests/EmitGeneratedCurationsActionTests.swift index 850891df32..29b07688b5 100644 --- a/Tests/SwiftDocCUtilitiesTests/EmitGeneratedCurationsActionTests.swift +++ b/Tests/SwiftDocCUtilitiesTests/EmitGeneratedCurationsActionTests.swift @@ -17,7 +17,7 @@ class EmitGeneratedCurationsActionTests: XCTestCase { func testWritesDocumentationExtensionFilesToOutputDir() async throws { // This can't be in the test file system because `LocalFileSystemDataProvider` doesn't support `FileManagerProtocol`. - let bundleURL = try XCTUnwrap(Bundle.module.url(forResource: "MixedLanguageFramework", withExtension: "docc", subdirectory: "Test Bundles")) + let realCatalogURL = try XCTUnwrap(Bundle.module.url(forResource: "MixedLanguageFramework", withExtension: "docc", subdirectory: "Test Bundles")) func assertOutput( initialContent: [File], @@ -28,12 +28,17 @@ class EmitGeneratedCurationsActionTests: XCTestCase { line: UInt = #line ) async throws { let fs = try TestFileSystem(folders: [ - Folder(name: "output", content: initialContent) + Folder(name: "input", content: [ + CopyOfFolder(original: realCatalogURL) + ]), + Folder(name: "output", content: initialContent), ]) - let outputDir = URL(fileURLWithPath: "/output/Output.doccarchive") + let catalogURL = URL(fileURLWithPath: "/input/MixedLanguageFramework.docc") + let outputDir = URL(fileURLWithPath: "/output/Output.doccarchive") + var action = try EmitGeneratedCurationAction( - documentationCatalog: bundleURL, + documentationCatalog: catalogURL, additionalSymbolGraphDirectory: nil, outputURL: outputDir, depthLimit: depthLimit,