diff --git a/scarf/Packages/ScarfCore/Sources/ScarfCore/Services/ModelCatalogService.swift b/scarf/Packages/ScarfCore/Sources/ScarfCore/Services/ModelCatalogService.swift index 1785e11..6cf3834 100644 --- a/scarf/Packages/ScarfCore/Sources/ScarfCore/Services/ModelCatalogService.swift +++ b/scarf/Packages/ScarfCore/Sources/ScarfCore/Services/ModelCatalogService.swift @@ -285,6 +285,75 @@ public struct ModelCatalogService: Sendable { ) } + /// Result of validating a user-entered model ID against the + /// selected provider. See `validateModel(_:for:)`. + public enum ModelValidation: Equatable, Sendable { + /// Accept the save — the model is in the provider's catalog + /// (or the provider is overlay-only, where a free-form model + /// name is the normal path). + case valid + /// Accept with a warning — we don't have a catalog entry for + /// the provider at all, so can't check. Usually means the + /// user is offline or the local cache is missing. Save but + /// surface an advisory. + case unknownProvider(providerID: String) + /// Block the save — the provider exists but doesn't serve + /// that model. Includes a handful of close-by suggestions + /// for the UI to render as "did you mean…". + case invalid(providerName: String, suggestions: [String]) + } + + /// Validate `modelID` against `providerID` before persisting it as + /// `model.default` in `config.yaml`. Centralises the logic so both + /// Mac's ModelPickerSheet and ScarfGo's scoped settings editor + /// (Phase 4.3) use the same check. Pass-1 found that you could + /// save `claude-haiku-4-5-20251001` under provider `nous` — + /// Nous's catalog has no such model and Hermes later failed with + /// HTTP 404 at runtime. Catch that at save time, not 6 hours later. + public func validateModel(_ modelID: String, for providerID: String) -> ModelValidation { + let trimmed = modelID.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { + return .invalid(providerName: providerID, suggestions: []) + } + + // Overlay-only providers (Nous Portal, OpenAI Codex, Qwen + // OAuth, …) serve their own catalogs that aren't mirrored to + // models.dev, so we don't have a reliable way to check model + // IDs locally. Treat any non-empty value as provisionally + // valid — the worst case is the runtime 404 we hit in pass-1, + // but the UI has the error banner now (M7 #2) to surface that + // cleanly. + // + // Exception: if an overlay-only provider DOES appear in the + // models.dev cache (unlikely but possible as catalogs evolve), + // we fall through to the real check below. + let models = loadModels(for: providerID) + if models.isEmpty { + if Self.overlayOnlyProviders[providerID] != nil { + return .valid + } + return .unknownProvider(providerID: providerID) + } + + if models.contains(where: { $0.modelID == trimmed }) { + return .valid + } + + // No exact match — offer the closest names (by prefix) as + // suggestions. Up to 5, ordered by release date (newest + // first — already the sort order of loadModels). + let lowerTrimmed = trimmed.lowercased() + let byPrefix = models + .filter { $0.modelID.lowercased().hasPrefix(String(lowerTrimmed.prefix(3))) } + .prefix(5) + .map(\.modelID) + let suggestions = byPrefix.isEmpty + ? Array(models.prefix(5).map(\.modelID)) + : Array(byPrefix) + let providerName = providerByID(providerID)?.providerName ?? providerID + return .invalid(providerName: providerName, suggestions: suggestions) + } + // MARK: - Decoding private func loadCatalog() -> [String: ProviderEntry]? { diff --git a/scarf/Packages/ScarfCore/Tests/ScarfCoreTests/M0cServicesTests.swift b/scarf/Packages/ScarfCore/Tests/ScarfCoreTests/M0cServicesTests.swift index 2657c07..a4ebff6 100644 --- a/scarf/Packages/ScarfCore/Tests/ScarfCoreTests/M0cServicesTests.swift +++ b/scarf/Packages/ScarfCore/Tests/ScarfCoreTests/M0cServicesTests.swift @@ -207,6 +207,94 @@ import Foundation #expect(bad.loadProviders().isEmpty) } + @Test func validateModelAcceptsCatalogHit() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent("scarf-catalog-\(UUID().uuidString).json") + defer { try? FileManager.default.removeItem(at: tmp) } + let json = #""" + { + "deepseek": { + "id": "deepseek", + "name": "DeepSeek", + "models": { + "deepseek-v4-flash": {"name": "DeepSeek v4 Flash"}, + "deepseek-v4-pro": {"name": "DeepSeek v4 Pro"} + } + } + } + """# + try json.write(to: tmp, atomically: true, encoding: .utf8) + let svc = ModelCatalogService(path: tmp.path) + #expect(svc.validateModel("deepseek-v4-flash", for: "deepseek") == .valid) + } + + @Test func validateModelRejectsMissingModelWithSuggestions() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent("scarf-catalog-\(UUID().uuidString).json") + defer { try? FileManager.default.removeItem(at: tmp) } + let json = #""" + { + "deepseek": { + "id": "deepseek", + "name": "DeepSeek", + "models": { + "deepseek-v4-flash": {"name": "DeepSeek v4 Flash"}, + "deepseek-v4-pro": {"name": "DeepSeek v4 Pro"} + } + } + } + """# + try json.write(to: tmp, atomically: true, encoding: .utf8) + let svc = ModelCatalogService(path: tmp.path) + // The exact bug from pass-1: Anthropic-style model ID under + // deepseek provider. + let result = svc.validateModel("claude-haiku-4-5-20251001", for: "deepseek") + guard case let .invalid(providerName, suggestions) = result else { + Issue.record("Expected .invalid, got \(result)") + return + } + #expect(providerName == "DeepSeek") + // No prefix match on "cla" so we fall through to the first-5 + // suggestion fallback; both entries are present. + #expect(suggestions.count == 2) + } + + @Test func validateModelReportsUnknownProviderWhenNoCatalogEntry() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent("scarf-catalog-\(UUID().uuidString).json") + defer { try? FileManager.default.removeItem(at: tmp) } + // Empty catalog — provider "openai" won't be found + isn't + // overlay-only, so should return .unknownProvider. + try "{}".write(to: tmp, atomically: true, encoding: .utf8) + let svc = ModelCatalogService(path: tmp.path) + let result = svc.validateModel("gpt-5", for: "openai") + if case .unknownProvider(let pid) = result { + #expect(pid == "openai") + } else { + Issue.record("Expected .unknownProvider, got \(result)") + } + } + + @Test func validateModelAcceptsOverlayProvidersWithoutCatalog() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent("scarf-catalog-\(UUID().uuidString).json") + defer { try? FileManager.default.removeItem(at: tmp) } + try "{}".write(to: tmp, atomically: true, encoding: .utf8) + let svc = ModelCatalogService(path: tmp.path) + // Nous is an overlay-only provider — any non-empty string is + // accepted because the overlay has no local catalog mirror. + #expect(svc.validateModel("deepseek/deepseek-v4-flash", for: "nous") == .valid) + } + + @Test func validateModelRejectsEmptyInput() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent("scarf-catalog-\(UUID().uuidString).json") + defer { try? FileManager.default.removeItem(at: tmp) } + try "{}".write(to: tmp, atomically: true, encoding: .utf8) + let svc = ModelCatalogService(path: tmp.path) + let result = svc.validateModel("", for: "nous") + if case .invalid = result { + // expected + } else { + Issue.record("Expected .invalid for empty input, got \(result)") + } + } + // MARK: - ProjectDashboardService @Test func projectDashboardServiceRegistryRoundTrip() throws { diff --git a/scarf/scarf/Features/Settings/Views/Components/ModelPickerSheet.swift b/scarf/scarf/Features/Settings/Views/Components/ModelPickerSheet.swift index bf4b28d..f66b447 100644 --- a/scarf/scarf/Features/Settings/Views/Components/ModelPickerSheet.swift +++ b/scarf/scarf/Features/Settings/Views/Components/ModelPickerSheet.swift @@ -40,6 +40,12 @@ struct ModelPickerSheet: View { /// "Sign in to Nous Portal" button in the subscription summary. @State private var showNousSignIn: Bool = false + /// Validation failure surfaced on Select when the typed / selected + /// model isn't in the chosen provider's catalog. Pass-1 M7 #5 + /// cross-platform fix — previously Scarf let you save any string + /// and the failure only appeared hours later at runtime. + @State private var validationIssue: ModelValidationIssue? + @Environment(\.serverContext) private var serverContext private var catalog: ModelCatalogService { ModelCatalogService(context: serverContext) } private var subscriptionService: NousSubscriptionService { NousSubscriptionService(context: serverContext) } @@ -76,6 +82,27 @@ struct ModelPickerSheet: View { subscription = subscriptionService.loadState() } } + .alert(item: $validationIssue) { issue in + Alert( + title: Text("Model not available"), + message: Text(validationMessage(for: issue)), + primaryButton: .default(Text("Pick from catalog")) { + validationIssue = nil + customMode = false + }, + secondaryButton: .cancel(Text("Edit")) + ) + } + } + + private func validationMessage(for issue: ModelValidationIssue) -> String { + var msg = "\(issue.modelID) isn't in \(issue.providerName)'s catalog." + if !issue.suggestions.isEmpty { + msg += " Did you mean one of:\n• " + issue.suggestions.joined(separator: "\n• ") + } else { + msg += " Pick one from the catalog or double-check the spelling." + } + return msg } private var header: some View { @@ -423,15 +450,30 @@ struct ModelPickerSheet: View { } private func submitSelection() { + let (model, provider): (String, String) if customMode { - let model = customModelID.trimmingCharacters(in: .whitespaces) - let provider = resolvedCustomProvider() - onSelect(model, provider) + model = customModelID.trimmingCharacters(in: .whitespaces) + provider = resolvedCustomProvider() } else if isSelectedProviderOverlay { - let model = overlayModelID.trimmingCharacters(in: .whitespaces) - onSelect(model, selectedProviderID) + model = overlayModelID.trimmingCharacters(in: .whitespaces) + provider = selectedProviderID } else { - onSelect(selectedModelID, selectedProviderID) + model = selectedModelID + provider = selectedProviderID + } + + // Block unknown models before they land in config.yaml. + // Overlay-only providers short-circuit to .valid inside the + // validator because their catalogs aren't in models.dev. + switch catalog.validateModel(model, for: provider) { + case .valid, .unknownProvider: + onSelect(model, provider) + case .invalid(let providerName, let suggestions): + validationIssue = ModelValidationIssue( + modelID: model, + providerName: providerName, + suggestions: suggestions + ) } } @@ -445,3 +487,12 @@ struct ModelPickerSheet: View { .clipShape(Capsule()) } } + +/// Carrier for the catalog-validation alert. Identifiable so SwiftUI's +/// `.alert(item:)` can key off each unique issue. +private struct ModelValidationIssue: Identifiable { + let id = UUID() + let modelID: String + let providerName: String + let suggestions: [String] +} diff --git a/scarf/scarf/Localizable.xcstrings b/scarf/scarf/Localizable.xcstrings index cc3836d..da98134 100644 --- a/scarf/scarf/Localizable.xcstrings +++ b/scarf/scarf/Localizable.xcstrings @@ -906,6 +906,7 @@ "isCommentAutoGenerated" : true }, "7 Days" : { + "extractionState" : "stale", "localizations" : { "de" : { "stringUnit" : { @@ -949,6 +950,7 @@ }, "30 Days" : { + "extractionState" : "stale", "localizations" : { "de" : { "stringUnit" : { @@ -989,6 +991,7 @@ } }, "90 Days" : { + "extractionState" : "stale", "localizations" : { "de" : { "stringUnit" : { @@ -2043,10 +2046,15 @@ } } }, + "agent key · %@" : { + "comment" : "A label that shows the age of the agent key.", + "isCommentAutoGenerated" : true + }, "Agent-specific instructions (optional)" : { }, "All" : { + "extractionState" : "stale", "localizations" : { "de" : { "stringUnit" : { @@ -2207,6 +2215,7 @@ } }, "All Time" : { + "extractionState" : "stale", "localizations" : { "de" : { "stringUnit" : { @@ -8181,6 +8190,7 @@ } }, "Errors" : { + "extractionState" : "stale", "localizations" : { "de" : { "stringUnit" : { @@ -8301,6 +8311,7 @@ } }, "Execute" : { + "extractionState" : "stale", "localizations" : { "de" : { "stringUnit" : { @@ -8420,6 +8431,14 @@ } } }, + "expired" : { + "comment" : "A label for a credential that has expired.", + "isCommentAutoGenerated" : true + }, + "expires in %lldd" : { + "comment" : "A badge that shows the number of days remaining until a credential expires.", + "isCommentAutoGenerated" : true + }, "Export \"%@\" as Template" : { "comment" : "A title for the export form, showing the name of the project being exported.", "isCommentAutoGenerated" : true @@ -8724,6 +8743,7 @@ }, "Fetch" : { + "extractionState" : "stale", "localizations" : { "de" : { "stringUnit" : { @@ -10934,6 +10954,10 @@ } } }, + "Launch Dashboard" : { + "comment" : "A button that launches a web dashboard.", + "isCommentAutoGenerated" : true + }, "Layout" : { "localizations" : { "de" : { @@ -13837,6 +13861,10 @@ } } }, + "not running" : { + "comment" : "A label displayed when the web dashboard is not running.", + "isCommentAutoGenerated" : true + }, "Not signed in yet." : { "comment" : "A description of a model picker sheet's subscription", "isCommentAutoGenerated" : true @@ -14551,6 +14579,7 @@ } }, "Other" : { + "extractionState" : "stale", "localizations" : { "de" : { "stringUnit" : { @@ -16187,6 +16216,7 @@ } }, "Read" : { + "extractionState" : "stale", "localizations" : { "de" : { "stringUnit" : { @@ -23736,6 +23766,14 @@ "comment" : "A description of the process of adding a credential via the Nous Portal.", "isCommentAutoGenerated" : true }, + "Web Dashboard" : { + "comment" : "A label for the web dashboard.", + "isCommentAutoGenerated" : true + }, + "Web Dashboard on :%lld" : { + "comment" : "A label displaying the port number of the web dashboard.", + "isCommentAutoGenerated" : true + }, "Web Extract" : { "localizations" : { "de" : {