add iOS

iOS-SwiftUI-Code/GRDB.swift-6.29.3/GRDB/FTS/FTS3TokenizerDescriptor.swift

@@ -0,0 +1,120 @@
+/// The descriptor for an ``FTS3`` tokenizer.
+///
+/// `FTS3TokenizerDescriptor` can be used in both ``FTS3`` and ``FTS4`` tables.
+///
+/// For example:
+///
+/// ```swift
+/// try db.create(virtualTable: "book", using: FTS4()) { t in
+///     t.tokenizer = .simple // FTS3TokenizerDescriptor
+/// }
+/// ```
+///
+/// Related SQLite documentation: <https://www.sqlite.org/fts3.html#tokenizer>
+///
+/// ## Topics
+///
+/// ### Creating Tokenizer Descriptors
+///
+/// - ``porter``
+/// - ``simple``
+/// - ``unicode61(diacritics:separators:tokenCharacters:)``
+/// - ``FTS3/Diacritics``
+public struct FTS3TokenizerDescriptor: Sendable {
+    let name: String
+    let arguments: [String]
+    
+    init(_ name: String, arguments: [String] = []) {
+        self.name = name
+        self.arguments = arguments
+    }
+    
+    /// The simple tokenizer.
+    ///
+    /// For example:
+    ///
+    /// ```swift
+    /// try db.create(virtualTable: "book", using: FTS4()) { t in
+    ///     t.tokenizer = .simple
+    /// }
+    /// ```
+    ///
+    /// Related SQLite documentation: <https://www.sqlite.org/fts3.html#tokenizer>
+    public static let simple = FTS3TokenizerDescriptor("simple")
+    
+    /// The porter tokenizer.
+    ///
+    /// For example:
+    ///
+    /// ```swift
+    /// try db.create(virtualTable: "book", using: FTS4()) { t in
+    ///     t.tokenizer = .porter
+    /// }
+    /// ```
+    ///
+    /// Related SQLite documentation: <https://www.sqlite.org/fts3.html#tokenizer>
+    public static let porter = FTS3TokenizerDescriptor("porter")
+    
+    /// The unicode61 tokenizer.
+    ///
+    /// For example:
+    ///
+    /// ```swift
+    /// try db.create(virtualTable: "book", using: FTS4()) { t in
+    ///     t.tokenizer = .unicode61()
+    /// }
+    /// ```
+    ///
+    /// Related SQLite documentation: <https://www.sqlite.org/fts3.html#tokenizer>
+    ///
+    /// - parameters:
+    ///     - diacritics: By default SQLite will strip diacritics from
+    ///       latin characters.
+    ///     - separators: Unless empty (the default), SQLite will consider these
+    ///       characters as token separators.
+    ///     - tokenCharacters: Unless empty (the default), SQLite will consider
+    ///       these characters as token characters.
+    public static func unicode61(
+        diacritics: FTS3.Diacritics = .removeLegacy,
+        separators: Set<Character> = [],
+        tokenCharacters: Set<Character> = [])
+    -> FTS3TokenizerDescriptor
+    {
+        _unicode61(diacritics: diacritics, separators: separators, tokenCharacters: tokenCharacters)
+    }
+    
+    private static func _unicode61(
+        diacritics: FTS3.Diacritics,
+        separators: Set<Character> = [],
+        tokenCharacters: Set<Character> = [])
+    -> FTS3TokenizerDescriptor
+    {
+        var arguments: [String] = []
+        switch diacritics {
+        case .removeLegacy:
+            break
+        case .keep:
+            arguments.append("remove_diacritics=0")
+        #if GRDBCUSTOMSQLITE
+        case .remove:
+            arguments.append("remove_diacritics=2")
+        #elseif !GRDBCIPHER
+        case .remove:
+            arguments.append("remove_diacritics=2")
+        #endif
+        }
+        if !separators.isEmpty {
+            // TODO: test "=" and "\"", "(" and ")" as separators, with
+            // both FTS3Pattern(matchingAnyTokenIn:tokenizer:)
+            // and Database.create(virtualTable:using:)
+            arguments.append("separators=" + separators.sorted().map { String($0) }.joined())
+        }
+        if !tokenCharacters.isEmpty {
+            // TODO: test "=" and "\"", "(" and ")" as tokenCharacters, with
+            // both FTS3Pattern(matchingAnyTokenIn:tokenizer:)
+            // and Database.create(virtualTable:using:)
+            arguments.append("tokenchars=" + tokenCharacters.sorted().map { String($0) }.joined())
+        }
+        return FTS3TokenizerDescriptor("unicode61", arguments: arguments)
+    }
+}