234 lines
12 KiB
Swift
234 lines
12 KiB
Swift
import XCTest
|
||
import GRDB
|
||
|
||
class FTS3TokenizerTests: GRDBTestCase {
|
||
|
||
private func match(_ db: Database, _ content: String, _ query: String) -> Bool {
|
||
try! db.execute(sql: "INSERT INTO documents VALUES (?)", arguments: [content])
|
||
defer {
|
||
try! db.execute(sql: "DELETE FROM documents")
|
||
}
|
||
return try! Int.fetchOne(db, sql: "SELECT COUNT(*) FROM documents WHERE documents MATCH ?", arguments: [query])! > 0
|
||
}
|
||
|
||
func testSimpleTokenizer() throws {
|
||
let dbQueue = try makeDatabaseQueue()
|
||
try dbQueue.inDatabase { db in
|
||
try db.create(virtualTable: "documents", using: FTS3()) { t in
|
||
t.tokenizer = .simple
|
||
}
|
||
|
||
// simple match
|
||
XCTAssertTrue(match(db, "abcDÉF", "abcDÉF"))
|
||
|
||
// English stemming
|
||
XCTAssertFalse(match(db, "database", "databases"))
|
||
|
||
// diacritics in latin characters
|
||
XCTAssertFalse(match(db, "eéÉ", "Èèe"))
|
||
|
||
// unicode case
|
||
XCTAssertFalse(match(db, "jérôme", "JÉRÔME"))
|
||
}
|
||
}
|
||
|
||
func testPorterTokenizer() throws {
|
||
let dbQueue = try makeDatabaseQueue()
|
||
try dbQueue.inDatabase { db in
|
||
try db.create(virtualTable: "documents", using: FTS3()) { t in
|
||
t.tokenizer = .porter
|
||
}
|
||
|
||
// simple match
|
||
XCTAssertTrue(match(db, "abcDÉF", "abcDÉF"))
|
||
|
||
// English stemming
|
||
XCTAssertTrue(match(db, "database", "databases"))
|
||
|
||
// diacritics in latin characters
|
||
XCTAssertFalse(match(db, "eéÉ", "Èèe"))
|
||
|
||
// unicode case
|
||
XCTAssertFalse(match(db, "jérôme", "JÉRÔME"))
|
||
}
|
||
}
|
||
|
||
func testUnicode61Tokenizer() throws {
|
||
let dbQueue = try makeDatabaseQueue()
|
||
try dbQueue.inDatabase { db in
|
||
try db.create(virtualTable: "documents", using: FTS3()) { t in
|
||
t.tokenizer = .unicode61()
|
||
}
|
||
|
||
// simple match
|
||
XCTAssertTrue(match(db, "abcDÉF", "abcDÉF"))
|
||
|
||
// English stemming
|
||
XCTAssertFalse(match(db, "database", "databases"))
|
||
|
||
// diacritics in latin characters
|
||
XCTAssertTrue(match(db, "eéÉ", "Èèe"))
|
||
|
||
// unicode case
|
||
XCTAssertTrue(match(db, "jérôme", "JÉRÔME"))
|
||
}
|
||
}
|
||
|
||
func testUnicode61TokenizerDiacriticsKeep() throws {
|
||
let dbQueue = try makeDatabaseQueue()
|
||
try dbQueue.inDatabase { db in
|
||
try db.create(virtualTable: "documents", using: FTS3()) { t in
|
||
t.tokenizer = .unicode61(diacritics: .keep)
|
||
}
|
||
|
||
// simple match
|
||
XCTAssertTrue(match(db, "abcDÉF", "abcDÉF"))
|
||
|
||
// English stemming
|
||
XCTAssertFalse(match(db, "database", "databases"))
|
||
|
||
// diacritics in latin characters
|
||
XCTAssertFalse(match(db, "eéÉ", "Èèe"))
|
||
|
||
// unicode case
|
||
XCTAssertTrue(match(db, "jérôme", "JÉRÔME"))
|
||
}
|
||
}
|
||
|
||
#if GRDBCUSTOMSQLITE
|
||
func testUnicode61TokenizerDiacriticsRemove() throws {
|
||
let dbQueue = try makeDatabaseQueue()
|
||
try dbQueue.inDatabase { db in
|
||
try db.create(virtualTable: "documents", using: FTS3()) { t in
|
||
t.tokenizer = .unicode61(diacritics: .remove)
|
||
}
|
||
|
||
// simple match
|
||
XCTAssertTrue(match(db, "abcDÉF", "abcDÉF"))
|
||
|
||
// English stemming
|
||
XCTAssertFalse(match(db, "database", "databases"))
|
||
|
||
// diacritics in latin characters
|
||
XCTAssertTrue(match(db, "eéÉ", "Èèe"))
|
||
|
||
// unicode case
|
||
XCTAssertTrue(match(db, "jérôme", "JÉRÔME"))
|
||
}
|
||
}
|
||
#endif
|
||
|
||
func testUnicode61TokenizerSeparators() throws {
|
||
let dbQueue = try makeDatabaseQueue()
|
||
try dbQueue.inDatabase { db in
|
||
try db.create(virtualTable: "documents", using: FTS3()) { t in
|
||
t.tokenizer = .unicode61(separators: ["X"])
|
||
}
|
||
|
||
XCTAssertTrue(match(db, "abcXdef", "abcXdef"))
|
||
XCTAssertTrue(match(db, "abcXdef", "defXabc"))
|
||
XCTAssertTrue(match(db, "abcXdef", "abc"))
|
||
XCTAssertTrue(match(db, "abcXdef", "def"))
|
||
}
|
||
}
|
||
|
||
func testUnicode61TokenizerTokenCharacters() throws {
|
||
let dbQueue = try makeDatabaseQueue()
|
||
try dbQueue.inDatabase { db in
|
||
try db.create(virtualTable: "documents", using: FTS3()) { t in
|
||
t.tokenizer = .unicode61(tokenCharacters: Set(".-"))
|
||
}
|
||
|
||
XCTAssertTrue(match(db, "2016-10-04.txt", "2016-10-04.txt"))
|
||
XCTAssertFalse(match(db, "2016-10-04.txt", "2016"))
|
||
XCTAssertFalse(match(db, "2016-10-04.txt", "txt"))
|
||
}
|
||
}
|
||
|
||
func testTokenize() throws {
|
||
// Empty query
|
||
try XCTAssertEqual(FTS3.tokenize(""), [])
|
||
try XCTAssertEqual(FTS3.tokenize("", withTokenizer: .simple), [])
|
||
try XCTAssertEqual(FTS3.tokenize("", withTokenizer: .porter), [])
|
||
try XCTAssertEqual(FTS3.tokenize("", withTokenizer: .unicode61()), [])
|
||
try XCTAssertEqual(FTS3.tokenize("", withTokenizer: .unicode61(diacritics: .keep)), [])
|
||
|
||
try XCTAssertEqual(FTS3.tokenize("?!"), [])
|
||
try XCTAssertEqual(FTS3.tokenize("?!", withTokenizer: .simple), [])
|
||
try XCTAssertEqual(FTS3.tokenize("?!", withTokenizer: .porter), [])
|
||
try XCTAssertEqual(FTS3.tokenize("?!", withTokenizer: .unicode61()), [])
|
||
try XCTAssertEqual(FTS3.tokenize("?!", withTokenizer: .unicode61(diacritics: .keep)), [])
|
||
|
||
// Token queries
|
||
try XCTAssertEqual(FTS3.tokenize("Moby"), ["moby"])
|
||
try XCTAssertEqual(FTS3.tokenize("Moby", withTokenizer: .simple), ["moby"])
|
||
try XCTAssertEqual(FTS3.tokenize("Moby", withTokenizer: .porter), ["mobi"])
|
||
try XCTAssertEqual(FTS3.tokenize("Moby", withTokenizer: .unicode61()), ["moby"])
|
||
try XCTAssertEqual(FTS3.tokenize("Moby", withTokenizer: .unicode61(diacritics: .keep)), ["moby"])
|
||
|
||
try XCTAssertEqual(FTS3.tokenize("écarlates"), ["écarlates"])
|
||
try XCTAssertEqual(FTS3.tokenize("écarlates", withTokenizer: .simple), ["écarlates"])
|
||
try XCTAssertEqual(FTS3.tokenize("écarlates", withTokenizer: .porter), ["écarlates"])
|
||
try XCTAssertEqual(FTS3.tokenize("écarlates", withTokenizer: .unicode61()), ["ecarlates"])
|
||
try XCTAssertEqual(FTS3.tokenize("écarlates", withTokenizer: .unicode61(diacritics: .keep)), ["écarlates"])
|
||
|
||
try XCTAssertEqual(FTS3.tokenize("fooéı👨👨🏿🇫🇷🇨🇮"), ["fooéı👨👨🏿🇫🇷🇨🇮"])
|
||
try XCTAssertEqual(FTS3.tokenize("fooéı👨👨🏿🇫🇷🇨🇮", withTokenizer: .simple), ["fooéı👨👨🏿🇫🇷🇨🇮"])
|
||
try XCTAssertEqual(FTS3.tokenize("fooéı👨👨🏿🇫🇷🇨🇮", withTokenizer: .porter), ["fooéı👇�🇨🇮"]) // ¯\_(ツ)_/¯
|
||
try XCTAssertEqual(FTS3.tokenize("fooéı👨👨🏿🇫🇷🇨🇮", withTokenizer: .unicode61()), ["fooeı", "🏿"]) // ¯\_(ツ)_/¯
|
||
try XCTAssertEqual(FTS3.tokenize("fooéı👨👨🏿🇫🇷🇨🇮", withTokenizer: .unicode61(diacritics: .keep)), ["fooéı", "🏿"]) // ¯\_(ツ)_/¯
|
||
|
||
try XCTAssertEqual(FTS3.tokenize("SQLite database"), ["sqlite", "database"])
|
||
try XCTAssertEqual(FTS3.tokenize("SQLite database", withTokenizer: .simple), ["sqlite", "database"])
|
||
try XCTAssertEqual(FTS3.tokenize("SQLite database", withTokenizer: .porter), ["sqlite", "databas"])
|
||
try XCTAssertEqual(FTS3.tokenize("SQLite database", withTokenizer: .unicode61()), ["sqlite", "database"])
|
||
try XCTAssertEqual(FTS3.tokenize("SQLite database", withTokenizer: .unicode61(diacritics: .keep)), ["sqlite", "database"])
|
||
|
||
try XCTAssertEqual(FTS3.tokenize("Édouard Manet"), ["Édouard", "manet"])
|
||
try XCTAssertEqual(FTS3.tokenize("Édouard Manet", withTokenizer: .simple), ["Édouard", "manet"])
|
||
try XCTAssertEqual(FTS3.tokenize("Édouard Manet", withTokenizer: .porter), ["Édouard", "manet"])
|
||
try XCTAssertEqual(FTS3.tokenize("Édouard Manet", withTokenizer: .unicode61()), ["edouard", "manet"])
|
||
try XCTAssertEqual(FTS3.tokenize("Édouard Manet", withTokenizer: .unicode61(diacritics: .keep)), ["édouard", "manet"])
|
||
|
||
// Prefix queries
|
||
try XCTAssertEqual(FTS3.tokenize("*"), [])
|
||
try XCTAssertEqual(FTS3.tokenize("*", withTokenizer: .simple), [])
|
||
try XCTAssertEqual(FTS3.tokenize("*", withTokenizer: .porter), [])
|
||
try XCTAssertEqual(FTS3.tokenize("*", withTokenizer: .unicode61()), [])
|
||
try XCTAssertEqual(FTS3.tokenize("*", withTokenizer: .unicode61(diacritics: .keep)), [])
|
||
|
||
try XCTAssertEqual(FTS3.tokenize("Robin*"), ["robin"])
|
||
try XCTAssertEqual(FTS3.tokenize("Robin*", withTokenizer: .simple), ["robin"])
|
||
try XCTAssertEqual(FTS3.tokenize("Robin*", withTokenizer: .porter), ["robin"])
|
||
try XCTAssertEqual(FTS3.tokenize("Robin*", withTokenizer: .unicode61()), ["robin"])
|
||
try XCTAssertEqual(FTS3.tokenize("Robin*", withTokenizer: .unicode61(diacritics: .keep)), ["robin"])
|
||
|
||
// Phrase queries
|
||
try XCTAssertEqual(FTS3.tokenize("\"foulent muscles\""), ["foulent", "muscles"])
|
||
try XCTAssertEqual(FTS3.tokenize("\"foulent muscles\"", withTokenizer: .simple), ["foulent", "muscles"])
|
||
try XCTAssertEqual(FTS3.tokenize("\"foulent muscles\"", withTokenizer: .porter), ["foulent", "muscl"])
|
||
try XCTAssertEqual(FTS3.tokenize("\"foulent muscles\"", withTokenizer: .unicode61()), ["foulent", "muscles"])
|
||
try XCTAssertEqual(FTS3.tokenize("\"foulent muscles\"", withTokenizer: .unicode61(diacritics: .keep)), ["foulent", "muscles"])
|
||
|
||
try XCTAssertEqual(FTS3.tokenize("\"Kim Stan* Robin*\""), ["kim", "stan", "robin"])
|
||
try XCTAssertEqual(FTS3.tokenize("\"Kim Stan* Robin*\"", withTokenizer: .simple), ["kim", "stan", "robin"])
|
||
try XCTAssertEqual(FTS3.tokenize("\"Kim Stan* Robin*\"", withTokenizer: .porter), ["kim", "stan", "robin"])
|
||
try XCTAssertEqual(FTS3.tokenize("\"Kim Stan* Robin*\"", withTokenizer: .unicode61()), ["kim", "stan", "robin"])
|
||
try XCTAssertEqual(FTS3.tokenize("\"Kim Stan* Robin*\"", withTokenizer: .unicode61(diacritics: .keep)), ["kim", "stan", "robin"])
|
||
|
||
// Logical queries
|
||
try XCTAssertEqual(FTS3.tokenize("years AND months"), ["years", "and", "months"])
|
||
try XCTAssertEqual(FTS3.tokenize("years AND months", withTokenizer: .simple), ["years", "and", "months"])
|
||
try XCTAssertEqual(FTS3.tokenize("years AND months", withTokenizer: .porter), ["year", "and", "month"])
|
||
try XCTAssertEqual(FTS3.tokenize("years AND months", withTokenizer: .unicode61()), ["years", "and", "months"])
|
||
try XCTAssertEqual(FTS3.tokenize("years AND months", withTokenizer: .unicode61(diacritics: .keep)), ["years", "and", "months"])
|
||
|
||
// column queries
|
||
try XCTAssertEqual(FTS3.tokenize("title:brest"), ["title", "brest"])
|
||
try XCTAssertEqual(FTS3.tokenize("title:brest", withTokenizer: .simple), ["title", "brest"])
|
||
try XCTAssertEqual(FTS3.tokenize("title:brest", withTokenizer: .porter), ["titl", "brest"])
|
||
try XCTAssertEqual(FTS3.tokenize("title:brest", withTokenizer: .unicode61()), ["title", "brest"])
|
||
try XCTAssertEqual(FTS3.tokenize("title:brest", withTokenizer: .unicode61(diacritics: .keep)), ["title", "brest"])
|
||
}
|
||
}
|