From 18bf9487801e92de9b7b5b9009d0f074e29c66ff Mon Sep 17 00:00:00 2001 From: meili-bot <74670311+meili-bot@users.noreply.github.com> Date: Sun, 7 Apr 2024 20:35:02 +0200 Subject: [PATCH 1/8] Update README.md From d72554925ea8e466b52d6f2b6ce73008beaf5fc2 Mon Sep 17 00:00:00 2001 From: amit-ksh Date: Tue, 16 Apr 2024 23:16:36 +0530 Subject: [PATCH 2/8] Add searchCutoffMs index setting --- src/indexes.ts | 42 ++++++++ src/types/types.ts | 8 ++ tests/searchCutoffMs.ts | 225 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 275 insertions(+) create mode 100644 tests/searchCutoffMs.ts diff --git a/src/indexes.ts b/src/indexes.ts index c26cf97dc..f8a84801f 100644 --- a/src/indexes.ts +++ b/src/indexes.ts @@ -52,6 +52,7 @@ import { Dictionary, ProximityPrecision, Embedders, + SearchCutoffMsSettings, } from './types' import { removeUndefinedFromObject } from './utils' import { HttpRequests } from './http-requests' @@ -1334,6 +1335,47 @@ class Index = Record> { return task } + + /// + /// SEARCHCUTOFFMS SETTINGS + /// + + /** + * Get the SearchCutoffMs settings. + * + * @returns Promise containing object of SearchCutoffMs settings + */ + async getSearchCutoffMs(): Promise { + const url = `indexes/${this.uid}/settings/search-cutoff-ms` + return await this.httpRequest.get(url) + } + + /** + * Update the SearchCutoffMs settings. + * + * @param searchCutoffMs - Object containing SearchCutoffMsSettings + * @returns Promise containing an EnqueuedTask + */ + async updateSearchCutoffMs( + searchCutoffMs: SearchCutoffMsSettings + ): Promise { + const url = `indexes/${this.uid}/settings/search-cutoff-ms` + const task = await this.httpRequest.patch(url, searchCutoffMs) + + return new EnqueuedTask(task) + } + + /** + * Reset the SearchCutoffMs settings. + * + * @returns Promise containing an EnqueuedTask + */ + async resetSearchCutoffMs(): Promise { + const url = `indexes/${this.uid}/settings/search-cutoff-ms` + const task = await this.httpRequest.delete(url) + + return new EnqueuedTask(task) + } } export { Index } diff --git a/src/types/types.ts b/src/types/types.ts index 1e0e5fc03..606d20ccb 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -373,6 +373,10 @@ export type PaginationSettings = { maxTotalHits?: number | null } +export type SearchCutoffMsSettings = { + searchCutoffMs?: number | null +} + export type Settings = { filterableAttributes?: FilterableAttributes distinctAttribute?: DistinctAttribute @@ -390,6 +394,7 @@ export type Settings = { dictionary?: Dictionary proximityPrecision?: ProximityPrecision embedders?: Embedders + searchCutoffMs?: SearchCutoffMsSettings } /* @@ -930,6 +935,9 @@ export const ErrorStatusCode = { /** @see https://www.meilisearch.com/docs/reference/errors/error_codes#invalid_settings_pagination */ INVALID_SETTINGS_PAGINATION: 'invalid_settings_pagination', + /** @see https://www.meilisearch.com/docs/reference/errors/error_codes#invalid_settings_search_cutoff_ms */ + INVALID_SETTINGS_SEARCH_CUTOFF_MS: 'invalid_settings_search_cutoff_ms', + /** @see https://www.meilisearch.com/docs/reference/errors/error_codes#invalid_task_before_enqueued_at */ INVALID_TASK_BEFORE_ENQUEUED_AT: 'invalid_task_before_enqueued_at', diff --git a/tests/searchCutoffMs.ts b/tests/searchCutoffMs.ts new file mode 100644 index 000000000..3b5c853d6 --- /dev/null +++ b/tests/searchCutoffMs.ts @@ -0,0 +1,225 @@ +import { ErrorStatusCode } from '../src/types' +import { + clearAllIndexes, + config, + BAD_HOST, + MeiliSearch, + getClient, + dataset, +} from './utils/meilisearch-test-utils' + +const index = { + uid: 'movies_test', +} + +const DEFAULT_SEARCHCUTOFFMS = 1500 + +jest.setTimeout(100 * 1000) + +afterAll(() => { + return clearAllIndexes(config) +}) + +describe.each([{ permission: 'Master' }, { permission: 'Admin' }])( + 'Test on searchCutoffMs', + ({ permission }) => { + beforeEach(async () => { + await clearAllIndexes(config) + const client = await getClient('Master') + const { taskUid } = await client.index(index.uid).addDocuments(dataset) + await client.waitForTask(taskUid) + }) + + test(`${permission} key: Get default searchCutoffMs settings`, async () => { + const client = await getClient(permission) + const response = await client.index(index.uid).getSearchCutoffMs() + + expect(response).toEqual({ searchCutoffMs: DEFAULT_SEARCHCUTOFFMS }) + }) + + test(`${permission} key: Update searchCutoffMs to valid value`, async () => { + const client = await getClient(permission) + const newSearchCutoffMs = { + searchCutoffMs: 100, + } + const task = await client + .index(index.uid) + .updateSearchCutoffMs(newSearchCutoffMs) + await client.waitForTask(task.taskUid) + + const response = await client.index(index.uid).getSearchCutoffMs() + + expect(response).toEqual(newSearchCutoffMs) + }) + + test(`${permission} key: Update searchCutoffMs to null`, async () => { + const client = await getClient(permission) + const newSearchCutoffMs = { + searchCutoffMs: null, + } + const task = await client + .index(index.uid) + .updateSearchCutoffMs(newSearchCutoffMs) + await client.index(index.uid).waitForTask(task.taskUid) + + const response = await client.index(index.uid).getSearchCutoffMs() + + expect(response).toEqual({ searchCutoffMs: DEFAULT_SEARCHCUTOFFMS }) + }) + + test(`${permission} key: Update searchCutoffMs with invalid value`, async () => { + const client = await getClient(permission) + const newSearchCutoffMs = { + searchCutoffMs: 'hello', // bad searchCutoffMs value + } as any + + await expect( + client.index(index.uid).updateSearchCutoffMs(newSearchCutoffMs) + ).rejects.toHaveProperty( + 'code', + ErrorStatusCode.INVALID_SETTINGS_SEARCH_CUTOFF_MS + ) + }) + + test(`${permission} key: Reset searchCutoffMs`, async () => { + const client = await getClient(permission) + const newSearchCutoffMs = { + searchCutoffMs: 100, + } + const updateTask = await client + .index(index.uid) + .updateSearchCutoffMs(newSearchCutoffMs) + await client.waitForTask(updateTask.taskUid) + const task = await client.index(index.uid).resetSearchCutoffMs() + await client.waitForTask(task.taskUid) + + const response = await client.index(index.uid).getSearchCutoffMs() + + expect(response).toEqual({ searchCutoffMs: DEFAULT_SEARCHCUTOFFMS }) + }) + } +) + +describe.each([{ permission: 'Search' }])( + 'Test on searchCutoffMs', + ({ permission }) => { + beforeEach(async () => { + const client = await getClient('Master') + const { taskUid } = await client.createIndex(index.uid) + await client.waitForTask(taskUid) + }) + + test(`${permission} key: try to get searchCutoffMs and be denied`, async () => { + const client = await getClient(permission) + await expect( + client.index(index.uid).getSearchCutoffMs() + ).rejects.toHaveProperty('code', ErrorStatusCode.INVALID_API_KEY) + }) + + test(`${permission} key: try to update searchCutoffMs and be denied`, async () => { + const client = await getClient(permission) + await expect( + client.index(index.uid).updateSearchCutoffMs({ searchCutoffMs: 100 }) + ).rejects.toHaveProperty('code', ErrorStatusCode.INVALID_API_KEY) + }) + + test(`${permission} key: try to reset searchCutoffMs and be denied`, async () => { + const client = await getClient(permission) + await expect( + client.index(index.uid).resetSearchCutoffMs() + ).rejects.toHaveProperty('code', ErrorStatusCode.INVALID_API_KEY) + }) + } +) + +describe.each([{ permission: 'No' }])( + 'Test on searchCutoffMs', + ({ permission }) => { + beforeAll(async () => { + const client = await getClient('Master') + const { taskUid } = await client.createIndex(index.uid) + await client.waitForTask(taskUid) + }) + + test(`${permission} key: try to get searchCutoffMs and be denied`, async () => { + const client = await getClient(permission) + await expect( + client.index(index.uid).getSearchCutoffMs() + ).rejects.toHaveProperty( + 'code', + ErrorStatusCode.MISSING_AUTHORIZATION_HEADER + ) + }) + + test(`${permission} key: try to update searchCutoffMs and be denied`, async () => { + const client = await getClient(permission) + await expect( + client.index(index.uid).updateSearchCutoffMs({ searchCutoffMs: 100 }) + ).rejects.toHaveProperty( + 'code', + ErrorStatusCode.MISSING_AUTHORIZATION_HEADER + ) + }) + + test(`${permission} key: try to reset searchCutoffMs and be denied`, async () => { + const client = await getClient(permission) + await expect( + client.index(index.uid).resetSearchCutoffMs() + ).rejects.toHaveProperty( + 'code', + ErrorStatusCode.MISSING_AUTHORIZATION_HEADER + ) + }) + } +) + +describe.each([ + { host: BAD_HOST, trailing: false }, + { host: `${BAD_HOST}/api`, trailing: false }, + { host: `${BAD_HOST}/trailing/`, trailing: true }, +])('Tests on url construction', ({ host, trailing }) => { + test(`Test getSearchCutoffMs route`, async () => { + const route = `indexes/${index.uid}/settings/search-cutoff-ms` + const client = new MeiliSearch({ host }) + const strippedHost = trailing ? host.slice(0, -1) : host + await expect( + client.index(index.uid).getSearchCutoffMs() + ).rejects.toHaveProperty( + 'message', + `request to ${strippedHost}/${route} failed, reason: connect ECONNREFUSED ${BAD_HOST.replace( + 'http://', + '' + )}` + ) + }) + + test(`Test updateSearchCutoffMs route`, async () => { + const route = `indexes/${index.uid}/settings/search-cutoff-ms` + const client = new MeiliSearch({ host }) + const strippedHost = trailing ? host.slice(0, -1) : host + await expect( + client.index(index.uid).updateSearchCutoffMs({ searchCutoffMs: null }) + ).rejects.toHaveProperty( + 'message', + `request to ${strippedHost}/${route} failed, reason: connect ECONNREFUSED ${BAD_HOST.replace( + 'http://', + '' + )}` + ) + }) + + test(`Test resetSearchCutoffMs route`, async () => { + const route = `indexes/${index.uid}/settings/search-cutoff-ms` + const client = new MeiliSearch({ host }) + const strippedHost = trailing ? host.slice(0, -1) : host + await expect( + client.index(index.uid).resetSearchCutoffMs() + ).rejects.toHaveProperty( + 'message', + `request to ${strippedHost}/${route} failed, reason: connect ECONNREFUSED ${BAD_HOST.replace( + 'http://', + '' + )}` + ) + }) +}) From b26075ea1ad97026f98873304ae5fa0d651e3b64 Mon Sep 17 00:00:00 2001 From: amit-ksh Date: Thu, 18 Apr 2024 18:45:06 +0530 Subject: [PATCH 3/8] Add null to Embedder type --- src/types/types.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/types/types.ts b/src/types/types.ts index 1e0e5fc03..2ed300174 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -359,6 +359,7 @@ export type Embedder = | OpenAiEmbedder | HuggingFaceEmbedder | UserProvidedEmbedder + | null export type Embedders = Record | null From c1fdf310ee47add5fbec51a2be311cb32a58ca8d Mon Sep 17 00:00:00 2001 From: amit-ksh Date: Fri, 19 Apr 2024 14:00:47 +0530 Subject: [PATCH 4/8] Update SearchCutoffMs --- README.md | 22 ++++++++++++++++- src/indexes.ts | 10 ++++---- src/types/types.ts | 6 ++--- tests/__snapshots__/settings.test.ts.snap | 18 ++++++++++++++ tests/searchCutoffMs.ts | 30 +++++++++-------------- tests/settings.test.ts | 4 ++- 6 files changed, 60 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index df76a25f7..1bf364810 100644 --- a/README.md +++ b/README.md @@ -983,7 +983,7 @@ client.index('myIndex').resetProximityPrecision(): Promise ### Embedders -⚠️ This feature is experimental. Activate the [`vectorStore` experimental feature to use it](https://www.meilisearch.com/docs/reference/api/experimental_features#configure-experimental-features) +⚠️ This feature is experimental. Activate the [`vectorStore` experimental feature to use it](https://www.meilisearch.com/docs/reference/api/experimental_features#configure-experimental-features) #### [Get embedders](https://www.meilisearch.com/docs/reference/api/settings#get-embedders) @@ -1003,6 +1003,26 @@ client.index('myIndex').updateEmbedders(embedders: Embedders): Promise ``` +### SearchCutoffMs + +#### [Get SearchCutoffMs](https://www.meilisearch.com/docs/reference/api/settings#get-search-cutoff-ms) + +```ts +client.index('myIndex').getSearchCutoffMs(): Promise +``` + +#### [Update SearchCutoffMs](https://www.meilisearch.com/docs/reference/api/settings#update-search-cutoff-ms) + +```ts +client.index('myIndex').updateSearchCutoffMs(searchCutoffMs: SearchCutoffMs): Promise +``` + +#### [Reset SearchCutoffMs](https://www.meilisearch.com/docs/reference/api/settings#reset-search-cutoff-ms) + +```ts +client.index('myIndex').resetSearchCutoffMs(): Promise +``` + ### Keys #### [Get keys](https://www.meilisearch.com/docs/reference/api/keys#get-all-keys) diff --git a/src/indexes.ts b/src/indexes.ts index f8a84801f..1690402db 100644 --- a/src/indexes.ts +++ b/src/indexes.ts @@ -52,7 +52,7 @@ import { Dictionary, ProximityPrecision, Embedders, - SearchCutoffMsSettings, + SearchCutoffMs, } from './types' import { removeUndefinedFromObject } from './utils' import { HttpRequests } from './http-requests' @@ -1345,9 +1345,9 @@ class Index = Record> { * * @returns Promise containing object of SearchCutoffMs settings */ - async getSearchCutoffMs(): Promise { + async getSearchCutoffMs(): Promise { const url = `indexes/${this.uid}/settings/search-cutoff-ms` - return await this.httpRequest.get(url) + return await this.httpRequest.get(url) } /** @@ -1357,10 +1357,10 @@ class Index = Record> { * @returns Promise containing an EnqueuedTask */ async updateSearchCutoffMs( - searchCutoffMs: SearchCutoffMsSettings + searchCutoffMs: SearchCutoffMs ): Promise { const url = `indexes/${this.uid}/settings/search-cutoff-ms` - const task = await this.httpRequest.patch(url, searchCutoffMs) + const task = await this.httpRequest.put(url, searchCutoffMs) return new EnqueuedTask(task) } diff --git a/src/types/types.ts b/src/types/types.ts index 606d20ccb..8b66e0d73 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -373,9 +373,7 @@ export type PaginationSettings = { maxTotalHits?: number | null } -export type SearchCutoffMsSettings = { - searchCutoffMs?: number | null -} +export type SearchCutoffMs = number | null export type Settings = { filterableAttributes?: FilterableAttributes @@ -394,7 +392,7 @@ export type Settings = { dictionary?: Dictionary proximityPrecision?: ProximityPrecision embedders?: Embedders - searchCutoffMs?: SearchCutoffMsSettings + searchCutoffMs?: SearchCutoffMs } /* diff --git a/tests/__snapshots__/settings.test.ts.snap b/tests/__snapshots__/settings.test.ts.snap index 654f57729..5252c6889 100644 --- a/tests/__snapshots__/settings.test.ts.snap +++ b/tests/__snapshots__/settings.test.ts.snap @@ -27,6 +27,7 @@ exports[`Test on settings Admin key: Get default settings of an index 1`] = ` "sort", "exactness", ], + "searchCutoffMs": null, "searchableAttributes": [ "*", ], @@ -73,6 +74,7 @@ exports[`Test on settings Admin key: Get default settings of empty index with pr "sort", "exactness", ], + "searchCutoffMs": null, "searchableAttributes": [ "*", ], @@ -119,6 +121,7 @@ exports[`Test on settings Admin key: Reset settings 1`] = ` "sort", "exactness", ], + "searchCutoffMs": null, "searchableAttributes": [ "*", ], @@ -165,6 +168,7 @@ exports[`Test on settings Admin key: Reset settings of empty index 1`] = ` "sort", "exactness", ], + "searchCutoffMs": null, "searchableAttributes": [ "*", ], @@ -211,6 +215,7 @@ exports[`Test on settings Admin key: Update searchableAttributes settings on emp "sort", "exactness", ], + "searchCutoffMs": null, "searchableAttributes": [ "title", ], @@ -257,6 +262,7 @@ exports[`Test on settings Admin key: Update searchableAttributes settings on emp "sort", "exactness", ], + "searchCutoffMs": null, "searchableAttributes": [ "title", ], @@ -308,6 +314,7 @@ exports[`Test on settings Admin key: Update settings 1`] = ` "id:asc", "typo", ], + "searchCutoffMs": 1000, "searchableAttributes": [ "title", ], @@ -366,6 +373,7 @@ exports[`Test on settings Admin key: Update settings on empty index with primary "title:asc", "typo", ], + "searchCutoffMs": null, "searchableAttributes": [ "*", ], @@ -414,6 +422,7 @@ exports[`Test on settings Admin key: Update settings with all null values 1`] = "sort", "exactness", ], + "searchCutoffMs": null, "searchableAttributes": [ "*", ], @@ -460,6 +469,7 @@ exports[`Test on settings Master key: Get default settings of an index 1`] = ` "sort", "exactness", ], + "searchCutoffMs": null, "searchableAttributes": [ "*", ], @@ -506,6 +516,7 @@ exports[`Test on settings Master key: Get default settings of empty index with p "sort", "exactness", ], + "searchCutoffMs": null, "searchableAttributes": [ "*", ], @@ -552,6 +563,7 @@ exports[`Test on settings Master key: Reset settings 1`] = ` "sort", "exactness", ], + "searchCutoffMs": null, "searchableAttributes": [ "*", ], @@ -598,6 +610,7 @@ exports[`Test on settings Master key: Reset settings of empty index 1`] = ` "sort", "exactness", ], + "searchCutoffMs": null, "searchableAttributes": [ "*", ], @@ -644,6 +657,7 @@ exports[`Test on settings Master key: Update searchableAttributes settings on em "sort", "exactness", ], + "searchCutoffMs": null, "searchableAttributes": [ "title", ], @@ -690,6 +704,7 @@ exports[`Test on settings Master key: Update searchableAttributes settings on em "sort", "exactness", ], + "searchCutoffMs": null, "searchableAttributes": [ "title", ], @@ -741,6 +756,7 @@ exports[`Test on settings Master key: Update settings 1`] = ` "id:asc", "typo", ], + "searchCutoffMs": 1000, "searchableAttributes": [ "title", ], @@ -799,6 +815,7 @@ exports[`Test on settings Master key: Update settings on empty index with primar "title:asc", "typo", ], + "searchCutoffMs": null, "searchableAttributes": [ "*", ], @@ -847,6 +864,7 @@ exports[`Test on settings Master key: Update settings with all null values 1`] = "sort", "exactness", ], + "searchCutoffMs": null, "searchableAttributes": [ "*", ], diff --git a/tests/searchCutoffMs.ts b/tests/searchCutoffMs.ts index 3b5c853d6..b0cb391aa 100644 --- a/tests/searchCutoffMs.ts +++ b/tests/searchCutoffMs.ts @@ -12,7 +12,7 @@ const index = { uid: 'movies_test', } -const DEFAULT_SEARCHCUTOFFMS = 1500 +const DEFAULT_SEARCHCUTOFFMS = null jest.setTimeout(100 * 1000) @@ -34,14 +34,12 @@ describe.each([{ permission: 'Master' }, { permission: 'Admin' }])( const client = await getClient(permission) const response = await client.index(index.uid).getSearchCutoffMs() - expect(response).toEqual({ searchCutoffMs: DEFAULT_SEARCHCUTOFFMS }) + expect(response).toEqual(DEFAULT_SEARCHCUTOFFMS) }) test(`${permission} key: Update searchCutoffMs to valid value`, async () => { const client = await getClient(permission) - const newSearchCutoffMs = { - searchCutoffMs: 100, - } + const newSearchCutoffMs = 100 const task = await client .index(index.uid) .updateSearchCutoffMs(newSearchCutoffMs) @@ -54,9 +52,7 @@ describe.each([{ permission: 'Master' }, { permission: 'Admin' }])( test(`${permission} key: Update searchCutoffMs to null`, async () => { const client = await getClient(permission) - const newSearchCutoffMs = { - searchCutoffMs: null, - } + const newSearchCutoffMs = null const task = await client .index(index.uid) .updateSearchCutoffMs(newSearchCutoffMs) @@ -64,14 +60,12 @@ describe.each([{ permission: 'Master' }, { permission: 'Admin' }])( const response = await client.index(index.uid).getSearchCutoffMs() - expect(response).toEqual({ searchCutoffMs: DEFAULT_SEARCHCUTOFFMS }) + expect(response).toEqual(DEFAULT_SEARCHCUTOFFMS) }) test(`${permission} key: Update searchCutoffMs with invalid value`, async () => { const client = await getClient(permission) - const newSearchCutoffMs = { - searchCutoffMs: 'hello', // bad searchCutoffMs value - } as any + const newSearchCutoffMs = 'hello' as any // bad searchCutoffMs value await expect( client.index(index.uid).updateSearchCutoffMs(newSearchCutoffMs) @@ -83,9 +77,7 @@ describe.each([{ permission: 'Master' }, { permission: 'Admin' }])( test(`${permission} key: Reset searchCutoffMs`, async () => { const client = await getClient(permission) - const newSearchCutoffMs = { - searchCutoffMs: 100, - } + const newSearchCutoffMs = 100 const updateTask = await client .index(index.uid) .updateSearchCutoffMs(newSearchCutoffMs) @@ -95,7 +87,7 @@ describe.each([{ permission: 'Master' }, { permission: 'Admin' }])( const response = await client.index(index.uid).getSearchCutoffMs() - expect(response).toEqual({ searchCutoffMs: DEFAULT_SEARCHCUTOFFMS }) + expect(response).toEqual(DEFAULT_SEARCHCUTOFFMS) }) } ) @@ -119,7 +111,7 @@ describe.each([{ permission: 'Search' }])( test(`${permission} key: try to update searchCutoffMs and be denied`, async () => { const client = await getClient(permission) await expect( - client.index(index.uid).updateSearchCutoffMs({ searchCutoffMs: 100 }) + client.index(index.uid).updateSearchCutoffMs(100) ).rejects.toHaveProperty('code', ErrorStatusCode.INVALID_API_KEY) }) @@ -154,7 +146,7 @@ describe.each([{ permission: 'No' }])( test(`${permission} key: try to update searchCutoffMs and be denied`, async () => { const client = await getClient(permission) await expect( - client.index(index.uid).updateSearchCutoffMs({ searchCutoffMs: 100 }) + client.index(index.uid).updateSearchCutoffMs(100) ).rejects.toHaveProperty( 'code', ErrorStatusCode.MISSING_AUTHORIZATION_HEADER @@ -198,7 +190,7 @@ describe.each([ const client = new MeiliSearch({ host }) const strippedHost = trailing ? host.slice(0, -1) : host await expect( - client.index(index.uid).updateSearchCutoffMs({ searchCutoffMs: null }) + client.index(index.uid).updateSearchCutoffMs(null) ).rejects.toHaveProperty( 'message', `request to ${strippedHost}/${route} failed, reason: connect ECONNREFUSED ${BAD_HOST.replace( diff --git a/tests/settings.test.ts b/tests/settings.test.ts index 4316c84ae..2f3c73631 100644 --- a/tests/settings.test.ts +++ b/tests/settings.test.ts @@ -90,6 +90,7 @@ describe.each([{ permission: 'Master' }, { permission: 'Admin' }])( separatorTokens: ['&sep', '/', '|'], nonSeparatorTokens: ['&sep', '/', '|'], dictionary: ['J. K.', 'J. R. R.'], + searchCutoffMs: 1000, } // Add the settings const task = await client.index(index.uid).updateSettings(newSettings) @@ -104,7 +105,7 @@ describe.each([{ permission: 'Master' }, { permission: 'Admin' }])( test(`${permission} key: Update settings with all null values`, async () => { const client = await getClient(permission) - const newSettings = { + const newSettings: Settings = { filterableAttributes: null, sortableAttributes: null, distinctAttribute: null, @@ -132,6 +133,7 @@ describe.each([{ permission: 'Master' }, { permission: 'Admin' }])( separatorTokens: null, nonSeparatorTokens: null, dictionary: null, + searchCutoffMs: null, } // Add the settings const task = await client.index(index.uid).updateSettings(newSettings) From 4e5e70b96cb1780c62883c93cd1eadb51be8cdbc Mon Sep 17 00:00:00 2001 From: Morgane Dubus <30866152+mdubus@users.noreply.github.com> Date: Tue, 23 Apr 2024 19:26:23 +0200 Subject: [PATCH 5/8] feat: hybrid search improvements for v1.8.x (#1647) --- src/types/types.ts | 34 +++- tests/__snapshots__/settings.test.ts.snap | 214 ++++++++++++++++++++++ tests/embedders.test.ts | 87 ++++++++- tests/get_search.test.ts | 14 +- tests/search.test.ts | 14 +- tests/settings.test.ts | 60 ++++++ 6 files changed, 419 insertions(+), 4 deletions(-) diff --git a/src/types/types.ts b/src/types/types.ts index fde3b69b9..46b59835a 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -215,7 +215,6 @@ export type SearchResponse< query: string facetDistribution?: FacetDistribution facetStats?: FacetStats - vector?: number[] } & (undefined extends S ? Partial : true extends IsFinitePagination> @@ -335,12 +334,18 @@ export type NonSeparatorTokens = string[] | null export type Dictionary = string[] | null export type ProximityPrecision = 'byWord' | 'byAttribute' +export type Distribution = { + mean: number + sigma: number +} + export type OpenAiEmbedder = { source: 'openAi' model?: string apiKey?: string documentTemplate?: string dimensions?: number + distribution?: Distribution } export type HuggingFaceEmbedder = { @@ -348,17 +353,44 @@ export type HuggingFaceEmbedder = { model?: string revision?: string documentTemplate?: string + distribution?: Distribution } export type UserProvidedEmbedder = { source: 'userProvided' dimensions: number + distribution?: Distribution +} + +export type RestEmbedder = { + source: 'rest' + url: string + apiKey?: string + dimensions?: number + documentTemplate?: string + inputField?: string[] | null + inputType?: 'text' | 'textArray' + query?: Record | null + pathToEmbeddings?: string[] | null + embeddingObject?: string[] | null + distribution?: Distribution +} + +export type OllamaEmbedder = { + source: 'ollama' + url?: string + apiKey?: string + model?: string + documentTemplate?: string + distribution?: Distribution } export type Embedder = | OpenAiEmbedder | HuggingFaceEmbedder | UserProvidedEmbedder + | RestEmbedder + | OllamaEmbedder | null export type Embedders = Record | null diff --git a/tests/__snapshots__/settings.test.ts.snap b/tests/__snapshots__/settings.test.ts.snap index 5252c6889..9589e4bb1 100644 --- a/tests/__snapshots__/settings.test.ts.snap +++ b/tests/__snapshots__/settings.test.ts.snap @@ -94,6 +94,53 @@ exports[`Test on settings Admin key: Get default settings of empty index with pr } `; +exports[`Test on settings Admin key: Reset embedders settings 1`] = ` +{ + "dictionary": [], + "displayedAttributes": [ + "*", + ], + "distinctAttribute": null, + "faceting": { + "maxValuesPerFacet": 100, + "sortFacetValuesBy": { + "*": "alpha", + }, + }, + "filterableAttributes": [], + "nonSeparatorTokens": [], + "pagination": { + "maxTotalHits": 1000, + }, + "proximityPrecision": "byWord", + "rankingRules": [ + "words", + "typo", + "proximity", + "attribute", + "sort", + "exactness", + ], + "searchCutoffMs": null, + "searchableAttributes": [ + "*", + ], + "separatorTokens": [], + "sortableAttributes": [], + "stopWords": [], + "synonyms": {}, + "typoTolerance": { + "disableOnAttributes": [], + "disableOnWords": [], + "enabled": true, + "minWordSizeForTypos": { + "oneTypo": 5, + "twoTypos": 9, + }, + }, +} +`; + exports[`Test on settings Admin key: Reset settings 1`] = ` { "dictionary": [], @@ -188,6 +235,66 @@ exports[`Test on settings Admin key: Reset settings of empty index 1`] = ` } `; +exports[`Test on settings Admin key: Update embedders settings 1`] = ` +{ + "dictionary": [], + "displayedAttributes": [ + "*", + ], + "distinctAttribute": null, + "embedders": { + "default": { + "apiKey": " { @@ -97,6 +110,10 @@ describe.each([{ permission: 'Master' }, { permission: 'Admin' }])( model: 'sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2', documentTemplate: "A movie titled '{{doc.title}}' whose description starts with {{doc.overview|truncatewords: 20}}", + distribution: { + mean: 0.7, + sigma: 0.3, + }, }, } const task: EnqueuedTask = await client @@ -109,6 +126,74 @@ describe.each([{ permission: 'Master' }, { permission: 'Admin' }])( expect(response).toEqual(newEmbedder) }) + test(`${permission} key: Update embedders with 'rest' source`, async () => { + const client = await getClient(permission) + const newEmbedder: Embedders = { + default: { + source: 'rest', + url: 'https://api.openai.com/v1/embeddings', + apiKey: '', + dimensions: 1536, + documentTemplate: + "A movie titled '{{doc.title}}' whose description starts with {{doc.overview|truncatewords: 20}}", + inputField: ['input'], + inputType: 'textArray', + query: { + model: 'text-embedding-ada-002', + }, + pathToEmbeddings: ['data'], + embeddingObject: ['embedding'], + distribution: { + mean: 0.7, + sigma: 0.3, + }, + }, + } + const task: EnqueuedTask = await client + .index(index.uid) + .updateEmbedders(newEmbedder) + await client.waitForTask(task.taskUid) + + const response: Embedders = await client.index(index.uid).getEmbedders() + + expect(response).toEqual({ + default: { + ...newEmbedder.default, + apiKey: ' { + const client = await getClient(permission) + const newEmbedder: Embedders = { + default: { + source: 'ollama', + url: 'http://localhost:11434/api/embeddings', + apiKey: '', + model: 'nomic-embed-text', + documentTemplate: 'blabla', + distribution: { + mean: 0.7, + sigma: 0.3, + }, + }, + } + const task: EnqueuedTask = await client + .index(index.uid) + .updateEmbedders(newEmbedder) + await client.waitForTask(task.taskUid) + + const response: Embedders = await client.index(index.uid).getEmbedders() + + expect(response).toEqual({ + default: { + ...newEmbedder.default, + apiKey: ' { const client = await getClient(permission) diff --git a/tests/get_search.test.ts b/tests/get_search.test.ts index 690caec90..d8f53ac37 100644 --- a/tests/get_search.test.ts +++ b/tests/get_search.test.ts @@ -473,7 +473,19 @@ describe.each([ .index(emptyIndex.uid) .searchGet('', { vector: [1], hybridSemanticRatio: 1.0 }) - expect(response.vector).toEqual([1]) + expect(response).toHaveProperty('hits') + expect(response).toHaveProperty('semanticHitCount') + // Those fields are no longer returned by the search response + // We want to ensure that they don't appear in it anymore + expect(response).not.toHaveProperty('vector') + expect(response).not.toHaveProperty('_semanticScore') + }) + + test(`${permission} key: search without vectors`, async () => { + const client = await getClient(permission) + const response = await client.index(index.uid).search('prince', {}) + + expect(response).not.toHaveProperty('semanticHitCount') }) test(`${permission} key: Try to search on deleted index and fail`, async () => { diff --git a/tests/search.test.ts b/tests/search.test.ts index b5053156e..27aea8b27 100644 --- a/tests/search.test.ts +++ b/tests/search.test.ts @@ -859,7 +859,19 @@ describe.each([ }, }) - expect(response.vector).toEqual([1]) + expect(response).toHaveProperty('hits') + expect(response).toHaveProperty('semanticHitCount') + // Those fields are no longer returned by the search response + // We want to ensure that they don't appear in it anymore + expect(response).not.toHaveProperty('vector') + expect(response).not.toHaveProperty('_semanticScore') + }) + + test(`${permission} key: search without vectors`, async () => { + const client = await getClient(permission) + const response = await client.index(index.uid).search('prince', {}) + + expect(response).not.toHaveProperty('semanticHitCount') }) test(`${permission} key: Try to search on deleted index and fail`, async () => { diff --git a/tests/settings.test.ts b/tests/settings.test.ts index 2f3c73631..d19f2e131 100644 --- a/tests/settings.test.ts +++ b/tests/settings.test.ts @@ -6,6 +6,8 @@ import { MeiliSearch, getClient, dataset, + getKey, + HOST, } from './utils/meilisearch-test-utils' const index = { @@ -146,6 +148,41 @@ describe.each([{ permission: 'Master' }, { permission: 'Admin' }])( expect(response).toMatchSnapshot() }) + test(`${permission} key: Update embedders settings `, async () => { + const client = await getClient(permission) + const key = await getKey(permission) + + await fetch(`${HOST}/experimental-features`, { + body: JSON.stringify({ vectorStore: true }), + headers: { + Authorization: `Bearer ${key}`, + 'Content-Type': 'application/json', + }, + method: 'PATCH', + }) + + const newSettings: Settings = { + embedders: { + default: { + source: 'openAi', + apiKey: '', + model: 'text-embedding-3-small', + documentTemplate: 'A document template', + dimensions: 1536, + distribution: { + mean: 0.7, + sigma: 0.3, + }, + }, + }, + } + const task = await client.index(index.uid).updateSettings(newSettings) + await client.index(index.uid).waitForTask(task.taskUid) + const response = await client.index(index.uid).getSettings() + + expect(response).toMatchSnapshot() + }) + test(`${permission} key: Update settings on empty index with primary key`, async () => { const client = await getClient(permission) const newSettings = { @@ -183,6 +220,29 @@ describe.each([{ permission: 'Master' }, { permission: 'Admin' }])( expect(response).toMatchSnapshot() }) + test(`${permission} key: Reset embedders settings `, async () => { + const client = await getClient(permission) + const key = await getKey(permission) + + await fetch(`${HOST}/experimental-features`, { + body: JSON.stringify({ vectorStore: true }), + headers: { + Authorization: `Bearer ${key}`, + 'Content-Type': 'application/json', + }, + method: 'PATCH', + }) + + const newSettings: Settings = { + embedders: null, + } + const task = await client.index(index.uid).updateSettings(newSettings) + await client.index(index.uid).waitForTask(task.taskUid) + const response = await client.index(index.uid).getSettings() + + expect(response).toMatchSnapshot() + }) + test(`${permission} key: Update searchableAttributes settings on empty index`, async () => { const client = await getClient(permission) const newSettings = { From c06e4b4fa1f6b46c052c9ca6f1163e755cb19875 Mon Sep 17 00:00:00 2001 From: curquiza Date: Tue, 30 Apr 2024 12:04:13 +0200 Subject: [PATCH 6/8] Add code samples --- .code-samples.meilisearch.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.code-samples.meilisearch.yaml b/.code-samples.meilisearch.yaml index 529d7dfb3..654eac4db 100644 --- a/.code-samples.meilisearch.yaml +++ b/.code-samples.meilisearch.yaml @@ -199,7 +199,8 @@ update_settings_1: |- }, faceting: { maxValuesPerFacet: 200 - } + }, + searchCutoffMs: 150 }) reset_settings_1: |- client.index('movies').resetSettings() @@ -636,6 +637,12 @@ update_proximity_precision_settings_1: |- client.index('books').updateProximityPrecision('byAttribute') reset_proximity_precision_settings_1: |- client.index('books').resetProximityPrecision() +get_search_cutoff_1: |- + client.index('movies').getSearchCutoffMs() +update_search_cutoff_1: |- + client.index('movies').updateSearchCutoffMs(150) +reset_search_cutoff_1: |- + client.index('movies').resetSearchCutoffMs() search_parameter_guide_facet_stats_1: |- client.index('movie_ratings').search('Batman', { facets: ['genres', 'rating'] }) geosearch_guide_filter_settings_1: |- @@ -743,3 +750,7 @@ facet_search_3: |- }) search_parameter_guide_show_ranking_score_details_1: |- client.index('movies').search('dragon', { showRankingScoreDetails: true }) +negative_search_1: |- + client.index('movies').search('-escape') +negative_search_2: |- + client.index('movies').search('-"escape"') From e345c652b86e3e2a8992d88384436fed7b0a5412 Mon Sep 17 00:00:00 2001 From: Morgane Dubus Date: Thu, 2 May 2024 17:07:46 +0200 Subject: [PATCH 7/8] Fix embedder test on settings route + associated snapshot --- tests/__snapshots__/settings.test.ts.snap | 26 +++++++---------------- tests/settings.test.ts | 12 +++-------- 2 files changed, 11 insertions(+), 27 deletions(-) diff --git a/tests/__snapshots__/settings.test.ts.snap b/tests/__snapshots__/settings.test.ts.snap index 9589e4bb1..0f359e2dd 100644 --- a/tests/__snapshots__/settings.test.ts.snap +++ b/tests/__snapshots__/settings.test.ts.snap @@ -244,15 +244,10 @@ exports[`Test on settings Admin key: Update embedders settings 1`] = ` "distinctAttribute": null, "embedders": { "default": { - "apiKey": "', - model: 'text-embedding-3-small', - documentTemplate: 'A document template', - dimensions: 1536, - distribution: { - mean: 0.7, - sigma: 0.3, - }, + source: 'huggingFace', + model: + 'sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2', }, }, } From d4a7648700d9795824741635e002886c4390cf90 Mon Sep 17 00:00:00 2001 From: Morgane Dubus Date: Thu, 2 May 2024 18:00:44 +0200 Subject: [PATCH 8/8] Skip ollama embedder test --- tests/embedders.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/embedders.test.ts b/tests/embedders.test.ts index 3346b8bc2..9573b64cc 100644 --- a/tests/embedders.test.ts +++ b/tests/embedders.test.ts @@ -164,7 +164,7 @@ describe.each([{ permission: 'Master' }, { permission: 'Admin' }])( }) }) - test(`${permission} key: Update embedders with 'ollama' source`, async () => { + test.skip(`${permission} key: Update embedders with 'ollama' source`, async () => { const client = await getClient(permission) const newEmbedder: Embedders = { default: {