Skip to content

Commit d87cf1d

Browse files
hkt74hkt74
authoredFeb 20, 2025··
Fix flaky integration tests (#343)
* Make function calling tests to produce more deterministic results by switching model to 1.5 pro * Reduce cache token size and switching model to 1.5 flash-002
·
v0.24.0v0.23.0
1 parent e193f86 commit d87cf1d

File tree

5 files changed

+108
-119
lines changed

5 files changed

+108
-119
lines changed
 

‎.changeset/lucky-grapes-rest.md

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@google/generative-ai": patch
3+
---
4+
5+
Fix flaky integration test with tools

‎test-integration/node/cache-content.test.ts

+2-7
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,9 @@ use(chaiAsPromised);
2929
describe("cacheContent", function () {
3030
this.timeout(60e3);
3131
this.slow(10e3);
32-
const model = "models/gemini-1.5-pro-001";
33-
let text: string = "";
32+
const model = "gemini-1.5-flash-002";
33+
const text = "Purple cats drink chicken soup.";
3434

35-
// Minimum cache size is 32768 tokens.
36-
for (let i = 0; i < 6554; i++) {
37-
text += "Purple cats drink chicken soup.";
38-
text += i % 8 === 7 ? "\n" : " ";
39-
}
4035
it("createCache", async () => {
4136
// cacheManager create
4237
const ttlSeconds = 5;

‎test-integration/node/generate-content-tools.test.ts

+85-95
Original file line numberDiff line numberDiff line change
@@ -33,87 +33,81 @@ describe("generateContent - tools", function () {
3333
// eslint-disable-next-line no-restricted-properties
3434
it("non-streaming, tools usage", async () => {
3535
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || "");
36-
const model = genAI.getGenerativeModel(
37-
{
38-
model: "gemini-1.5-flash-latest",
39-
tools: [
40-
{
41-
functionDeclarations: [
42-
{
43-
name: "find_movies",
44-
description:
45-
"find movie titles currently playing in theaters based on any description, genre, title words, etc.",
46-
parameters: {
47-
type: SchemaType.OBJECT,
48-
properties: {
49-
location: {
50-
type: SchemaType.STRING,
51-
description:
52-
"The city and state, e.g. San Francisco, CA or a zip code e.g. 95616",
53-
},
54-
description: {
55-
type: SchemaType.STRING,
56-
description:
57-
"Any kind of description including category or genre, title words, attributes, etc.",
58-
},
36+
const model = genAI.getGenerativeModel({
37+
model: "gemini-1.5-pro-latest",
38+
tools: [
39+
{
40+
functionDeclarations: [
41+
{
42+
name: "find_movies",
43+
description:
44+
"find movie titles currently playing in theaters based on any description, genre, title words, etc.",
45+
parameters: {
46+
type: SchemaType.OBJECT,
47+
properties: {
48+
location: {
49+
type: SchemaType.STRING,
50+
description: "The city and state, e.g. San Francisco, CA",
51+
},
52+
description: {
53+
type: SchemaType.STRING,
54+
description:
55+
"Any kind of description including category or genre, title words, attributes, etc.",
5956
},
60-
required: ["description"],
6157
},
58+
required: ["description"],
6259
},
63-
{
64-
name: "find_theaters",
65-
description:
66-
"find theaters based on location and optionally movie title which are is currently playing in theaters",
67-
parameters: {
68-
type: SchemaType.OBJECT,
69-
properties: {
70-
location: {
71-
type: SchemaType.STRING,
72-
description:
73-
"The city and state, e.g. San Francisco, CA or a zip code e.g. 95616",
74-
},
75-
movie: {
76-
type: SchemaType.STRING,
77-
description: "Any movie title",
78-
},
60+
},
61+
{
62+
name: "find_theaters",
63+
description:
64+
"find theaters based on location and optionally movie title which are is currently playing in theaters",
65+
parameters: {
66+
type: SchemaType.OBJECT,
67+
properties: {
68+
location: {
69+
type: SchemaType.STRING,
70+
description: "The city and state, e.g. San Francisco, CA",
71+
},
72+
movie: {
73+
type: SchemaType.STRING,
74+
description: "Any movie title",
7975
},
80-
required: ["location"],
8176
},
77+
required: ["location"],
8278
},
83-
{
84-
name: "get_showtimes",
85-
description:
86-
"Find the start times for movies playing in a specific theater",
87-
parameters: {
88-
type: SchemaType.OBJECT,
89-
properties: {
90-
location: {
91-
type: SchemaType.STRING,
92-
description:
93-
"The city and state, e.g. San Francisco, CA or a zip code e.g. 95616",
94-
},
95-
movie: {
96-
type: SchemaType.STRING,
97-
description: "Any movie title",
98-
},
99-
theater: {
100-
type: SchemaType.STRING,
101-
description: "Name of the theater",
102-
},
103-
date: {
104-
type: SchemaType.STRING,
105-
description: "Date for requested showtime",
106-
},
79+
},
80+
{
81+
name: "get_showtimes",
82+
description:
83+
"Find the start times for movies playing in a specific theater",
84+
parameters: {
85+
type: SchemaType.OBJECT,
86+
properties: {
87+
location: {
88+
type: SchemaType.STRING,
89+
description: "The city and state, e.g. San Francisco, CA",
90+
},
91+
movie: {
92+
type: SchemaType.STRING,
93+
description: "Any movie title",
94+
},
95+
theater: {
96+
type: SchemaType.STRING,
97+
description: "Name of the theater",
98+
},
99+
date: {
100+
type: SchemaType.STRING,
101+
description: "Date for requested showtime",
107102
},
108-
required: ["location", "movie", "theater", "date"],
109103
},
104+
required: ["location", "movie", "theater", "date"],
110105
},
111-
],
112-
},
113-
],
114-
},
115-
{ apiVersion: "v1beta" },
116-
);
106+
},
107+
],
108+
},
109+
],
110+
});
117111

118112
const src1 = {
119113
role: "user",
@@ -183,30 +177,27 @@ describe("generateContent - tools", function () {
183177
});
184178
it("streaming, tools usage", async () => {
185179
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || "");
186-
const model = genAI.getGenerativeModel(
187-
{
188-
model: "gemini-1.5-flash-latest",
189-
tools: [
190-
{
191-
functionDeclarations: [
192-
{
193-
name: "getTemperature",
194-
description:
195-
"Get current temperature in degrees Celsius in a given city",
196-
parameters: {
197-
type: SchemaType.OBJECT,
198-
properties: {
199-
city: { type: SchemaType.STRING },
200-
},
201-
required: ["city"],
180+
const model = genAI.getGenerativeModel({
181+
model: "gemini-1.5-pro-latest",
182+
tools: [
183+
{
184+
functionDeclarations: [
185+
{
186+
name: "getTemperature",
187+
description:
188+
"Get current temperature in degrees Celsius in a given city",
189+
parameters: {
190+
type: SchemaType.OBJECT,
191+
properties: {
192+
city: { type: SchemaType.STRING },
202193
},
194+
required: ["city"],
203195
},
204-
],
205-
},
206-
],
207-
},
208-
{ apiVersion: "v1beta" },
209-
);
196+
},
197+
],
198+
},
199+
],
200+
});
210201

211202
const src1: Content = {
212203
role: "user",
@@ -259,7 +250,6 @@ describe("generateContent - tools", function () {
259250
contents: [src1, src2, fn1, src3, fn1],
260251
});
261252
const response = await result.response;
262-
console.log(response.text());
263253
expect(response.text()).to.match(/(\bsame\b|\byes\b)/i);
264254
});
265255
});

‎test-integration/node/start-chat-tools.test.ts

+15-16
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,10 @@ describe("startChat - tools", function () {
4343
parameters: {
4444
type: SchemaType.OBJECT,
4545
properties: {
46-
city: { type: SchemaType.STRING },
46+
city: {
47+
type: SchemaType.STRING,
48+
description: "A city name, for example, San Francisco",
49+
},
4750
},
4851
required: ["city"],
4952
},
@@ -71,23 +74,19 @@ describe("startChat - tools", function () {
7174
this.slow(10e3);
7275
it("stream false", async () => {
7376
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || "");
74-
const model = genAI.getGenerativeModel(
75-
{
76-
model: "gemini-1.5-flash-latest",
77-
safetySettings: [
78-
{
79-
category: HarmCategory.HARM_CATEGORY_HARASSMENT,
80-
threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH,
81-
},
82-
],
83-
tools,
84-
},
85-
{ apiVersion: "v1beta" },
86-
);
77+
const model = genAI.getGenerativeModel({
78+
model: "gemini-1.5-pro-latest",
79+
safetySettings: [
80+
{
81+
category: HarmCategory.HARM_CATEGORY_HARASSMENT,
82+
threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH,
83+
},
84+
],
85+
tools,
86+
});
8787
const chat = model.startChat();
8888
const result1 = await chat.sendMessage([part1]);
89-
expect(result1.response.text()).to.be.empty;
90-
expect(result1.response.functionCall()).not.to.be.empty;
89+
expect(result1.response.functionCalls()).not.to.be.empty;
9190
const result2 = await chat.sendMessage([part2]);
9291
expect(result2.response.text()).to.not.be.empty;
9392
const history = await chat.getHistory();

‎test-integration/node/start-chat.test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ describe("startChat", function () {
9090
it("stream true, try to send message before previous stream is done", async () => {
9191
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY || "");
9292
const model = genAI.getGenerativeModel({
93-
model: "gemini-1.5-flash-latest",
93+
model: "gemini-1.5-pro-latest",
9494
safetySettings: [
9595
{
9696
category: HarmCategory.HARM_CATEGORY_HARASSMENT,

0 commit comments

Comments
 (0)
Please sign in to comment.