Merge pull request #99 from transitive-bullshit/fix/cloudflare-protections

.env.example

@@ -9,5 +9,5 @@
 # ChatGPT
 # -----------------------------------------------------------------------------
 
-# see the readme for how to find this
-SESSION_TOKEN=
+EMAIL=
+PASSWORD=

src/demo-conversation.ts → demos/demo-conversation.ts

 import dotenv from 'dotenv-safe'
 import { oraPromise } from 'ora'
 
-import { ChatGPTAPI } from '.'
+import { ChatGPTAPI } from '../src'
+import { getOpenAIAuthInfo } from './openai-auth-puppeteer'
 
 dotenv.config()
 
@@ -13,7 +14,15 @@ dotenv.config()
  * ```
  */
 async function main() {
-  const api = new ChatGPTAPI({ sessionToken: process.env.SESSION_TOKEN })
+  const email = process.env.EMAIL
+  const password = process.env.PASSWORD
+
+  const authInfo = await getOpenAIAuthInfo({
+    email,
+    password
+  })
+
+  const api = new ChatGPTAPI({ ...authInfo })
   await api.ensureAuth()
 
   const conversation = api.getConversation()

src/demo.ts → demos/demo.ts

 import dotenv from 'dotenv-safe'
 import { oraPromise } from 'ora'
 
-import { ChatGPTAPI } from '.'
+import { ChatGPTAPI } from '../src'
+import { getOpenAIAuthInfo } from './openai-auth-puppeteer'
 
 dotenv.config()
 
@@ -13,7 +14,15 @@ dotenv.config()
  * ```
  */
 async function main() {
-  const api = new ChatGPTAPI({ sessionToken: process.env.SESSION_TOKEN })
+  const email = process.env.EMAIL
+  const password = process.env.PASSWORD
+
+  const authInfo = await getOpenAIAuthInfo({
+    email,
+    password
+  })
+
+  const api = new ChatGPTAPI({ ...authInfo })
   await api.ensureAuth()
 
   const prompt =

demos/openai-auth-puppeteer.ts

+import delay from 'delay'
+import {
+  type Browser,
+  type Page,
+  type Protocol,
+  type PuppeteerLaunchOptions
+} from 'puppeteer'
+import puppeteer from 'puppeteer-extra'
+import StealthPlugin from 'puppeteer-extra-plugin-stealth'
+
+puppeteer.use(StealthPlugin())
+
+export type OpenAIAuthInfo = {
+  userAgent: string
+  clearanceToken: string
+  sessionToken: string
+  cookies?: Record<string, Protocol.Network.Cookie>
+}
+
+/**
+ * Bypasses OpenAI's use of Cloudflare to get the cookies required to use
+ * ChatGPT. Uses Puppeteer with a stealth plugin under the hood.
+ */
+export async function getOpenAIAuthInfo({
+  email,
+  password,
+  timeout = 2 * 60 * 1000,
+  browser
+}: {
+  email: string
+  password: string
+  timeout?: number
+  browser?: Browser
+}): Promise<OpenAIAuthInfo> {
+  let page: Page
+  let origBrowser = browser
+
+  try {
+    if (!browser) {
+      browser = await getBrowser()
+    }
+
+    const userAgent = await browser.userAgent()
+    page = (await browser.pages())[0] || (await browser.newPage())
+    page.setDefaultTimeout(timeout)
+
+    await page.goto('https://chat.openai.com/auth/login')
+    await page.waitForSelector('#__next .btn-primary', { timeout })
+    await delay(1000)
+
+    if (email && password) {
+      await Promise.all([
+        page.click('#__next .btn-primary'),
+        page.waitForNavigation({
+          waitUntil: 'networkidle0'
+        })
+      ])
+      await page.type('#username', email, { delay: 10 })
+      await page.click('button[type="submit"]')
+      await page.waitForSelector('#password')
+      await page.type('#password', password, { delay: 10 })
+      await Promise.all([
+        page.click('button[type="submit"]'),
+        page.waitForNavigation({
+          waitUntil: 'networkidle0'
+        })
+      ])
+    }
+
+    const pageCookies = await page.cookies()
+    const cookies = pageCookies.reduce(
+      (map, cookie) => ({ ...map, [cookie.name]: cookie }),
+      {}
+    )
+
+    const authInfo: OpenAIAuthInfo = {
+      userAgent,
+      clearanceToken: cookies['cf_clearance']?.value,
+      sessionToken: cookies['__Secure-next-auth.session-token']?.value,
+      cookies
+    }
+
+    return authInfo
+  } catch (err) {
+    console.error(err)
+    throw null
+  } finally {
+    if (origBrowser) {
+      if (page) {
+        await page.close()
+      }
+    } else if (browser) {
+      await browser.close()
+    }
+
+    page = null
+    browser = null
+  }
+}
+
+export async function getBrowser(launchOptions?: PuppeteerLaunchOptions) {
+  const macChromePath =
+    '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'
+
+  return puppeteer.launch({
+    headless: false,
+    args: ['--no-sandbox', '--exclude-switches', 'enable-automation'],
+    ignoreHTTPSErrors: true,
+    // executablePath: executablePath()
+    executablePath: macChromePath,
+    ...launchOptions
+  })
+}

package.json

@@ -34,7 +34,6 @@
     "prepare": "husky install",
     "pre-commit": "lint-staged",
     "test": "run-p test:*",
-    "test:unit": "ava",
     "test:prettier": "prettier '**/*.{js,jsx,ts,tsx}' --check"
   },
   "dependencies": {
@@ -51,12 +50,16 @@
     "@types/uuid": "^9.0.0",
     "ava": "^5.1.0",
     "del-cli": "^5.0.0",
+    "delay": "^5.0.0",
     "dotenv-safe": "^8.2.0",
     "husky": "^8.0.2",
     "lint-staged": "^13.0.3",
     "npm-run-all": "^4.1.5",
     "ora": "^6.1.2",
     "prettier": "^2.8.0",
+    "puppeteer": "^19.4.0",
+    "puppeteer-extra": "^3.3.4",
+    "puppeteer-extra-plugin-stealth": "^2.11.1",
     "tsup": "^6.5.0",
     "tsx": "^3.12.1",
     "typedoc": "^0.23.21",

readme.md

-# Update December 11, 2022
+# Update December 12, 2022
 
-Today, OpenAI added additional Cloudflare protections that make it more difficult to access the unofficial API. _This is affecting all ChatGPT API wrappers at the moment_, including the Python ones. See [this issue](https://github.com/transitive-bullshit/chatgpt-api/issues/96).
+Yesterday, OpenAI added additional Cloudflare protections that make it more difficult to access the unofficial API.
 
-**As a temporary workaround**, make sure you're using the latest version of this package and Node.js >= 18:
+The demos have been updated to use Puppeteer to log in to ChatGPT and extract the Cloudflare `cf_clearance` cookie and OpenAI session token. 🔥
 
-1. Log into https://chat.openai.com/chat and copy a fresh session token (same instructions as below).
-
-2. Copy the value of the `cf_clearance` cookie and store it in a `CLEARANCE_TOKEN` environment variable in addition to your `SESSION_TOKEN`.
-
-3. Copy your browser's `user-agent` header from any request in your browser's network tab.
-
-4. Use both tokens when creating the API wrapper:
+To use the updated version, first make sure you're using the latest version of this package and Node.js >= 18:
 
 ```ts
 const api = new ChatGPTAPI({
   sessionToken: process.env.SESSION_TOKEN,
   clearanceToken: process.env.CLEARANCE_TOKEN,
-  userAgent: '' // replace to match your real browser's user agent
+  userAgent: '' // needs to match your browser's user agent
 })
 
 await api.ensureAuth()
@@ -24,9 +18,10 @@ await api.ensureAuth()
 
 Restrictions on this method:
 
-- Cloudflare `cf_clearance` **tokens expire after 2 hours**, so right now you'll have to manually log in and extract it by hand every so often
+- Cloudflare `cf_clearance` **tokens expire after 2 hours**, so right now we recommend that you refresh your `cf_clearance` token every ~45 minutes or so.
 - Your `user-agent` and `IP address` **must match** from the real browser window you're logged in with to the one you're using for `ChatGPTAPI`.
   - This means that you currently can't log in with your laptop and then run the bot on a server or proxy somewhere.
+- Cloudflare will still sometimes ask you to complete a CAPTCHA, so you may need to keep an eye on it and manually resolve the CAPTCHA. Automated CAPTCHA bypass is a WIP.
 - You must use `node >= 18`. I'm using `v19.2.0` in my testing, but for some reason, all `fetch` requests using Node.js `v16` and `v17` fail at the moment (these use `undici` under the hood, whereas Node.js v18 and above use a built-in `fetch` based on `undici`).
 - You should not be using this account while the bot is using it, because that browser window may refresh one of your tokens and invalidate the bot's session.
 
@@ -47,7 +42,7 @@ Travis
 
 [![NPM](https://img.shields.io/npm/v/chatgpt.svg)](https://www.npmjs.com/package/chatgpt) [![Build Status](https://github.com/transitive-bullshit/chatgpt-api/actions/workflows/test.yml/badge.svg)](https://github.com/transitive-bullshit/chatgpt-api/actions/workflows/test.yml) [![MIT License](https://img.shields.io/badge/license-MIT-blue)](https://github.com/transitive-bullshit/chatgpt-api/blob/main/license) [![Prettier Code Formatting](https://img.shields.io/badge/code_style-prettier-brightgreen.svg)](https://prettier.io)
 
-- [Update December 11, 2022](#update-december-11-2022)
+- [Update December 12, 2022](#update-december-12-2022)
   - [Intro](#intro)
   - [Install](#install)
   - [Usage](#usage)
@@ -77,12 +72,12 @@ npm install chatgpt
 import { ChatGPTAPI } from 'chatgpt'
 
 async function example() {
-  // sessionToken is required; see below for details
   const api = new ChatGPTAPI({
-    sessionToken: process.env.SESSION_TOKEN
+    sessionToken: process.env.SESSION_TOKEN,
+    clearanceToken: process.env.CLEARANCE_TOKEN,
+    userAgent: 'TODO'
   })
 
-  // ensure the API is properly authenticated
   await api.ensureAuth()
 
   // send a message and wait for the response
@@ -100,6 +95,8 @@ ChatGPT responses are formatted as markdown by default. If you want to work with
 ```ts
 const api = new ChatGPTAPI({
   sessionToken: process.env.SESSION_TOKEN,
+  clearanceToken: process.env.CLEARANCE_TOKEN,
+  userAgent: 'TODO',
   markdown: false
 })
 ```
@@ -108,7 +105,9 @@ If you want to automatically track the conversation, you can use `ChatGPTAPI.get
 
 ```ts
 const api = new ChatGPTAPI({
-  sessionToken: process.env.SESSION_TOKEN
+  sessionToken: process.env.SESSION_TOKEN,
+  clearanceToken: process.env.CLEARANCE_TOKEN,
+  userAgent: 'TODO'
 })
 
 const conversation = api.getConversation()
@@ -145,7 +144,9 @@ async function example() {
   const { ChatGPTAPI } = await import('chatgpt')
 
   const api = new ChatGPTAPI({
-    sessionToken: process.env.SESSION_TOKEN
+    sessionToken: process.env.SESSION_TOKEN,
+    clearanceToken: process.env.CLEARANCE_TOKEN,
+    userAgent: 'TODO'
   })
   await api.ensureAuth()
 
@@ -162,23 +163,21 @@ See the [auto-generated docs](./docs/classes/ChatGPTAPI.md) for more info on met
 
 ### Demos
 
-A [basic demo](./src/demo.ts) is included for testing purposes:
+To run the included demos:
+
+1. clone repo
+2. install node deps
+3. set `EMAIL` and `PASSWORD` in .env
+
+A [basic demo](./demos/demo.ts) is included for testing purposes:
 
 ```bash
-# 1. clone repo
-# 2. install node deps
-# 3. set `SESSION_TOKEN` in .env
-# 4. run:
 npx tsx src/demo.ts
 ```
 
-A [conversation demo](./src/demo-conversation.ts) is also included:
+A [conversation demo](./demos/demo-conversation.ts) is also included:
 
 ```bash
-# 1. clone repo
-# 2. install node deps
-# 3. set `SESSION_TOKEN` in .env
-# 4. run:
 npx tsx src/demo-conversation.ts
 ```
 
@@ -186,15 +185,18 @@ npx tsx src/demo-conversation.ts
 
 **This package requires a valid session token from ChatGPT to access it's unofficial REST API.**
 
-To get a session token:
+As of December 11, 2021, it also requires a valid Cloudflare clearance token.
+
+There are two options to get these; either manually, or automated. For the automated way, see the `demos/` folder using Puppeteer.
+
+To get a session token manually:
 
 1. Go to https://chat.openai.com/chat and log in or sign up.
 2. Open dev tools.
 3. Open `Application` > `Cookies`.
    ![ChatGPT cookies](./media/session-token.png)
 4. Copy the value for `__Secure-next-auth.session-token` and save it to your environment.
-
-If you want to run the built-in demo, store this value as `SESSION_TOKEN` in a local `.env` file.
+5. Copy the value for `cf_clearance` and save it to your environment.
 
 > **Note**
 > This package will switch to using the official API once it's released.

src/chatgpt-api.test.ts

@@ -11,13 +11,16 @@ const isCI = !!process.env.CI
 test('ChatGPTAPI invalid session token', async (t) => {
   t.timeout(30 * 1000) // 30 seconds
 
-  t.throws(() => new ChatGPTAPI({ sessionToken: null }), {
+  t.throws(() => new ChatGPTAPI({ sessionToken: null, clearanceToken: null }), {
     message: 'ChatGPT invalid session token'
   })
 
   await t.throwsAsync(
     async () => {
-      const chatgpt = new ChatGPTAPI({ sessionToken: 'invalid' })
+      const chatgpt = new ChatGPTAPI({
+        sessionToken: 'invalid',
+        clearanceToken: 'invalid'
+      })
       await chatgpt.ensureAuth()
     },
     {
@@ -33,13 +36,18 @@ test('ChatGPTAPI valid session token', async (t) => {
   }
 
   t.notThrows(
-    () => new ChatGPTAPI({ sessionToken: 'fake valid session token' })
+    () =>
+      new ChatGPTAPI({
+        sessionToken: 'fake valid session token',
+        clearanceToken: 'invalid'
+      })
   )
 
   await t.notThrowsAsync(
     (async () => {
       const chatgpt = new ChatGPTAPI({
-        sessionToken: process.env.SESSION_TOKEN
+        sessionToken: process.env.SESSION_TOKEN,
+        clearanceToken: process.env.CLEARANCE_TOKEN
       })
 
       // Don't make any real API calls using our session token if we're running on CI
@@ -62,7 +70,10 @@ if (!isCI) {
 
     await t.throwsAsync(
       async () => {
-        const chatgpt = new ChatGPTAPI({ sessionToken: expiredSessionToken })
+        const chatgpt = new ChatGPTAPI({
+          sessionToken: expiredSessionToken,
+          clearanceToken: 'invalid'
+        })
         await chatgpt.ensureAuth()
       },
       {
@@ -81,7 +92,8 @@ if (!isCI) {
     await t.throwsAsync(
       async () => {
         const chatgpt = new ChatGPTAPI({
-          sessionToken: process.env.SESSION_TOKEN
+          sessionToken: process.env.SESSION_TOKEN,
+          clearanceToken: process.env.CLEARANCE_TOKEN
         })
 
         await chatgpt.sendMessage('test', {
@@ -100,7 +112,8 @@ if (!isCI) {
     await t.throwsAsync(
       async () => {
         const chatgpt = new ChatGPTAPI({
-          sessionToken: process.env.SESSION_TOKEN
+          sessionToken: process.env.SESSION_TOKEN,
+          clearanceToken: process.env.CLEARANCE_TOKEN
         })
 
         const abortController = new AbortController()

src/chatgpt-api.ts

@@ -10,7 +10,7 @@ import { markdownToText } from './utils'
 
 const KEY_ACCESS_TOKEN = 'accessToken'
 const USER_AGENT =
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36'
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36'
 
 export class ChatGPTAPI {
   protected _sessionToken: string
@@ -22,7 +22,6 @@ export class ChatGPTAPI {
   protected _headers: Record<string, string>
 
   // Stores access tokens for `accessTokenTTL` milliseconds before needing to refresh
-  // (defaults to 60 seconds)
   protected _accessTokenCache: ExpiryMap<string, string>
 
   protected _user: types.User | null = null
@@ -52,10 +51,12 @@ export class ChatGPTAPI {
     /** @defaultValue `'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36'` **/
     userAgent?: string
 
-    /** @defaultValue 60000 (60 seconds) */
+    /** @defaultValue 1 hour */
     accessTokenTTL?: number
 
     accessToken?: string
+
+    headers?: Record<string, string>
   }) {
     const {
       sessionToken,
@@ -64,8 +65,9 @@ export class ChatGPTAPI {
       apiBaseUrl = 'https://chat.openai.com/api',
       backendApiBaseUrl = 'https://chat.openai.com/backend-api',
       userAgent = USER_AGENT,
-      accessTokenTTL = 60000, // 60 seconds
-      accessToken
+      accessTokenTTL = 60 * 60000, // 1 hour
+      accessToken,
+      headers
     } = opts
 
     this._sessionToken = sessionToken
@@ -75,11 +77,18 @@ export class ChatGPTAPI {
     this._backendApiBaseUrl = backendApiBaseUrl
     this._userAgent = userAgent
     this._headers = {
-      'User-Agent': this._userAgent,
+      'user-agent': this._userAgent,
       'x-openai-assistant-app-id': '',
       'accept-language': 'en-US,en;q=0.9',
       origin: 'https://chat.openai.com',
-      referer: 'https://chat.openai.com/chat'
+      referer: 'https://chat.openai.com/chat',
+      'sec-ch-ua':
+        '"Not?A_Brand";v="8", "Chromium";v="108", "Google Chrome";v="108"',
+      'sec-ch-ua-platform': '"macOS"',
+      'sec-fetch-dest': 'empty',
+      'sec-fetch-mode': 'cors',
+      'sec-fetch-site': 'same-origin',
+      ...headers
     }
 
     this._accessTokenCache = new ExpiryMap<string, string>(accessTokenTTL)
@@ -90,6 +99,10 @@ export class ChatGPTAPI {
     if (!this._sessionToken) {
       throw new types.ChatGPTError('ChatGPT invalid session token')
     }
+
+    if (!this._clearanceToken) {
+      throw new types.ChatGPTError('ChatGPT invalid clearance token')
+    }
   }
 
   /**
@@ -269,11 +282,15 @@ export class ChatGPTAPI {
 
     let response: Response
     try {
+      const headers = {
+        ...this._headers,
+        cookie: `cf_clearance=${this._clearanceToken}; __Secure-next-auth.session-token=${this._sessionToken}`,
+        accept: '*/*'
+      }
+      console.log(`${this._apiBaseUrl}/auth/session`, headers)
+
       const res = await fetch(`${this._apiBaseUrl}/auth/session`, {
-        headers: {
-          ...this._headers,
-          cookie: `cf_clearance=${this._clearanceToken}; __Secure-next-auth.session-token=${this._sessionToken}`
-        }
+        headers
       }).then((r) => {
         response = r