Merge pull request #99 from transitive-bullshit/fix/cloudflare-protections

7e1ed3b3 · Travis Fischer · GitHub · 0740333f · be046679 · 7e1ed3b3
Commit 7e1ed3b3 authored Dec 12, 2022 by Travis Fischer Committed by GitHub Dec 12, 2022
9 changed files
--- a/.env.example
+++ b/.env.example
@@ -9,5 +9,5 @@
 # ChatGPT
 # -----------------------------------------------------------------------------
-# see the readme for how to find this
+EMAIL=
-SESSION_TOKEN=
+PASSWORD=
--- a/src/demo-conversation.ts
+++ b/src/demo-conversation.ts
 import dotenv from 'dotenv-safe'
 import { oraPromise } from 'ora'
-import { ChatGPTAPI } from '.'
+import { ChatGPTAPI } from '../src'
+import { getOpenAIAuthInfo } from './openai-auth-puppeteer'
 dotenv.config()
@@ -13,7 +14,15 @@ dotenv.config()
 * ```
 */
 async function main() {
-  const api = new ChatGPTAPI({ sessionToken: process.env.SESSION_TOKEN })
+  const email = process.env.EMAIL
+  const password = process.env.PASSWORD
+  const authInfo = await getOpenAIAuthInfo({
+    email,
+    password
+  })
+  const api = new ChatGPTAPI({ ...authInfo })
  await api.ensureAuth()
  const conversation = api.getConversation()

--- a/src/demo.ts
+++ b/src/demo.ts
 import dotenv from 'dotenv-safe'
 import { oraPromise } from 'ora'
-import { ChatGPTAPI } from '.'
+import { ChatGPTAPI } from '../src'
+import { getOpenAIAuthInfo } from './openai-auth-puppeteer'
 dotenv.config()
@@ -13,7 +14,15 @@ dotenv.config()
 * ```
 */
 async function main() {
-  const api = new ChatGPTAPI({ sessionToken: process.env.SESSION_TOKEN })
+  const email = process.env.EMAIL
+  const password = process.env.PASSWORD
+  const authInfo = await getOpenAIAuthInfo({
+    email,
+    password
+  })
+  const api = new ChatGPTAPI({ ...authInfo })
  await api.ensureAuth()
  const prompt =

--- a/demos/openai-auth-puppeteer.ts
+++ b/demos/openai-auth-puppeteer.ts
+import delay from 'delay'
+import {
+  type Browser,
+  type Page,
+  type Protocol,
+  type PuppeteerLaunchOptions
+} from 'puppeteer'
+import puppeteer from 'puppeteer-extra'
+import StealthPlugin from 'puppeteer-extra-plugin-stealth'
+puppeteer.use(StealthPlugin())
+export type OpenAIAuthInfo = {
+  userAgent: string
+  clearanceToken: string
+  sessionToken: string
+  cookies?: Record<string, Protocol.Network.Cookie>
+}
+/**
+ * Bypasses OpenAI's use of Cloudflare to get the cookies required to use
+ * ChatGPT. Uses Puppeteer with a stealth plugin under the hood.
+ */
+export async function getOpenAIAuthInfo({
+  email,
+  password,
+  timeout = 2 * 60 * 1000,
+  browser
+}: {
+  email: string
+  password: string
+  timeout?: number
+  browser?: Browser
+}): Promise<OpenAIAuthInfo> {
+  let page: Page
+  let origBrowser = browser
+  try {
+    if (!browser) {
+      browser = await getBrowser()
+    }
+    const userAgent = await browser.userAgent()
+    page = (await browser.pages())[0] || (await browser.newPage())
+    page.setDefaultTimeout(timeout)
+    await page.goto('https://chat.openai.com/auth/login')
+    await page.waitForSelector('#__next .btn-primary', { timeout })
+    await delay(1000)
+    if (email && password) {
+      await Promise.all([
+        page.click('#__next .btn-primary'),
+        page.waitForNavigation({
+          waitUntil: 'networkidle0'
+        })
+      ])
+      await page.type('#username', email, { delay: 10 })
+      await page.click('button[type="submit"]')
+      await page.waitForSelector('#password')
+      await page.type('#password', password, { delay: 10 })
+      await Promise.all([
+        page.click('button[type="submit"]'),
+        page.waitForNavigation({
+          waitUntil: 'networkidle0'
+        })
+      ])
+    }
+    const pageCookies = await page.cookies()
+    const cookies = pageCookies.reduce(
+      (map, cookie) => ({ ...map, [cookie.name]: cookie }),
+      {}
+    )
+    const authInfo: OpenAIAuthInfo = {
+      userAgent,
+      clearanceToken: cookies['cf_clearance']?.value,
+      sessionToken: cookies['__Secure-next-auth.session-token']?.value,
+      cookies
+    }
+    return authInfo
+  } catch (err) {
+    console.error(err)
+    throw null
+  } finally {
+    if (origBrowser) {
+      if (page) {
+        await page.close()
+      }
+    } else if (browser) {
+      await browser.close()
+    }
+    page = null
+    browser = null
+  }
+}
+export async function getBrowser(launchOptions?: PuppeteerLaunchOptions) {
+  const macChromePath =
+    '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'
+  return puppeteer.launch({
+    headless: false,
+    args: ['--no-sandbox', '--exclude-switches', 'enable-automation'],
+    ignoreHTTPSErrors: true,
+    // executablePath: executablePath()
+    executablePath: macChromePath,
+    ...launchOptions
+  })
+}
--- a/package.json
+++ b/package.json
@@ -34,7 +34,6 @@
    "prepare": "husky install",
    "pre-commit": "lint-staged",
    "test": "run-p test:*",
-    "test:unit": "ava",
    "test:prettier": "prettier '**/*.{js,jsx,ts,tsx}' --check"
  },
  "dependencies": {
@@ -51,12 +50,16 @@
    "@types/uuid": "^9.0.0",
    "ava": "^5.1.0",
    "del-cli": "^5.0.0",
+    "delay": "^5.0.0",
    "dotenv-safe": "^8.2.0",
    "husky": "^8.0.2",
    "lint-staged": "^13.0.3",
    "npm-run-all": "^4.1.5",
    "ora": "^6.1.2",
    "prettier": "^2.8.0",
+    "puppeteer": "^19.4.0",
+    "puppeteer-extra": "^3.3.4",
+    "puppeteer-extra-plugin-stealth": "^2.11.1",
    "tsup": "^6.5.0",
    "tsx": "^3.12.1",
    "typedoc": "^0.23.21",

--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
--- a/readme.md
+++ b/readme.md
-# Update December 11, 2022
+# Update December 12, 2022
-Today, OpenAI added additional Cloudflare protections that make it more difficult to access the unofficial API. _This is affecting all ChatGPT API wrappers at the moment_, including the Python ones. See [this issue](https://github.com/transitive-bullshit/chatgpt-api/issues/96).
+Yesterday, OpenAI added additional Cloudflare protections that make it more difficult to access the unofficial API.
-**As a temporary workaround**, make sure you're using the latest version of this package and Node.js >= 18:
+The demos have been updated to use Puppeteer to log in to ChatGPT and extract the Cloudflare `cf_clearance` cookie and OpenAI session token. 🔥
-1. Log into https://chat.openai.com/chat and copy a fresh session token (same instructions as below).
+To use the updated version, first make sure you're using the latest version of this package and Node.js >= 18:
-2. Copy the value of the `cf_clearance` cookie and store it in a `CLEARANCE_TOKEN` environment variable in addition to your `SESSION_TOKEN`.
-3. Copy your browser's `user-agent` header from any request in your browser's network tab.
-4. Use both tokens when creating the API wrapper:
 ```ts
 const api = new ChatGPTAPI({
  sessionToken: process.env.SESSION_TOKEN,
  clearanceToken: process.env.CLEARANCE_TOKEN,
-  userAgent: '' // replace to match your real browser's user agent
+  userAgent: '' // needs to match your browser's user agent
 })
 await api.ensureAuth()
@@ -24,9 +18,10 @@ await api.ensureAuth()
 Restrictions on this method:
- Cloudflare `cf_clearance` **tokens expire after 2 hours**, so right now you'll have to manually log in and extract it by hand every so often
+- Cloudflare `cf_clearance` **tokens expire after 2 hours**, so right now we recommend that you refresh your `cf_clearance` token every ~45 minutes or so.
 - Your `user-agent` and `IP address` **must match** from the real browser window you're logged in with to the one you're using for `ChatGPTAPI`.
  - This means that you currently can't log in with your laptop and then run the bot on a server or proxy somewhere.
+- Cloudflare will still sometimes ask you to complete a CAPTCHA, so you may need to keep an eye on it and manually resolve the CAPTCHA. Automated CAPTCHA bypass is a WIP.
 - You must use `node >= 18`. I'm using `v19.2.0` in my testing, but for some reason, all `fetch` requests using Node.js `v16` and `v17` fail at the moment (these use `undici` under the hood, whereas Node.js v18 and above use a built-in `fetch` based on `undici`).
 - You should not be using this account while the bot is using it, because that browser window may refresh one of your tokens and invalidate the bot's session.
@@ -47,7 +42,7 @@ Travis
 [![NPM](https://img.shields.io/npm/v/chatgpt.svg)](https://www.npmjs.com/package/chatgpt) [![Build Status](https://github.com/transitive-bullshit/chatgpt-api/actions/workflows/test.yml/badge.svg)](https://github.com/transitive-bullshit/chatgpt-api/actions/workflows/test.yml) [![MIT License](https://img.shields.io/badge/license-MIT-blue)](https://github.com/transitive-bullshit/chatgpt-api/blob/main/license) [![Prettier Code Formatting](https://img.shields.io/badge/code_style-prettier-brightgreen.svg)](https://prettier.io)
- [Update December 11, 2022](#update-december-11-2022)
+- [Update December 12, 2022](#update-december-12-2022)
  - [Intro](#intro)
  - [Install](#install)
  - [Usage](#usage)
@@ -77,12 +72,12 @@ npm install chatgpt
 import { ChatGPTAPI } from 'chatgpt'
 async function example() {
-  // sessionToken is required; see below for details
  const api = new ChatGPTAPI({
-    sessionToken: process.env.SESSION_TOKEN
+    sessionToken: process.env.SESSION_TOKEN,
+    clearanceToken: process.env.CLEARANCE_TOKEN,
+    userAgent: 'TODO'
  })
-  // ensure the API is properly authenticated
  await api.ensureAuth()
  // send a message and wait for the response
@@ -100,6 +95,8 @@ ChatGPT responses are formatted as markdown by default. If you want to work with
 ```ts
 const api = new ChatGPTAPI({
  sessionToken: process.env.SESSION_TOKEN,
+  clearanceToken: process.env.CLEARANCE_TOKEN,
+  userAgent: 'TODO',
  markdown: false
 })
 ```
@@ -108,7 +105,9 @@ If you want to automatically track the conversation, you can use `ChatGPTAPI.get
 ```ts
 const api = new ChatGPTAPI({
-  sessionToken: process.env.SESSION_TOKEN
+  sessionToken: process.env.SESSION_TOKEN,
+  clearanceToken: process.env.CLEARANCE_TOKEN,
+  userAgent: 'TODO'
 })
 const conversation = api.getConversation()
@@ -145,7 +144,9 @@ async function example() {
  const { ChatGPTAPI } = await import('chatgpt')
  const api = new ChatGPTAPI({
-    sessionToken: process.env.SESSION_TOKEN
+    sessionToken: process.env.SESSION_TOKEN,
+    clearanceToken: process.env.CLEARANCE_TOKEN,
+    userAgent: 'TODO'
  })
  await api.ensureAuth()
@@ -162,23 +163,21 @@ See the [auto-generated docs](./docs/classes/ChatGPTAPI.md) for more info on met
 ### Demos
-A [basic demo](./src/demo.ts) is included for testing purposes:
+To run the included demos:
+1. clone repo
+2. install node deps
+3. set `EMAIL` and `PASSWORD` in .env
+A [basic demo](./demos/demo.ts) is included for testing purposes:
 ```bash
-# 1. clone repo
-# 2. install node deps
-# 3. set `SESSION_TOKEN` in .env
-# 4. run:
 npx tsx src/demo.ts
 ```
-A [conversation demo](./src/demo-conversation.ts) is also included:
+A [conversation demo](./demos/demo-conversation.ts) is also included:
 ```bash
-# 1. clone repo
-# 2. install node deps
-# 3. set `SESSION_TOKEN` in .env
-# 4. run:
 npx tsx src/demo-conversation.ts
 ```
@@ -186,15 +185,18 @@ npx tsx src/demo-conversation.ts
 **This package requires a valid session token from ChatGPT to access it's unofficial REST API.**
-To get a session token:
+As of December 11, 2021, it also requires a valid Cloudflare clearance token.
+There are two options to get these; either manually, or automated. For the automated way, see the `demos/` folder using Puppeteer.
+To get a session token manually:
 1. Go to https://chat.openai.com/chat and log in or sign up.
 2. Open dev tools.
 3. Open `Application` > `Cookies`.
   ![ChatGPT cookies](./media/session-token.png)
 4. Copy the value for `__Secure-next-auth.session-token` and save it to your environment.
+5. Copy the value for `cf_clearance` and save it to your environment.
-If you want to run the built-in demo, store this value as `SESSION_TOKEN` in a local `.env` file.
 > **Note**
 > This package will switch to using the official API once it's released.

--- a/src/chatgpt-api.test.ts
+++ b/src/chatgpt-api.test.ts
@@ -11,13 +11,16 @@ const isCI = !!process.env.CI
 test('ChatGPTAPI invalid session token', async (t) => {
  t.timeout(30 * 1000) // 30 seconds
-  t.throws(() => new ChatGPTAPI({ sessionToken: null }), {
+  t.throws(() => new ChatGPTAPI({ sessionToken: null, clearanceToken: null }), {
    message: 'ChatGPT invalid session token'
  })
  await t.throwsAsync(
    async () => {
-      const chatgpt = new ChatGPTAPI({ sessionToken: 'invalid' })
+      const chatgpt = new ChatGPTAPI({
+        sessionToken: 'invalid',
+        clearanceToken: 'invalid'
+      })
      await chatgpt.ensureAuth()
    },
    {
@@ -33,13 +36,18 @@ test('ChatGPTAPI valid session token', async (t) => {
  }
  t.notThrows(
-    () => new ChatGPTAPI({ sessionToken: 'fake valid session token' })
+    () =>
+      new ChatGPTAPI({
+        sessionToken: 'fake valid session token',
+        clearanceToken: 'invalid'
+      })
  )
  await t.notThrowsAsync(
    (async () => {
      const chatgpt = new ChatGPTAPI({
-        sessionToken: process.env.SESSION_TOKEN
+        sessionToken: process.env.SESSION_TOKEN,
+        clearanceToken: process.env.CLEARANCE_TOKEN
      })
      // Don't make any real API calls using our session token if we're running on CI
@@ -62,7 +70,10 @@ if (!isCI) {
    await t.throwsAsync(
      async () => {
-        const chatgpt = new ChatGPTAPI({ sessionToken: expiredSessionToken })
+        const chatgpt = new ChatGPTAPI({
+          sessionToken: expiredSessionToken,
+          clearanceToken: 'invalid'
+        })
        await chatgpt.ensureAuth()
      },
      {
@@ -81,7 +92,8 @@ if (!isCI) {
    await t.throwsAsync(
      async () => {
        const chatgpt = new ChatGPTAPI({
-          sessionToken: process.env.SESSION_TOKEN
+          sessionToken: process.env.SESSION_TOKEN,
+          clearanceToken: process.env.CLEARANCE_TOKEN
        })
        await chatgpt.sendMessage('test', {
@@ -100,7 +112,8 @@ if (!isCI) {
    await t.throwsAsync(
      async () => {
        const chatgpt = new ChatGPTAPI({
-          sessionToken: process.env.SESSION_TOKEN
+          sessionToken: process.env.SESSION_TOKEN,
+          clearanceToken: process.env.CLEARANCE_TOKEN
        })
        const abortController = new AbortController()

--- a/src/chatgpt-api.ts
+++ b/src/chatgpt-api.ts
@@ -10,7 +10,7 @@ import { markdownToText } from './utils'
 const KEY_ACCESS_TOKEN = 'accessToken'
 const USER_AGENT =
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36'
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36'
 export class ChatGPTAPI {
  protected _sessionToken: string
@@ -22,7 +22,6 @@ export class ChatGPTAPI {
  protected _headers: Record<string, string>
  // Stores access tokens for `accessTokenTTL` milliseconds before needing to refresh
-  // (defaults to 60 seconds)
  protected _accessTokenCache: ExpiryMap<string, string>
  protected _user: types.User | null = null
@@ -52,10 +51,12 @@ export class ChatGPTAPI {
    /** @defaultValue `'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36'` **/
    userAgent?: string
-    /** @defaultValue 60000 (60 seconds) */
+    /** @defaultValue 1 hour */
    accessTokenTTL?: number
    accessToken?: string
+    headers?: Record<string, string>
  }) {
    const {
      sessionToken,
@@ -64,8 +65,9 @@ export class ChatGPTAPI {
      apiBaseUrl = 'https://chat.openai.com/api',
      backendApiBaseUrl = 'https://chat.openai.com/backend-api',
      userAgent = USER_AGENT,
-      accessTokenTTL = 60000, // 60 seconds
+      accessTokenTTL = 60 * 60000, // 1 hour
-      accessToken
+      accessToken,
+      headers
    } = opts
    this._sessionToken = sessionToken
@@ -75,11 +77,18 @@ export class ChatGPTAPI {
    this._backendApiBaseUrl = backendApiBaseUrl
    this._userAgent = userAgent
    this._headers = {
-      'User-Agent': this._userAgent,
+      'user-agent': this._userAgent,
      'x-openai-assistant-app-id': '',
      'accept-language': 'en-US,en;q=0.9',
      origin: 'https://chat.openai.com',
-      referer: 'https://chat.openai.com/chat'
+      referer: 'https://chat.openai.com/chat',
+      'sec-ch-ua':
+        '"Not?A_Brand";v="8", "Chromium";v="108", "Google Chrome";v="108"',
+      'sec-ch-ua-platform': '"macOS"',
+      'sec-fetch-dest': 'empty',
+      'sec-fetch-mode': 'cors',
+      'sec-fetch-site': 'same-origin',
+      ...headers
    }
    this._accessTokenCache = new ExpiryMap<string, string>(accessTokenTTL)
@@ -90,6 +99,10 @@ export class ChatGPTAPI {
    if (!this._sessionToken) {
      throw new types.ChatGPTError('ChatGPT invalid session token')
    }
+    if (!this._clearanceToken) {
+      throw new types.ChatGPTError('ChatGPT invalid clearance token')
+    }
  }
  /**
@@ -269,11 +282,15 @@ export class ChatGPTAPI {
    let response: Response
    try {
+      const headers = {
+        ...this._headers,
+        cookie: `cf_clearance=${this._clearanceToken}; __Secure-next-auth.session-token=${this._sessionToken}`,
+        accept: '*/*'
+      }
+      console.log(`${this._apiBaseUrl}/auth/session`, headers)
      const res = await fetch(`${this._apiBaseUrl}/auth/session`, {
-        headers: {
+        headers
-          ...this._headers,
-          cookie: `cf_clearance=${this._clearanceToken}; __Secure-next-auth.session-token=${this._sessionToken}`
-        }
      }).then((r) => {
        response = r