Make Github stars dynamic and improve database init (#5000)

I extracted the init database logic into its own file. 
You can now run it with yarn database:init.
Added database entry for GitHub stars. 

Do you want me to remove the init route or is it used for something else
?

---------

Co-authored-by: Ady Beraud <a.beraud96@gmail.com>
Co-authored-by: Félix Malfait <felix.malfait@gmail.com>

package.json

@@ -308,6 +308,7 @@
     "ts-loader": "^9.2.3",
     "ts-node": "10.9.1",
     "tsconfig-paths": "^4.2.0",
+    "tsx": "^4.7.2",
     "typescript": "5.3.3",
     "vite": "^5.0.0",
     "vite-plugin-checker": "^0.6.2",

packages/twenty-website/package.json

@@ -8,6 +8,8 @@
     "build": "npx next build",
     "start": "npx next start",
     "lint": "npx next lint",
+    "github:sync": "npx tsx src/github-sync/github-sync.ts",
+    "database:migrate": "npx tsx src/database/migrate-database.ts",
     "database:generate:pg": "npx drizzle-kit generate:pg --config=src/database/postgres/drizzle-posgres.config.ts",
     "database:generate:sqlite": "npx drizzle-kit generate:sqlite --config=src/database/sqlite/drizzle-sqlite.config.ts"
   },

packages/twenty-website/src/app/_components/ui/layout/header/HeaderDesktop.tsx

@@ -11,8 +11,13 @@ import {
   LogoContainer,
 } from '@/app/_components/ui/layout/header/styled';
 import { Logo } from '@/app/_components/ui/layout/Logo';
+import { formatNumberOfStars } from '@/shared-utils/formatNumberOfStars';
 
-export const HeaderDesktop = () => {
+type Props = {
+  numberOfStars: number;
+};
+
+export const HeaderDesktop = ({ numberOfStars }: Props) => {
   return (
     <DesktopNav>
       <LogoContainer>
@@ -26,7 +31,9 @@ export const HeaderDesktop = () => {
           Docs <ExternalArrow />
         </ListItem>
         <ListItem href="https://github.com/twentyhq/twenty">
-          <GithubIcon color="rgb(71,71,71)" /> 8.3k <ExternalArrow />
+          <GithubIcon color="rgb(71,71,71)" />
+          {formatNumberOfStars(numberOfStars)}
+          <ExternalArrow />
         </ListItem>
       </LinkList>
       <CallToAction />

packages/twenty-website/src/app/_components/ui/layout/header/HeaderMobile.tsx

@@ -18,6 +18,7 @@ import {
   NavOpen,
 } from '@/app/_components/ui/layout/header/styled';
 import { Logo } from '@/app/_components/ui/layout/Logo';
+import { formatNumberOfStars } from '@/shared-utils/formatNumberOfStars';
 
 const IBMPlexMono = IBM_Plex_Mono({
   weight: '500',
@@ -25,7 +26,11 @@ const IBMPlexMono = IBM_Plex_Mono({
   display: 'swap',
 });
 
-export const HeaderMobile = () => {
+type Props = {
+  numberOfStars: number;
+};
+
+export const HeaderMobile = ({ numberOfStars }: Props) => {
   const isTwentyDev = false;
 
   const [menuOpen, setMenuOpen] = useState(false);
@@ -64,7 +69,8 @@ export const HeaderMobile = () => {
             Docs <ExternalArrow />
           </ListItem>
           <ListItem href="https://github.com/twentyhq/twenty">
-            <GithubIcon color="rgb(71,71,71)" /> 8.3k <ExternalArrow />
+            <GithubIcon color="rgb(71,71,71)" />{' '}
+            {formatNumberOfStars(numberOfStars)} <ExternalArrow />
           </ListItem>
         </MobileLinkList>
         <CallToAction />

packages/twenty-website/src/app/_components/ui/layout/header/index.tsx

@@ -1,13 +1,20 @@
-'use client';
+import { desc } from 'drizzle-orm';
 
 import { HeaderDesktop } from '@/app/_components/ui/layout/header/HeaderDesktop';
 import { HeaderMobile } from '@/app/_components/ui/layout/header/HeaderMobile';
+import { findOne } from '@/database/database';
+import { githubStarsModel } from '@/database/model';
+
+export const AppHeader = async () => {
+  const githubStars = await findOne(
+    githubStarsModel,
+    desc(githubStarsModel.timestamp),
+  );
 
-export const AppHeader = () => {
   return (
     <>
-      <HeaderDesktop />
+      <HeaderDesktop numberOfStars={githubStars?.[0]?.numberOfStars} />
-      <HeaderMobile />
+      <HeaderMobile numberOfStars={githubStars?.[0]?.numberOfStars} />
     </>
   );
 };

packages/twenty-website/src/app/api/github-stars/route.tsx

@@ -0,0 +1,26 @@
+import { desc } from 'drizzle-orm';
+
+import { findOne } from '@/database/database';
+import { githubStarsModel } from '@/database/model';
+import { formatNumberOfStars } from '@/shared-utils/formatNumberOfStars';
+
+export const dynamic = 'force-dynamic';
+
+export async function GET() {
+  try {
+    const githubStars = await findOne(
+      githubStarsModel,
+      desc(githubStarsModel.timestamp),
+    );
+
+    const formattedGithubNumberOfStars = formatNumberOfStars(
+      githubStars[0].numberOfStars,
+    );
+
+    return Response.json(formattedGithubNumberOfStars);
+  } catch (error: any) {
+    return new Response(`Github stars error: ${error?.message}`, {
+      status: 500,
+    });
+  }
+}

packages/twenty-website/src/app/contributors/api/init/route.tsx

@@ -1,46 +0,0 @@
-export const dynamic = 'force-dynamic';
-
-import { global } from '@apollo/client/utilities/globals';
-import { graphql } from '@octokit/graphql';
-
-import { fetchAssignableUsers } from '@/app/contributors/api/fetch-assignable-users';
-import { fetchIssuesPRs } from '@/app/contributors/api/fetch-issues-prs';
-import { saveIssuesToDB } from '@/app/contributors/api/save-issues-to-db';
-import { savePRsToDB } from '@/app/contributors/api/save-prs-to-db';
-import { IssueNode, PullRequestNode } from '@/app/contributors/api/types';
-import { migrate } from '@/database/database';
-
-export async function GET() {
-  if (!global.process.env.GITHUB_TOKEN) {
-    return new Response('No GitHub token provided', { status: 500 });
-  }
-
-  const query = graphql.defaults({
-    headers: {
-      Authorization: 'bearer ' + global.process.env.GITHUB_TOKEN,
-    },
-  });
-
-  await migrate();
-
-  const assignableUsers = await fetchAssignableUsers(query);
-  const fetchedPRs = (await fetchIssuesPRs(
-    query,
-    null,
-    false,
-    [],
-  )) as Array<PullRequestNode>;
-  const fetchedIssues = (await fetchIssuesPRs(
-    query,
-    null,
-    true,
-    [],
-  )) as Array<IssueNode>;
-
-  savePRsToDB(fetchedPRs, assignableUsers);
-  saveIssuesToDB(fetchedIssues, assignableUsers);
-
-  return new Response('Data synced', {
-    status: 200,
-  });
-}

packages/twenty-website/src/app/contributors/api/update/route.tsx

@@ -1,13 +1,13 @@
 import { graphql } from '@octokit/graphql';
 import { desc } from 'drizzle-orm';
 
-import { fetchAssignableUsers } from '@/app/contributors/api/fetch-assignable-users';
-import { saveIssuesToDB } from '@/app/contributors/api/save-issues-to-db';
-import { savePRsToDB } from '@/app/contributors/api/save-prs-to-db';
-import { searchIssuesPRs } from '@/app/contributors/api/search-issues-prs';
-import { IssueNode, PullRequestNode } from '@/app/contributors/api/types';
 import { findOne } from '@/database/database';
 import { issueModel, pullRequestModel } from '@/database/model';
+import { fetchAssignableUsers } from '@/github-sync/contributors/fetch-assignable-users';
+import { saveIssuesToDB } from '@/github-sync/contributors/save-issues-to-db';
+import { savePRsToDB } from '@/github-sync/contributors/save-prs-to-db';
+import { searchIssuesPRs } from '@/github-sync/contributors/search-issues-prs';
+import { IssueNode, PullRequestNode } from '@/github-sync/contributors/types';
 
 export async function GET() {
   if (!global.process.env.GITHUB_TOKEN) {

packages/twenty-website/src/database/migrate-database.ts

@@ -0,0 +1,8 @@
+import { migrate } from '@/database/database';
+
+export const migrateDatabase = async () => {
+  await migrate();
+  process.exit(0);
+};
+
+migrateDatabase();

packages/twenty-website/src/database/model.ts

@@ -1,4 +1,5 @@
 import {
+  pgGithubStars,
   pgIssueLabels,
   pgIssues,
   pgLabels,
@@ -31,17 +32,19 @@ export const issueLabelModel = isSqliteDriver
   ? sqlLiteIssueLabels
   : pgIssueLabels;
 
-export type User = typeof sqlLiteUsers.$inferSelect;
+export const githubStarsModel = pgGithubStars;
-export type PullRequest = typeof sqlLitePullRequests.$inferSelect;
-export type Issue = typeof sqlLiteIssues.$inferSelect;
-export type Label = typeof sqlLiteLabels.$inferSelect;
-export type PullRequestLabel = typeof sqlLitePullRequestLabels.$inferSelect;
-export type IssueLabel = typeof sqlLiteIssueLabels.$inferSelect;
 
-export type UserInsert = typeof sqlLiteUsers.$inferInsert;
+export type User = typeof pgUsers.$inferSelect;
-export type PullRequestInsert = typeof sqlLitePullRequests.$inferInsert;
+export type PullRequest = typeof pgPullRequests.$inferSelect;
-export type IssueInsert = typeof sqlLiteIssues.$inferInsert;
+export type Issue = typeof pgIssues.$inferSelect;
-export type LabelInsert = typeof sqlLiteLabels.$inferInsert;
+export type Label = typeof pgLabels.$inferSelect;
-export type PullRequestLabelInsert =
+export type PullRequestLabel = typeof pgPullRequestLabels.$inferSelect;
-  typeof sqlLitePullRequestLabels.$inferInsert;
+export type IssueLabel = typeof pgIssueLabels.$inferSelect;
-export type IssueLabelInsert = typeof sqlLiteIssueLabels.$inferInsert;
+
+export type UserInsert = typeof pgUsers.$inferInsert;
+export type PullRequestInsert = typeof pgPullRequests.$inferInsert;
+export type IssueInsert = typeof pgIssues.$inferInsert;
+export type LabelInsert = typeof pgLabels.$inferInsert;
+export type PullRequestLabelInsert = typeof pgPullRequestLabels.$inferInsert;
+export type IssueLabelInsert = typeof pgIssueLabels.$inferInsert;
+export type GithubStars = typeof pgGithubStars.$inferInsert;

packages/twenty-website/src/database/postgres/migrations/0001_marvelous_eddie_brock.sql

@@ -0,0 +1,4 @@
+CREATE TABLE IF NOT EXISTS "githubStars" (
+	"timestamp" timestamp DEFAULT now() NOT NULL,
+	"numberOfStars" integer
+);

packages/twenty-website/src/database/postgres/migrations/meta/0000_snapshot.json

@@ -27,12 +27,8 @@
           "name": "issueLabels_issueId_issues_id_fk",
           "tableFrom": "issueLabels",
           "tableTo": "issues",
-          "columnsFrom": [
+          "columnsFrom": ["issueId"],
-            "issueId"
+          "columnsTo": ["id"],
-          ],
-          "columnsTo": [
-            "id"
-          ],
           "onDelete": "no action",
           "onUpdate": "no action"
         },
@@ -40,12 +36,8 @@
           "name": "issueLabels_labelId_labels_id_fk",
           "tableFrom": "issueLabels",
           "tableTo": "labels",
-          "columnsFrom": [
+          "columnsFrom": ["labelId"],
-            "labelId"
+          "columnsTo": ["id"],
-          ],
-          "columnsTo": [
-            "id"
-          ],
           "onDelete": "no action",
           "onUpdate": "no action"
         }
@@ -118,12 +110,8 @@
           "name": "issues_authorId_users_id_fk",
           "tableFrom": "issues",
           "tableTo": "users",
-          "columnsFrom": [
+          "columnsFrom": ["authorId"],
-            "authorId"
+          "columnsTo": ["id"],
-          ],
-          "columnsTo": [
-            "id"
-          ],
           "onDelete": "no action",
           "onUpdate": "no action"
         }
@@ -194,12 +182,8 @@
           "name": "pullRequestLabels_pullRequestExternalId_pullRequests_id_fk",
           "tableFrom": "pullRequestLabels",
           "tableTo": "pullRequests",
-          "columnsFrom": [
+          "columnsFrom": ["pullRequestExternalId"],
-            "pullRequestExternalId"
+          "columnsTo": ["id"],
-          ],
-          "columnsTo": [
-            "id"
-          ],
           "onDelete": "no action",
           "onUpdate": "no action"
         },
@@ -207,12 +191,8 @@
           "name": "pullRequestLabels_labelId_labels_id_fk",
           "tableFrom": "pullRequestLabels",
           "tableTo": "labels",
-          "columnsFrom": [
+          "columnsFrom": ["labelId"],
-            "labelId"
+          "columnsTo": ["id"],
-          ],
-          "columnsTo": [
-            "id"
-          ],
           "onDelete": "no action",
           "onUpdate": "no action"
         }
@@ -285,12 +265,8 @@
           "name": "pullRequests_authorId_users_id_fk",
           "tableFrom": "pullRequests",
           "tableTo": "users",
-          "columnsFrom": [
+          "columnsFrom": ["authorId"],
-            "authorId"
+          "columnsTo": ["id"],
-          ],
-          "columnsTo": [
-            "id"
-          ],
           "onDelete": "no action",
           "onUpdate": "no action"
         }

packages/twenty-website/src/database/postgres/migrations/meta/_journal.json

@@ -8,6 +8,13 @@
       "when": 1707921820164,
       "tag": "0000_absent_giant_man",
       "breakpoints": true
+    },
+    {
+      "idx": 1,
+      "version": "5",
+      "when": 1713792223113,
+      "tag": "0001_marvelous_eddie_brock",
+      "breakpoints": true
     }
   ]
 }

packages/twenty-website/src/database/postgres/schema-postgres.ts

@@ -1,4 +1,4 @@
-import { pgTable, text } from 'drizzle-orm/pg-core';
+import { integer, pgTable, text, timestamp } from 'drizzle-orm/pg-core';
 
 export const pgUsers = pgTable('users', {
   id: text('id').primaryKey(),
@@ -50,3 +50,8 @@ export const pgIssueLabels = pgTable('issueLabels', {
   issueId: text('issueId').references(() => pgIssues.id),
   labelId: text('labelId').references(() => pgLabels.id),
 });
+
+export const pgGithubStars = pgTable('githubStars', {
+  timestamp: timestamp('timestamp').notNull().defaultNow(),
+  numberOfStars: integer('numberOfStars'),
+});

packages/twenty-website/src/github-sync/contributors/fetch-assignable-users.tsx

@@ -1,6 +1,6 @@
 import { graphql } from '@octokit/graphql';
 
-import { Repository } from '@/app/contributors/api/types';
+import { Repository } from '@/github-sync/contributors/types';
 
 export async function fetchAssignableUsers(
   query: typeof graphql,

packages/twenty-website/src/github-sync/contributors/fetch-issues-prs.tsx

@@ -4,7 +4,7 @@ import {
   IssueNode,
   PullRequestNode,
   Repository,
-} from '@/app/contributors/api/types';
+} from '@/github-sync/contributors/types';
 
 export async function fetchIssuesPRs(
   query: typeof graphql,

packages/twenty-website/src/github-sync/contributors/save-issues-to-db.tsx

@@ -1,4 +1,3 @@
-import { IssueNode } from '@/app/contributors/api/types';
 import { insertMany } from '@/database/database';
 import {
   issueLabelModel,
@@ -6,6 +5,7 @@ import {
   labelModel,
   userModel,
 } from '@/database/model';
+import { IssueNode } from '@/github-sync/contributors/types';
 
 export async function saveIssuesToDB(
   issues: Array<IssueNode>,

packages/twenty-website/src/github-sync/contributors/save-prs-to-db.tsx

@@ -1,4 +1,3 @@
-import { PullRequestNode } from '@/app/contributors/api/types';
 import { insertMany } from '@/database/database';
 import {
   labelModel,
@@ -6,6 +5,7 @@ import {
   pullRequestModel,
   userModel,
 } from '@/database/model';
+import { PullRequestNode } from '@/github-sync/contributors/types';
 
 export async function savePRsToDB(
   prs: Array<PullRequestNode>,

packages/twenty-website/src/github-sync/contributors/search-issues-prs.tsx

@@ -4,7 +4,7 @@ import {
   IssueNode,
   PullRequestNode,
   SearchIssuesPRsQuery,
-} from '@/app/contributors/api/types';
+} from '@/github-sync/contributors/types';
 
 export async function searchIssuesPRs(
   query: typeof graphql,

packages/twenty-website/src/github-sync/contributors/types.tsx

@@ -64,11 +64,16 @@ export interface AssignableUsers {
   nodes: AssignableUserNode[];
 }
 
+export interface Stargazers {
+  totalCount: number;
+}
+
 export interface Repository {
   repository: {
     pullRequests: PullRequests;
     issues: Issues;
     assignableUsers: AssignableUsers;
+    stargazers: Stargazers;
   };
 }
 

packages/twenty-website/src/github-sync/fetch-and-save-github-data.ts

@@ -0,0 +1,44 @@
+import { global } from '@apollo/client/utilities/globals';
+import { graphql } from '@octokit/graphql';
+
+import { fetchAssignableUsers } from '@/github-sync/contributors/fetch-assignable-users';
+import { fetchIssuesPRs } from '@/github-sync/contributors/fetch-issues-prs';
+import { saveIssuesToDB } from '@/github-sync/contributors/save-issues-to-db';
+import { savePRsToDB } from '@/github-sync/contributors/save-prs-to-db';
+import { IssueNode, PullRequestNode } from '@/github-sync/contributors/types';
+import { fetchAndSaveGithubStars } from '@/github-sync/github-stars/fetch-and-save-github-stars';
+
+export const fetchAndSaveGithubData = async () => {
+  if (!global.process.env.GITHUB_TOKEN) {
+    return new Error('No GitHub token provided');
+  }
+
+  console.log('Synching data..');
+
+  const query = graphql.defaults({
+    headers: {
+      Authorization: 'bearer ' + global.process.env.GITHUB_TOKEN,
+    },
+  });
+
+  await fetchAndSaveGithubStars(query);
+
+  const assignableUsers = await fetchAssignableUsers(query);
+  const fetchedPRs = (await fetchIssuesPRs(
+    query,
+    null,
+    false,
+    [],
+  )) as Array<PullRequestNode>;
+  const fetchedIssues = (await fetchIssuesPRs(
+    query,
+    null,
+    true,
+    [],
+  )) as Array<IssueNode>;
+
+  savePRsToDB(fetchedPRs, assignableUsers);
+  saveIssuesToDB(fetchedIssues, assignableUsers);
+
+  console.log('data synched!');
+};

packages/twenty-website/src/github-sync/github-stars/fetch-and-save-github-stars.tsx

@@ -0,0 +1,27 @@
+import { graphql } from '@octokit/graphql';
+
+import { insertMany } from '@/database/database';
+import { githubStarsModel } from '@/database/model';
+import { Repository } from '@/github-sync/contributors/types';
+
+export const fetchAndSaveGithubStars = async (
+  query: typeof graphql,
+): Promise<void> => {
+  const { repository } = await query<Repository>(`
+        query {
+          repository(owner: "twentyhq", name: "twenty") {
+            stargazers {
+              totalCount
+            }
+          }
+        }
+    `);
+
+  const numberOfStars = repository.stargazers.totalCount;
+
+  await insertMany(githubStarsModel, [
+    {
+      numberOfStars,
+    },
+  ]);
+};

packages/twenty-website/src/github-sync/github-sync.ts

@@ -0,0 +1,8 @@
+import { fetchAndSaveGithubData } from '@/github-sync/fetch-and-save-github-data';
+
+export const githubSync = async () => {
+  await fetchAndSaveGithubData();
+  process.exit(0);
+};
+
+githubSync();

packages/twenty-website/src/shared-utils/formatNumberOfStars.ts

@@ -0,0 +1,3 @@
+export const formatNumberOfStars = (numberOfStars: number) => {
+  return Math.floor(numberOfStars / 100) / 10 + 'k';
+};

yarn.lock

@@ -25843,7 +25843,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"esbuild@npm:^0.19.7":
+"esbuild@npm:^0.19.7, esbuild@npm:~0.19.10":
   version: 0.19.12
   resolution: "esbuild@npm:0.19.12"
   dependencies:
@@ -28372,6 +28372,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"get-tsconfig@npm:^4.7.2":
+  version: 4.7.3
+  resolution: "get-tsconfig@npm:4.7.3"
+  dependencies:
+    resolve-pkg-maps: "npm:^1.0.0"
+  checksum: b15ca9d5d0887ebfccadc9fe88b6ff3827a5691ec90e7608a5e9c74bef959c14aba62f6bb88ac7f50322395731789a2cf654244f00e10f4f76349911b6846d6f
+  languageName: node
+  linkType: hard
+
 "getpass@npm:^0.1.1":
   version: 0.1.7
   resolution: "getpass@npm:0.1.7"
@@ -46211,6 +46220,22 @@ __metadata:
   languageName: node
   linkType: hard
 
+"tsx@npm:^4.7.2":
+  version: 4.7.2
+  resolution: "tsx@npm:4.7.2"
+  dependencies:
+    esbuild: "npm:~0.19.10"
+    fsevents: "npm:~2.3.3"
+    get-tsconfig: "npm:^4.7.2"
+  dependenciesMeta:
+    fsevents:
+      optional: true
+  bin:
+    tsx: dist/cli.mjs
+  checksum: 0338598cc3b7b01a47939297797dfb77a1d675acb33bf71e816faf2b8cb76da3994d341d2920d105dbe98cd01a4babd80ca4b9a5a36120813dd79c4fc1c32df1
+  languageName: node
+  linkType: hard
+
 "tty-browserify@npm:0.0.1, tty-browserify@npm:^0.0.1":
   version: 0.0.1
   resolution: "tty-browserify@npm:0.0.1"
@@ -46647,6 +46672,7 @@ __metadata:
     tsconfig-paths: "npm:^4.2.0"
     tslib: "npm:^2.3.0"
     tsup: "npm:^8.0.1"
+    tsx: "npm:^4.7.2"
     type-fest: "npm:4.10.1"
     typeorm: "npm:^0.3.17"
     typescript: "npm:5.3.3"