Added GitHub init (#5317)

- Added github:init to allow full import, as opposed to gitHub:sync
which allows partial sync and therefore respecting Github API Limit
quota.

---------

Co-authored-by: Ady Beraud <a.beraud96@gmail.com>

packages/twenty-website/src/github/contributors/fetch-assignable-users.tsx

@@ -0,0 +1,21 @@
+import { graphql } from '@octokit/graphql';
+
+import { Repository } from '@/github/contributors/types';
+
+export async function fetchAssignableUsers(
+  query: typeof graphql,
+): Promise<Set<string>> {
+  const { repository } = await query<Repository>(`
+      query {
+        repository(owner: "twentyhq", name: "twenty") {
+          assignableUsers(first: 100) {
+            nodes {
+              login
+            }
+          }
+        }
+      }
+    `);
+
+  return new Set(repository.assignableUsers.nodes.map((user) => user.login));
+}

packages/twenty-website/src/github/contributors/fetch-issues-prs.tsx

@@ -0,0 +1,111 @@
+import { graphql } from '@octokit/graphql';
+
+import {
+  IssueNode,
+  PullRequestNode,
+  Repository,
+} from '@/github/contributors/types';
+
+// TODO: We should implement a true partial sync instead of using pageLimit.
+// Check search-issues-prs.tsx and modify "updated:>2024-02-27" to make it dynamic
+
+export async function fetchIssuesPRs(
+  query: typeof graphql,
+  cursor: string | null = null,
+  isIssues = false,
+  accumulatedData: Array<PullRequestNode | IssueNode> = [],
+  pageLimit: number,
+  currentPage = 0,
+): Promise<Array<PullRequestNode | IssueNode>> {
+  const { repository } = await query<Repository>(
+    `
+      query ($cursor: String) {
+        repository(owner: "twentyhq", name: "twenty") {
+          pullRequests(first: 30, after: $cursor, orderBy: {field: CREATED_AT, direction: DESC}) @skip(if: ${isIssues}) {
+            nodes {
+              id
+              title
+              body
+              url
+              createdAt
+              updatedAt
+              closedAt
+              mergedAt
+              author {
+                resourcePath
+                login
+                avatarUrl(size: 460)
+                url
+              }
+              labels(first: 100) {
+                nodes {
+                  id
+                  name
+                  color
+                  description
+                }
+              }
+            }
+            pageInfo {
+              hasNextPage
+              endCursor
+            }
+          }
+          issues(first: 100, after: $cursor, orderBy: {field: CREATED_AT, direction: DESC}) @skip(if: ${!isIssues}) {
+            nodes {
+              id
+              title
+              body
+              url
+              createdAt
+              updatedAt
+              closedAt
+              author {
+                resourcePath
+                login
+                avatarUrl
+                url
+              }
+              labels(first: 100) {
+                nodes {
+                  id
+                  name
+                  color
+                  description
+                }
+              }
+            }
+            pageInfo {
+              hasNextPage
+              endCursor
+            }
+          }
+        }
+      }
+    `,
+    { cursor },
+  );
+
+  const newAccumulatedData: Array<PullRequestNode | IssueNode> = [
+    ...accumulatedData,
+    ...(isIssues ? repository.issues.nodes : repository.pullRequests.nodes),
+  ];
+  const pageInfo = isIssues
+    ? repository.issues.pageInfo
+    : repository.pullRequests.pageInfo;
+
+  const newCurrentPage = currentPage + 1;
+
+  if ((!pageLimit || newCurrentPage < pageLimit) && pageInfo.hasNextPage) {
+    return fetchIssuesPRs(
+      query,
+      pageInfo.endCursor,
+      isIssues,
+      newAccumulatedData,
+      pageLimit,
+      currentPage + 1,
+    );
+  } else {
+    return newAccumulatedData;
+  }
+}

packages/twenty-website/src/github/contributors/save-issues-to-db.tsx

@@ -0,0 +1,69 @@
+import { insertMany } from '@/database/database';
+import {
+  issueLabelModel,
+  issueModel,
+  labelModel,
+  userModel,
+} from '@/database/model';
+import { IssueNode } from '@/github/contributors/types';
+
+export async function saveIssuesToDB(
+  issues: Array<IssueNode>,
+  assignableUsers: Set<string>,
+) {
+  for (const issue of issues) {
+    if (issue.author == null) {
+      continue;
+    }
+    await insertMany(
+      userModel,
+      [
+        {
+          id: issue.author.login,
+          avatarUrl: issue.author.avatarUrl,
+          url: issue.author.url,
+          isEmployee: assignableUsers.has(issue.author.login) ? '1' : '0',
+        },
+      ],
+      { onConflictKey: 'id' },
+    );
+
+    await insertMany(
+      issueModel,
+      [
+        {
+          id: issue.id,
+          title: issue.title,
+          body: issue.body,
+          url: issue.url,
+          createdAt: issue.createdAt,
+          updatedAt: issue.updatedAt,
+          closedAt: issue.closedAt,
+          authorId: issue.author.login,
+        },
+      ],
+      { onConflictKey: 'id' },
+    );
+
+    for (const label of issue.labels.nodes) {
+      await insertMany(
+        labelModel,
+        [
+          {
+            id: label.id,
+            name: label.name,
+            color: label.color,
+            description: label.description,
+          },
+        ],
+        { onConflictKey: 'id' },
+      );
+      await insertMany(issueLabelModel, [
+        {
+          pullRequestId: issue.id,
+          labelId: label.id,
+        },
+      ]);
+    }
+  }
+}

packages/twenty-website/src/github/contributors/save-prs-to-db.tsx

@@ -0,0 +1,71 @@
+import { insertMany } from '@/database/database';
+import {
+  labelModel,
+  pullRequestLabelModel,
+  pullRequestModel,
+  userModel,
+} from '@/database/model';
+import { PullRequestNode } from '@/github/contributors/types';
+
+export async function savePRsToDB(
+  prs: Array<PullRequestNode>,
+  assignableUsers: Set<string>,
+) {
+  for (const pr of prs) {
+    if (pr.author == null) {
+      continue;
+    }
+
+    await insertMany(
+      userModel,
+      [
+        {
+          id: pr.author.login,
+          avatarUrl: pr.author.avatarUrl,
+          url: pr.author.url,
+          isEmployee: assignableUsers.has(pr.author.login) ? '1' : '0',
+        },
+      ],
+      { onConflictKey: 'id' },
+    );
+
+    await insertMany(
+      pullRequestModel,
+      [
+        {
+          id: pr.id,
+          title: pr.title,
+          body: pr.body,
+          url: pr.url,
+          createdAt: pr.createdAt,
+          updatedAt: pr.updatedAt,
+          closedAt: pr.closedAt,
+          mergedAt: pr.mergedAt,
+          authorId: pr.author.login,
+        },
+      ],
+      { onConflictKey: 'id', onConflictUpdateObject: { title: pr.title } },
+    );
+
+    for (const label of pr.labels.nodes) {
+      await insertMany(
+        labelModel,
+        [
+          {
+            id: label.id,
+            name: label.name,
+            color: label.color,
+            description: label.description,
+          },
+        ],
+        { onConflictKey: 'id' },
+      );
+      await insertMany(pullRequestLabelModel, [
+        {
+          pullRequestId: pr.id,
+          labelId: label.id,
+        },
+      ]);
+    }
+  }
+}

packages/twenty-website/src/github/contributors/search-issues-prs.tsx

@@ -0,0 +1,99 @@
+import { graphql } from '@octokit/graphql';
+
+import {
+  IssueNode,
+  PullRequestNode,
+  SearchIssuesPRsQuery,
+} from '@/github/contributors/types';
+
+export async function searchIssuesPRs(
+  query: typeof graphql,
+  cursor: string | null = null,
+  isIssues = false,
+  accumulatedData: Array<PullRequestNode | IssueNode> = [],
+): Promise<Array<PullRequestNode | IssueNode>> {
+  const { search } = await query<SearchIssuesPRsQuery>(
+    `
+        query searchPullRequestsAndIssues($cursor: String) {
+          search(query: "repo:twentyhq/twenty ${
+            isIssues ? 'is:issue' : 'is:pr'
+          } updated:>2024-02-27", type: ISSUE, first: 100, after: $cursor) {
+            edges {
+              node {
+                ... on PullRequest {
+                  id
+                  title
+                  body
+                  url
+                  createdAt
+                  updatedAt
+                  closedAt
+                  mergedAt
+                  author {
+                    resourcePath
+                    login
+                    avatarUrl(size: 460)
+                    url
+                  }
+                  labels(first: 100) {
+                    nodes {
+                      id
+                      name
+                      color
+                      description
+                    }
+                  }
+                }
+                ... on Issue {
+                  id
+                  title
+                  body
+                  url
+                  createdAt
+                  updatedAt
+                  closedAt
+                  author {
+                    resourcePath
+                    login
+                    avatarUrl
+                    url
+                  }
+                  labels(first: 100) {
+                    nodes {
+                      id
+                      name
+                      color
+                      description
+                    }
+                  }
+                }
+              }
+            }
+            pageInfo {
+              hasNextPage
+              endCursor
+            }
+          }
+        }
+        `,
+    {
+      cursor,
+    },
+  );
+  const newAccumulatedData: Array<PullRequestNode | IssueNode> = [
+    ...accumulatedData,
+    ...search.edges.map(({ node }) => node),
+  ];
+  const pageInfo = search.pageInfo;
+
+  if (pageInfo.hasNextPage) {
+    return searchIssuesPRs(
+      query,
+      pageInfo.endCursor,
+      isIssues,
+      newAccumulatedData,
+    );
+  } else {
+    return newAccumulatedData;
+  }
+}

packages/twenty-website/src/github/contributors/types.tsx

@@ -0,0 +1,103 @@
+export interface LabelNode {
+  id: string;
+  name: string;
+  color: string;
+  description: string;
+}
+
+export interface AuthorNode {
+  resourcePath: string;
+  login: string;
+  avatarUrl: string;
+  url: string;
+}
+
+export interface PullRequestNode {
+  id: string;
+  title: string;
+  body: string;
+  url: string;
+  createdAt: string;
+  updatedAt: string;
+  closedAt: string;
+  mergedAt: string;
+  author: AuthorNode;
+  labels: {
+    nodes: LabelNode[];
+  };
+}
+
+export interface IssueNode {
+  id: string;
+  title: string;
+  body: string;
+  url: string;
+  createdAt: string;
+  updatedAt: string;
+  closedAt: string;
+  author: AuthorNode;
+  labels: {
+    nodes: LabelNode[];
+  };
+}
+
+export interface PageInfo {
+  hasNextPage: boolean;
+  endCursor: string | null;
+}
+
+export interface PullRequests {
+  nodes: PullRequestNode[];
+  pageInfo: PageInfo;
+}
+
+export interface Issues {
+  nodes: IssueNode[];
+  pageInfo: PageInfo;
+}
+
+export interface AssignableUserNode {
+  login: string;
+}
+
+export interface AssignableUsers {
+  nodes: AssignableUserNode[];
+}
+
+export interface Stargazers {
+  totalCount: number;
+}
+
+export interface Releases {
+  nodes: ReleaseNode[];
+}
+
+export interface ReleaseNode {
+  tagName: string;
+  name: string;
+  description: string;
+  publishedAt: string;
+}
+
+export interface Repository {
+  repository: {
+    pullRequests: PullRequests;
+    issues: Issues;
+    assignableUsers: AssignableUsers;
+    stargazers: Stargazers;
+    releases: Releases;
+  };
+}
+
+export interface SearchEdgeNode {
+  node: IssueNode | PullRequestNode;
+}
+
+export interface SearchEdges {
+  edges: SearchEdgeNode[];
+  pageInfo: PageInfo;
+}
+
+export interface SearchIssuesPRsQuery {
+  search: SearchEdges;
+}

packages/twenty-website/src/github/fetch-and-save-github-data.ts

@@ -0,0 +1,52 @@
+import { global } from '@apollo/client/utilities/globals';
+import { graphql } from '@octokit/graphql';
+
+import { fetchAssignableUsers } from '@/github/contributors/fetch-assignable-users';
+import { fetchIssuesPRs } from '@/github/contributors/fetch-issues-prs';
+import { saveIssuesToDB } from '@/github/contributors/save-issues-to-db';
+import { savePRsToDB } from '@/github/contributors/save-prs-to-db';
+import { IssueNode, PullRequestNode } from '@/github/contributors/types';
+import { fetchAndSaveGithubReleases } from '@/github/github-releases/fetch-and-save-github-releases';
+import { fetchAndSaveGithubStars } from '@/github/github-stars/fetch-and-save-github-stars';
+
+export const fetchAndSaveGithubData = async ({
+  pageLimit,
+}: {
+  pageLimit: number;
+}) => {
+  if (!global.process.env.GITHUB_TOKEN) {
+    return new Error('No GitHub token provided');
+  }
+
+  console.log('Synching data..');
+
+  const query = graphql.defaults({
+    headers: {
+      Authorization: 'bearer ' + global.process.env.GITHUB_TOKEN,
+    },
+  });
+
+  await fetchAndSaveGithubStars(query);
+  await fetchAndSaveGithubReleases(query);
+
+  const assignableUsers = await fetchAssignableUsers(query);
+  const fetchedPRs = (await fetchIssuesPRs(
+    query,
+    null,
+    false,
+    [],
+    pageLimit,
+  )) as Array<PullRequestNode>;
+  const fetchedIssues = (await fetchIssuesPRs(
+    query,
+    null,
+    true,
+    [],
+    pageLimit,
+  )) as Array<IssueNode>;
+
+  await savePRsToDB(fetchedPRs, assignableUsers);
+  await saveIssuesToDB(fetchedIssues, assignableUsers);
+
+  console.log('data synched!');
+};

packages/twenty-website/src/github/github-releases/fetch-and-save-github-releases.tsx

@@ -0,0 +1,26 @@
+import { graphql } from '@octokit/graphql';
+
+import { insertMany } from '@/database/database';
+import { githubReleasesModel } from '@/database/model';
+import { Repository } from '@/github/contributors/types';
+
+export const fetchAndSaveGithubReleases = async (
+  query: typeof graphql,
+): Promise<void> => {
+  const { repository } = await query<Repository>(`
+        query {
+          repository(owner: "twentyhq", name: "twenty") {
+            releases(first: 100) {  
+                nodes {
+                  tagName
+                  publishedAt
+                }
+            }
+          }
+        }
+    `);
+
+  await insertMany(githubReleasesModel, repository.releases.nodes, {
+    onConflictKey: 'tagName',
+  });
+};

packages/twenty-website/src/github/github-stars/fetch-and-save-github-stars.tsx

@@ -0,0 +1,27 @@
+import { graphql } from '@octokit/graphql';
+
+import { insertMany } from '@/database/database';
+import { githubStarsModel } from '@/database/model';
+import { Repository } from '@/github/contributors/types';
+
+export const fetchAndSaveGithubStars = async (
+  query: typeof graphql,
+): Promise<void> => {
+  const { repository } = await query<Repository>(`
+        query {
+          repository(owner: "twentyhq", name: "twenty") {
+            stargazers {
+              totalCount
+            }
+          }
+        }
+    `);
+
+  const numberOfStars = repository.stargazers.totalCount;
+
+  await insertMany(githubStarsModel, [
+    {
+      numberOfStars,
+    },
+  ]);
+};

packages/twenty-website/src/github/github-sync.ts

@@ -0,0 +1,15 @@
+import { fetchAndSaveGithubData } from '@/github/fetch-and-save-github-data';
+
+export const githubSync = async () => {
+  const pageLimitFlagIndex = process.argv.indexOf('--pageLimit');
+  let pageLimit = 0;
+
+  if (pageLimitFlagIndex > -1) {
+    pageLimit = parseInt(process.argv[pageLimitFlagIndex + 1], 10);
+  }
+
+  await fetchAndSaveGithubData({ pageLimit });
+  process.exit(0);
+};
+
+githubSync();