✨ GitHub Notion Sync

.gitignore

@@ -4,3 +4,4 @@ node_modules/
 !.docker/data/.keep
 dist/
 private.json
+config.json

scripts/notion/GithubNotionSync.js

@@ -1,18 +1,57 @@
 const fetch = require("node-fetch");
 const { Client } = require("@notionhq/client");
+const ntc = require("./ntc");
+const bodyParser = require("body-parser");
+const express = require("express");
+const app = express();
 
 class GithubNotionSync {
-	constructor(config, githubOrg, notionDatabase) {
+	constructor({ github, notion }) {
-		this.githubAuth = config.githubAuth;
+		this.githubAuth = github.token;
-		this.notionAuth = config.notionAuth;
+		this.githubWebhookSecret = github.webhookSecret;
-		this.databaseID = notionDatabase;
+		this.notionAuth = notion.token;
-		this.org = githubOrg;
+		this.databaseID = notion.database_id;
+		this.org = github.organization;
 		this.notion = new Client({ auth: this.notionAuth });
 		this.urls = {
 			base: "https://api.github.com/",
 		};
 	}
 
+	async init() {
+		this.allNotionPages = await this.getAllNotionPages();
+		this.repos = await this.getAllIssueUrls();
+
+		app.use(bodyParser({}));
+		app.post("/github", this.handleWebhook);
+		app.listen(3010, () => {
+			console.log("Github <-> Notion sync listening on :3010");
+		});
+	}
+
+	handleWebhook(req, res) {
+		const { hook, issue } = req.body;
+		if (this.githubWebhookSecret && this.githubWebhookSecret !== hook.config.secret)
+			return res.status(400).send("invalid secret");
+
+		await this.addItemToDb(GithubNotionSync.convertIssue(issue));
+		res.sendStatus(200);
+	}
+
+	async execute() {
+		await this.init();
+		let issues = 0;
+
+		for (let repo of this.repos) {
+			for (let issue of await this.getAllIssuesPerRepo(repo)) {
+				await this.addItemToDb(issue);
+				issues++;
+			}
+		}
+
+		return issues;
+	}
+
 	/**
 	 * @returns array of urls in the following form:
 	 * https://api.github.com/repos/${this.org}/${repo_name}/issues
@@ -24,9 +63,7 @@ class GithubNotionSync {
 				"User-Agent": this.org,
 			},
 		}).then((r) => r.json());
-		return repos.map(
+		return repos.map((repo) => `${this.urls.base}repos/${this.org}/${repo.name}/issues`);
-			(repo) => `${this.urls.base}repos/${this.org}/${repo.name}/issues`
-		);
 	}
 
 	/**
@@ -45,38 +82,65 @@ class GithubNotionSync {
 	 * ```
 	 */
 	async getAllIssuesPerRepo(repoIssueUrl) {
-		let issues = await fetch(repoIssueUrl, {
+		var allIssues = [];
+		var page = 1;
+
+		do {
+			var issues = await fetch(`${repoIssueUrl}?state=all&direction=asc&per_page=100&page=${page}`, {
 				headers: {
 					Authorization: `token ${this.githubAuth}`,
 					"User-Agent": this.org,
 				},
 			}).then((r) => r.json());
-		return issues.map((x) => {
+			issues = issues.filter((x) => !x.pull_request).map(GithubNotionSync.convertIssue);
+			page++;
+
+			allIssues = allIssues.concat(issues);
+		} while (issues.length);
+		return allIssues;
+	}
+
+	static convertIssue(x) {
 		return {
-				url: x?.url,
+			url: x?.html_url,
 			title: x?.title,
 			body: x?.body,
 			number: x?.number,
 			state: x?.state,
-				label: x?.labels[0]?.name,
+			labels: x?.labels,
-				assignee: x?.assignee?.login,
+			assignees: x?.assignees,
 		};
+	}
+
+	/**
+	 * @returns {Promise<import("@notionhq/client/build/src/api-types").Page[]>}
+	 */
+	async getAllNotionPages() {
+		var allPages = [];
+		var start_cursor;
+
+		do {
+			var pages = await this.notion.databases.query({
+				database_id: this.databaseID,
+				page_size: 100,
+				...(start_cursor && { start_cursor }),
 			});
+			start_cursor = pages.next_cursor;
+			allPages = allPages.concat(pages.results);
+		} while (pages.has_more);
+
+		return allPages;
 	}
 
 	async addItemToDb(issue) {
-		try {
+		const options = {
-			const response = await this.notion.pages.create({
 			parent: { database_id: this.databaseID },
 			properties: {
 				Name: {
 					title: [
 						{
 							text: {
-									content: issue.title /*.replace(
+								content: issue.title,
-										/(\[.+\].)/g,
-										""
-									),*/,
 							},
 						},
 					],
@@ -86,34 +150,26 @@ class GithubNotionSync {
 						name: issue.state,
 					},
 				},
-					// "Type of Issue": {
+
-					// 	select: {
-					// 		name:
-					// 			issue.title.split("]")[0].replace("[", "") ||
-					// 			"none",
-					// 	},
-					// },
-					Label: {
-						select: {
-							name: issue.label || "none",
-						},
-					},
-					Assignee: {
-						select: {
-							name: issue.assignee || "none",
-						},
-					},
 				Repo: {
 					select: {
-							name: issue.url.match(
+						name: GithubNotionSync.getRepoNameFromUrl(issue.url),
-								/fosscord\/(fosscord-)?([\w.-]+)/
-							)[2],
 					},
 				},
 				Url: {
 					url: issue.url,
 				},
 				Number: { number: issue.number },
+				...(issue.assignee && {
+					Assignee: {
+						multi_select: issue.assignees.map((x) => ({ name: x.login })),
+					},
+				}),
+				...(issue.label && {
+					Label: {
+						multi_select: issue.labels.map((x) => ({ name: x.name, color: ntc(x.color) })),
+					},
+				}),
 			},
 			children: [
 				{
@@ -124,23 +180,44 @@ class GithubNotionSync {
 							{
 								type: "text",
 								text: {
-										content: `${
+									content: `${issue.body?.length > 1990 ? "issue body too long" : issue.body}`,
-											issue.body.length > 1990
-												? "issue body too long"
-												: issue.body
-										}`,
 								},
 							},
 						],
 					},
 				},
 			],
-			});
+		};
-			console.log(response);
+
+		const exists = this.allNotionPages.find(
+			(x) =>
+				x.properties.Number.number == issue.number &&
+				x.properties.Repo.select.name === GithubNotionSync.getRepoNameFromUrl(issue.url)
+		);
+
+		try {
+			if (exists) {
+				if (
+					exists.properties.State?.select.name !== issue.state ||
+					exists.properties.Label?.select.name !== issue.label ||
+					exists.properties.Assignee?.select.name !== issue.assignee
+				) {
+					console.log("update existing one");
+					await this.notion.pages.update({ page_id: exists.id, properties: options.properties });
+					exists.properties = options.properties;
+				}
+			} else {
+				console.log("adding new one");
+				this.allNotionPages.push(await this.notion.pages.create(options));
+			}
 		} catch (error) {
 			console.log(issue);
-			console.error(error.body);
+			console.error(error.body, error);
 		}
 	}
+
+	static getRepoNameFromUrl(url) {
+		return url?.match(/fosscord\/(fosscord-)?([\w.-]+)/)[2];
+	}
 }
 module.exports = { GithubNotionSync };

scripts/notion/config.example.json

@@ -0,0 +1,11 @@
+{
+	"github": {
+		"token": "",
+		"webhookSecret": "",
+		"organization": ""
+	},
+	"notion": {
+		"token": "",
+		"database_id": ""
+	}
+}

scripts/notion/main.js

@@ -1,19 +1,7 @@
-const config = require("./private.json");
+const config = require("./config.json");
 const { GithubNotionSync } = require("./GithubNotionSync");
-const sync = new GithubNotionSync(
+const sync = new GithubNotionSync(config);
-	config,
-	"fosscord",
-	"2c7fe9e73f9842d3bab3a4912dedd091"
-);
 
-(async () => {
+sync.execute()
-	const repos = await sync.getAllIssueUrls();
+	.then((x) => console.log(`synced ${x} issues`))
-	let issues = 0;
+	.catch(console.error);
-	for (repo of repos) {
-		for (issue of await sync.getAllIssuesPerRepo(repo)) {
-			await sync.addItemToDb(issue);
-			issues++;
-		}
-	}
-	console.log(`synced ${issues}`);
-})();

scripts/notion/ntc.js

@@ -0,0 +1,147 @@
+/*
+
+https://chir.ag/projects/name-that-color/
+
++-----------------------------------------------------------------+
+|     Created by Chirag Mehta - http://chir.ag/projects/ntc       |
+|-----------------------------------------------------------------|
+|               ntc js (Name that Color JavaScript)               |
++-----------------------------------------------------------------+
+
+All the functions, code, lists etc. have been written specifically
+for the Name that Color JavaScript by Chirag Mehta unless otherwise
+specified.
+
+This script is released under the: Creative Commons License:
+Attribution 2.5 http://creativecommons.org/licenses/by/2.5/
+
+Sample Usage:
+
+  <script type="text/javascript" src="ntc.js"></script>
+
+  <script type="text/javascript">
+
+    var n_match  = ntc.name("#6195ED");
+    n_rgb        = n_match[0]; // This is the RGB value of the closest matching color
+    n_name       = n_match[1]; // This is the text string for the name of the match
+    n_exactmatch = n_match[2]; // True if exact color match, False if close-match
+
+    alert(n_match);
+
+  </script>
+
+*/
+
+var ntc = {
+	init: function () {
+		var color, rgb, hsl;
+		for (var i = 0; i < ntc.names.length; i++) {
+			color = "#" + ntc.names[i][0];
+			rgb = ntc.rgb(color);
+			hsl = ntc.hsl(color);
+			ntc.names[i].push(rgb[0], rgb[1], rgb[2], hsl[0], hsl[1], hsl[2]);
+		}
+	},
+
+	name: function (color) {
+		color = color.toUpperCase();
+		if (color.length < 3 || color.length > 7) return ["#000000", "Invalid Color: " + color, false];
+		if (color.length % 3 == 0) color = "#" + color;
+		if (color.length == 4)
+			color =
+				"#" +
+				color.substr(1, 1) +
+				color.substr(1, 1) +
+				color.substr(2, 1) +
+				color.substr(2, 1) +
+				color.substr(3, 1) +
+				color.substr(3, 1);
+
+		var rgb = ntc.rgb(color);
+		var r = rgb[0],
+			g = rgb[1],
+			b = rgb[2];
+		var hsl = ntc.hsl(color);
+		var h = hsl[0],
+			s = hsl[1],
+			l = hsl[2];
+		var ndf1 = 0;
+		ndf2 = 0;
+		ndf = 0;
+		var cl = -1,
+			df = -1;
+
+		for (var i = 0; i < ntc.names.length; i++) {
+			if (color == "#" + ntc.names[i][0]) return ["#" + ntc.names[i][0], ntc.names[i][1], true];
+
+			ndf1 =
+				Math.pow(r - ntc.names[i][2], 2) + Math.pow(g - ntc.names[i][3], 2) + Math.pow(b - ntc.names[i][4], 2);
+			ndf2 =
+				Math.pow(h - ntc.names[i][5], 2) + Math.pow(s - ntc.names[i][6], 2) + Math.pow(l - ntc.names[i][7], 2);
+			ndf = ndf1 + ndf2 * 2;
+			if (df < 0 || df > ndf) {
+				df = ndf;
+				cl = i;
+			}
+		}
+
+		return cl < 0 ? "default" : ntc.names[cl][1];
+	},
+
+	// adopted from: Farbtastic 1.2
+	// http://acko.net/dev/farbtastic
+	hsl: function (color) {
+		var rgb = [
+			parseInt("0x" + color.substring(1, 3)) / 255,
+			parseInt("0x" + color.substring(3, 5)) / 255,
+			parseInt("0x" + color.substring(5, 7)) / 255,
+		];
+		var min, max, delta, h, s, l;
+		var r = rgb[0],
+			g = rgb[1],
+			b = rgb[2];
+
+		min = Math.min(r, Math.min(g, b));
+		max = Math.max(r, Math.max(g, b));
+		delta = max - min;
+		l = (min + max) / 2;
+
+		s = 0;
+		if (l > 0 && l < 1) s = delta / (l < 0.5 ? 2 * l : 2 - 2 * l);
+
+		h = 0;
+		if (delta > 0) {
+			if (max == r && max != g) h += (g - b) / delta;
+			if (max == g && max != b) h += 2 + (b - r) / delta;
+			if (max == b && max != r) h += 4 + (r - g) / delta;
+			h /= 6;
+		}
+		return [parseInt(h * 255), parseInt(s * 255), parseInt(l * 255)];
+	},
+
+	// adopted from: Farbtastic 1.2
+	// http://acko.net/dev/farbtastic
+	rgb: function (color) {
+		return [
+			parseInt("0x" + color.substring(1, 3)),
+			parseInt("0x" + color.substring(3, 5)),
+			parseInt("0x" + color.substring(5, 7)),
+		];
+	},
+
+	names: [
+		["808080", "gray"],
+		["0000FF", "blue"],
+		["0000FF", "red"],
+		["00FF00", "green"],
+		["FFFF00", "yellow"],
+		["FF00FF", "pink"],
+		["7c00b5", "purple"],
+		["ff8c00", "orange"],
+		["693900", "brown"],
+	],
+};
+
+module.exports = ntc;
+
+ntc.init();

scripts/notion/package.json

@@ -4,12 +4,18 @@
 	"description": "",
 	"main": "main.js",
 	"scripts": {
+		"start": "node main.js",
 		"test": "echo \"Error: no test specified\" && exit 1"
 	},
 	"author": "",
 	"license": "ISC",
 	"dependencies": {
 		"@notionhq/client": "^0.2.4",
+		"body-parser": "^1.19.0",
+		"express": "^4.17.1",
 		"node-fetch": "^2.6.1"
+	},
+	"devDependencies": {
+		"ts-node": "^10.2.0"
 	}
 }