commit 3e8d2ebae90b75686a6d79939c90202a16160dcb Author: Cris Stringfellow <22254235+crislin2046@users.noreply.github.com> Date: Sun Jan 15 02:07:52 2023 +0800 create diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..c401d68 Binary files /dev/null and b/.DS_Store differ diff --git a/.eslintrc.cjs b/.eslintrc.cjs new file mode 100644 index 0000000..ca9fa77 --- /dev/null +++ b/.eslintrc.cjs @@ -0,0 +1,16 @@ +module.exports = { + "env": { + "es2021": true, + "node": true + }, + "extends": "eslint:recommended", + "parserOptions": { + "ecmaVersion": 13, + "sourceType": "module" + }, + "ignorePatterns": [ + "build/**/*.js" + ], + "rules": { + } +}; diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..af1f7a8 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +custom: https://buy.stripe.com/3cs7tEcC53Yv3zG8xb diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b1e0135 --- /dev/null +++ b/.gitignore @@ -0,0 +1,124 @@ +.*.swp + +# Bundling and packaging +22120.exe +22120.nix +22120.mac +22120.win32.exe +22120.nix32 +bin/* +build/* + +#Leave these to allow install by npm -g +#22120.js +#*.22120.js + +# Library +public/library/cache.json +public/library/http* + + +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# TypeScript v1 declaration files +typings/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env +.env.test + +# parcel-bundler cache (https://parceljs.org/) +.cache + +# Next.js build output +.next + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and *not* Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port diff --git a/.npm.release b/.npm.release new file mode 100644 index 0000000..65e2464 --- /dev/null +++ b/.npm.release @@ -0,0 +1 @@ +Sun Jan 15 01:13:51 CST 2023 diff --git a/.npmignore b/.npmignore new file mode 100644 index 0000000..69422de --- /dev/null +++ b/.npmignore @@ -0,0 +1,5 @@ + +.*.swp + +# Bundling and packaging +bin/* diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..a09c9fe --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,112 @@ +Copyright Dosyago Corporation & Cris Stringfellow (https://dosaygo.com) + +22120 and all previously released versions, including binaries, NPM packages, and +Docker images (including all named archivist1 and any other names) +is re-licensed under the following PolyForm Strict License 1.0.0 and all previous +licenses are revoked. + + +# PolyForm Strict License 1.0.0 + + + +## Acceptance + +In order to get any license under these terms, you must agree +to them as both strict obligations and conditions to all +your licenses. + +## Copyright License + +The licensor grants you a copyright license for the software +to do everything you might do with the software that would +otherwise infringe the licensor's copyright in it for any +permitted purpose, other than distributing the software or +making changes or new works based on the software. + +## Patent License + +The licensor grants you a patent license for the software that +covers patent claims the licensor can license, or becomes able +to license, that you would infringe by using the software. + +## Noncommercial Purposes + +Any noncommercial purpose is a permitted purpose. + +## Personal Uses + +Personal use for research, experiment, and testing for +the benefit of public knowledge, personal study, private +entertainment, hobby projects, amateur pursuits, or religious +observance, without any anticipated commercial application, +is use for a permitted purpose. + +## Noncommercial Organizations + +Use by any charitable organization, educational institution, +public research organization, public safety or health +organization, environmental protection organization, +or government institution is use for a permitted purpose +regardless of the source of funding or obligations resulting +from the funding. + +## Fair Use + +You may have "fair use" rights for the software under the +law. These terms do not limit them. + +## No Other Rights + +These terms do not allow you to sublicense or transfer any of +your licenses to anyone else, or prevent the licensor from +granting licenses to anyone else. These terms do not imply +any other licenses. + +## Patent Defense + +If you make any written claim that the software infringes or +contributes to infringement of any patent, your patent license +for the software granted under these terms ends immediately. If +your company makes such a claim, your patent license ends +immediately for work on behalf of your company. + +## Violations + +The first time you are notified in writing that you have +violated any of these terms, or done anything with the software +not covered by your licenses, your licenses can nonetheless +continue if you come into full compliance with these terms, +and take practical steps to correct past violations, within +32 days of receiving notice. Otherwise, all your licenses +end immediately. + +## No Liability + +***As far as the law allows, the software comes as is, without +any warranty or condition, and the licensor will not be liable +to you for any damages arising out of these terms or the use +or nature of the software, under any kind of legal claim.*** + +## Definitions + +The **licensor** is the individual or entity offering these +terms, and the **software** is the software the licensor makes +available under these terms. + +**You** refers to the individual or entity agreeing to these +terms. + +**Your company** is any legal entity, sole proprietorship, +or other kind of organization that you work for, plus all +organizations that have control over, are under the control of, +or are under common control with that organization. **Control** +means ownership of substantially all the assets of an entity, +or the power to direct its management and policies by vote, +contract, or otherwise. Control can be direct or indirect. + +**Your licenses** are all the licenses granted to you for the +software under these terms. + +**Use** means anything you do with the software requiring one +of your licenses. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..98ea6c7 --- /dev/null +++ b/NOTICE @@ -0,0 +1,7 @@ +Copyright Dosyago Corporation & Cris Stringfellow (https://dosaygo.com) + +22120 and all previously released versions, including binaries, NPM packages, and +Docker images (including all named archivist1, and all other previous names) +is re-licensed under the following PolyForm Strict License 1.0.0 and all previous +licenses are revoked. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..12d0249 --- /dev/null +++ b/README.md @@ -0,0 +1,120 @@ +# :floppy_disk: [DiskerNet](https://github.com/c9fe/22120) [![source lines of code](https://sloc.xyz/github/crisdosyago/Diskernet)](https://sloc.xyz) [![npm downloads (22120)](https://img.shields.io/npm/dt/archivist1?label=npm%20downloads%20%2822120%29)](https://npmjs.com/package/archivist1) [![npm downloads (diskernet, since Jan 2022)](https://img.shields.io/npm/dt/diskernet?label=npm%20downloads%20%28diskernet%2C%20since%20Jan%202022%29)](https://npmjs.com/package/diskernet) [![binary downloads](https://img.shields.io/github/downloads/c9fe/22120/total?label=OS%20binary%20downloads)](https://GitHub.com/crisdosyago/DiskerNet/releases) [![visitors+++](https://hits.seeyoufarm.com/api/count/incr/badge.svg?url=https%3A%2F%2Fgithub.com%2Fc9fe%2F22120&count_bg=%2379C83D&title_bg=%23555555&icon=&icon_color=%23E7E7E7&title=%28today%2Ftotal%29%20visitors%2B%2B%2B%20since%20Oct%2027%202020&edge_flat=false)](https://hits.seeyoufarm.com) ![version](https://img.shields.io/npm/v/archivist1) [![DiskerNet slogan](https://img.shields.io/badge/%F0%9F%92%BE%20DiskerNet-an%20internet%20on%20yer%20disc-hotpink)](#DiskerNet) + +

DiskerNet

+ +

Introducing DiskerNet - the tool that connects to your browser and makes it easy to save and organize the things you discover online.

+ +

With DiskerNet, you can choose between two modes:

+ + + +

No matter which mode you choose, DiskerNet makes it easy to search, browse, and share your archived content. Plus, our tool is lightweight and doesn't require any special plugins or extensions.

+ +

Key features

+ + + +

Why choose DiskerNet?

+ + + +

To get started with DiskerNet, simply download the tool and connect it to your browser. You'll be amazed at how much easier it is to save and organize the things you discover online.

+ +

Licensing

+ +

DiskerNet is licensed under the Polyform Strict License 1.0.0. You can purchase a license for different uses below:

+ +- For personal, research, noncommercial purposes: Buy a Perpetual Non-commercial Use License of the current Version re-upped Monthly to the Latest Version, [USD$1.99 per month, paid yearly](https://buy.stripe.com/7sIg0acC5amT7PW6pl). Or [purchase an Unlimited Time version for the equivalent of 5 years](https://buy.stripe.com/14k5lw31veD96LS29b). +- For part of your internal tooling in your org: Buy a Perpetual Internal Use License of the current Version re-upped Monthly to the Latest Version, [USD $12.99 per month, paid yearly](https://buy.stripe.com/9AQaFQ59D52z3zGdRS), or [purchase an Unlimited Time version for the equivalent of 5 years](https://buy.stripe.com/fZe3do1Xr1Qn9Y4011). +- For anywhere in your business: Buy a Perpetual Small-medium Business License of the current Version re-upped Monthly to the Latest Version, [USD $99 per month, paid yearly](https://buy.stripe.com/eVa8xIcC5gLhfio6po). Or [purchase an Unlimited Time version for the equivalent of 5 years](https://buy.stripe.com/00g5lwcC566D7PW00Z). + +## Get it + +[Download a release](https://github.com/crisdosyago/Diskernet/releases) + +or ... + +**Get it on [npm](https://www.npmjs.com/package/diskernet):** + +```sh +$ npm i -g diskernet@latest +``` + +or... + +**Build your own binaries:** + +```sh +$ git clone https://github.com/crisdosyago/DiskerNet +$ cd DiskerNet +$ npm i +$ ./scripts/build_setup.sh +$ ./scripts/compile.sh +$ cd bin/ +``` + +### Frequently Asked Questions + +**What is the licensing for Diskernet?** + +Diskernet is licensed under the Polyform Strict License 1.0.0. This license allows individuals to use the tool for free for personal, noncommercial purposes. It also allows businesses and organizations to purchase a license for use in their internal tooling or anywhere in their business. + +**Why did you choose the Polyform license for Diskernet?** + +We chose the Polyform license for Diskernet because it offers several benefits. It protects our rights as the creators of the tool, it allows individuals to use the tool for free for personal use, and it allows businesses to purchase a license for use in their operations. We believe that the Polyform license strikes a good balance between the interests of the open source community and the rights of the creators of Diskernet. + +**Is the Polyform license open source?** + +The Polyform license is not an open source license as defined by the Open Source Initiative (OSI). However, it allows individuals to use the tool for free for personal, noncommercial purposes, and it allows businesses to purchase a license for use in their operations. + +**Can I modify or distribute Diskernet under the Polyform license?** + +No, the Polyform license does not allow users to modify or distribute Diskernet without the permission of the creators. This is to protect our rights as the creators of the tool and to ensure that our work is not used or distributed without our permission. + +**Can I use Diskernet for commercial purposes?** + +Yes, you can use Diskernet for commercial purposes if you purchase a license from us. The license allows businesses and organizations to use Diskernet for their own purposes, including in their internal tooling or anywhere in their business. + +**What is Diskernet?** + +Diskernet is a tool for archiving and organizing online content. It connects to your browser and automatically saves the pages you visit, allowing you to easily search, browse, and share your archived content. + +**What are the key features of Diskernet?** + +The key features of Diskernet include: + +- Connects to your browser and automatically archives your browsing activity +- Two modes: archive everything or only bookmark-worthy content +- Easy-to-use interface for searching, browsing, and sharing your archives +- Lightweight and doesn't require any special plugins or extensions + +**Why should I use Diskernet?** + +There are several reasons to use Diskernet: + +- Never lose track of your favorite online content again +- Save time by quickly finding the information you need +- Share your archives with others, or keep them private +- Easy to use and doesn't require any extra tools or plugins + +**How do I get started with Diskernet?** + +Getting started with Diskernet is easy! Simply download the tool and connect it to your browser. You'll be amazed at how much easier it is to save and organize the things you discover online. + +**Does Diskernet work with all browsers?** + +No, right now Diskernet is compatible with Chrome and Chromium only (although theoretically compatible with Edge owing to the Remote Debugging Protocol). + diff --git a/docs/OLD-README.md b/docs/OLD-README.md new file mode 100644 index 0000000..0d3cb86 --- /dev/null +++ b/docs/OLD-README.md @@ -0,0 +1,275 @@ +# :floppy_disk: [DiskerNet](https://github.com/c9fe/22120) [![source lines of code](https://sloc.xyz/github/crisdosyago/Diskernet)](https://sloc.xyz) [![npm downloads (22120)](https://img.shields.io/npm/dt/archivist1?label=npm%20downloads%20%2822120%29)](https://npmjs.com/package/archivist1) [![npm downloads (diskernet, since Jan 2022)](https://img.shields.io/npm/dt/diskernet?label=npm%20downloads%20%28diskernet%2C%20since%20Jan%202022%29)](https://npmjs.com/package/diskernet) [![binary downloads](https://img.shields.io/github/downloads/c9fe/22120/total?label=OS%20binary%20downloads)](https://GitHub.com/crisdosyago/DiskerNet/releases) [![visitors+++](https://hits.seeyoufarm.com/api/count/incr/badge.svg?url=https%3A%2F%2Fgithub.com%2Fc9fe%2F22120&count_bg=%2379C83D&title_bg=%23555555&icon=&icon_color=%23E7E7E7&title=%28today%2Ftotal%29%20visitors%2B%2B%2B%20since%20Oct%2027%202020&edge_flat=false)](https://hits.seeyoufarm.com) ![version](https://img.shields.io/npm/v/archivist1) + +:floppy_disk: - an internet on yer Disk + +**DiskerNet** (codename *PROJECT 22120*) is an archivist browser controller that caches everything you browse, a library server with full text search to serve your archive. + +**Now with full text search over your archive.** + +This feature is just released in version 2 so it will improve over time. + +## And one more thing... + +**Coming to a future release, soon!**: The ability to publish your own search engine that you curated with the best resources based on your expert knowledge and experience. + +## Get it + +[Download a release](https://github.com/crisdosyago/Diskernet/releases) + +or ... + +**Get it on [npm](https://www.npmjs.com/package/diskernet):** + +```sh +$ npm i -g diskernet@latest +``` + +or... + +**Build your own binaries:** + +```sh +$ git clone https://github.com/crisdosyago/DiskerNet +$ cd DiskerNet +$ npm i +$ ./scripts/build_setup.sh +$ ./scripts/compile.sh +$ cd bin/ +``` + + +---------------- +- [Overview](#classical_building-22120---) + * [License](#license) + * [About](#about) + * [Get 22120](#get-22120) + * [Using](#using) + + [Pick save mode or serve mode](#pick-save-mode-or-serve-mode) + + [Exploring your 22120 archive](#exploring-your-22120-archive) + * [Format](#format) + * [Why not WARC (or another format like MHTML) ?](#why-not-warc-or-another-format-like-mhtml-) + * [How it works](#how-it-works) + * [FAQ](#faq) + + [Do I need to download something?](#do-i-need-to-download-something) + + [Can I use this with a browser that's not Chrome-based?](#can-i-use-this-with-a-browser-thats-not-chrome-based) + + [How does this interact with Ad blockers?](#how-does-this-interact-with-ad-blockers) + + [How secure is running chrome with remote debugging port open?](#how-secure-is-running-chrome-with-remote-debugging-port-open) + + [Is this free?](#is-this-free) + + [What if it can't find my chrome?](#what-if-it-cant-find-my-chrome) + + [What's the roadmap?](#whats-the-roadmap) + + [What about streaming content?](#what-about-streaming-content) + + [Can I black list domains to not archive them?](#can-i-black-list-domains-to-not-archive-them) + + [Is there a DEBUG mode for troubleshooting?](#is-there-a-debug-mode-for-troubleshooting) + + [Can I version the archive?](#can-i-version-the-archive) + + [Can I change the archive path?](#can-i-change-the-archive-path) + + [Can I change this other thing?](#can-i-change-this-other-thing) + +------------------ + +## License + +22120 is licensed under Polyform Strict License 1.0.0 (no modification, no distribution). You can purchase a license for different uses below: + + +- for personal, research, noncommercial purposes: +[Buy a Perpetual Non-commercial Use License of the current Version re-upped Monthly to the Latest Version, USD$1.99 per month](https://buy.stripe.com/fZeg0a45zdz58U028z) [Read license](https://github.com/DOSYCORPS/polyform-licenses/blob/1.0.0/PolyForm-Noncommercial-1.0.0.md) +- for part of your internal tooling in your org: [Buy a Perpetual Internal Use License of the current Version re-upped Monthly to the Latest Version, USD $12.99 per month](https://buy.stripe.com/00g4hsgSlbqXb288wY) [Read license](https://github.com/DOSYCORPS/polyform-licenses/blob/1.0.0/PolyForm-Internal-Use-1.0.0.md) +- for anywhere in your business: [Buy a Perpetual Small-medium Business License of the current Version re-upped Monthly to the Latest Version, USD $99 per month](https://buy.stripe.com/aEUbJUgSl2UreekdRj) [Read license](https://github.com/DOSYCORPS/polyform-licenses/blob/1.0.0/PolyForm-Small-Business-1.0.0.md) + +

Top

+ +## About + +**This project literally makes your web browsing available COMPLETELY OFFLINE.** Your browser does not even know the difference. It's literally that amazing. Yes. + +Save your browsing, then switch off the net and go to `http://localhost:22120` and switch mode to **serve** then browse what you browsed before. It all still works. + +**warning: if you have Chrome open, it will close it automatically when you open 22120, and relaunch it. You may lose any unsaved work.** + +

Top

+ +## Get 22120 + +3 ways to get it: + +1. Get binary from the [releases page.](https://github.com/c9fe/22120/releases), or +2. Run with npx: `npx diskernet@latest`, or + - `npm i -g diskernet@latest && exlibris` +3. Clone this repo and run as a Node.JS app: `npm i && npm start` + +

Top

+ +## Using + +### Pick save mode or serve mode + +Go to http://localhost:22120 in your browser, +and follow the instructions. + +

Top

+ +### Exploring your 22120 archive + +Archive will be located in `22120-arc/public/library`\* + +But it's not public, don't worry! + +You can also check out the archive index, for a listing of every title in the archive. The index is accessible from the control page, which by default is at [http://localhost:22120](http://localhost:22120) (unless you changed the port). + +\**Note:`22120-arc` is the archive root of a single archive, and by defualt it is placed in your home directory. But you can change the parent directory for `22120-arc` to have multiple archvies.* + +

Top

+ +## Format + +The archive format is: + +`22120-arc/public/library//.json` + +Inside the JSON file, is a JSON object with headers, response code, key and a base 64 encoded response body. + +

Top

+ +## Why not WARC (or another format like MHTML) ? + +**The case for the 22120 format.** + +Other formats (like MHTML and SingleFile) save translations of the resources you archive. They create modifications, such as altering the internal structure of the HTML, changing hyperlinks and URLs into "flat" embedded data URIs, or local references, and require other "hacks* in order to save a "perceptually similar" copy of the archived resource. + +22120 throws all that out, and calls rubbish on it. 22120 saves a *verbatim* **high-fidelity** copy of the resources your archive. It does not alter their internal structure in any way. Instead it records each resource in its own metadata file. In that way it is more similar to HAR and WARC, but still radically different. Compared to WARC and HAR, our format is radically simplified, throwing out most of the metadata information and unnecessary fields these formats collect. + +**Why?** + +At 22120, we believe in the resources and in verbatim copies. We don't annoint ourselves as all knowing enough to modify the resource source of truth before we archive it, just so it can "fit the format* we choose. We don't believe we need to decorate with obtuse and superfluous metadata. We don't believe we should be modifying or altering resources we archive. We belive we should save them exactly as they were presented. We believe in simplicity. We believe the format should fit (or at least accommodate, and be suited to) the resource, not the other way around. We don't believe in conflating **metadata** with **content**; so we separate them. We believe separating metadata and content, and keeping the content pure and altered throughout the archiving process is not only the right thing to do, it simplifies every part of the audit trail, because we know that the modifications between archived copies of a resource of due to changes to the resources themselves, not artefacts of the format or archiving process. + +Both SingleFile and MHTML require mutilatious modifications of the resources so that the resources can be "forced to fit" the format. At 22120, we believe this is not required (and in any case should never be performed). We see it as akin to lopping off the arms of a Roman statue in order to fit it into a presentation and security display box. How ridiculous! The web may be a more "pliable" medium but that does not mean we should treat it without respect for its inherent content. + +**Why is changing the internal structure of resources so bad?** + +In our view, the internal structure of the resource as presented, *is the cannon*. Internal structure is not just substitutable "presentation" - no, in fact it encodes vital semantic information such as hyperlink relationships, source choices, and the "strokes" of the resource author as they create their content, even if it's mediated through a web server or web framework. + +**Why else is 22120 the obvious and natural choice?** + +22120 also archives resources exactly as they are sent to the browser. It runs connected to a browser, and so is able to access the full-scope of resources (with, currently, the exception of video, audio and websockets, for now) in their highest fidelity, without modification, that the browser receives and is able to archive them in the exact format presented to the user. Many resources undergo presentational and processing changes before they are presented to the user. This is the ubiquitous, "web app", where client-side scripting enabled by JavaScript, creates resources and resource views on the fly. These sorts of "hyper resources" or "realtime" or "client side" resources, prevalent in SPAs, are not able to be archived, at least not utilizing the normal archive flow, within traditional `wget`-based archiving tools. + +In short, the web is an *online* medium, and it should be archived and presented in the same fashion. 22120 archives content exactly as it is received and presented by a browser, and it also replays that content exactly as if the resource were being taken from online. Yes, it requires a browser for this exercise, but that browser need not be connected to the internet. It is only natural that viewing a web resource requires the web browser. And because of 22120 the browser doesn't know the difference! Resources presented to the browser form a remote web site, and resources given to the browser by 22120, are seen by the browser as ***exactly the same.*** This ensures that the people viewing the archive are also not let down and are given the change to have the exact same experience as if they were viewing the resource online. + +

Top

+ +## How it works + +Uses DevTools protocol to intercept all requests, and caches responses against a key made of (METHOD and URL) onto disk. It also maintains an in memory set of keys so it knows what it has on disk. + +

Top

+ +## FAQ + +### Do I need to download something? + +Yes. But....If you like **22120**, you might love the clientless hosted version coming in future. You'll be able to build your archives online from any device, without any download, then download the archive to run on any desktop. You'll need to sign up to use it, but you can jump the queue and sign up [today](https://dosyago.com). + +### Can I use this with a browser that's not Chrome-based? + +No. + +

Top

+ +### How does this interact with Ad blockers? + +Interacts just fine. The things ad blockers stop will not be archived. + +

Top

+ +### How secure is running chrome with remote debugging port open? + +Seems pretty secure. It's not exposed to the public internet, and pages you load that tried to use it cannot use the protocol for anything (except to open a new tab, which they can do anyway). It seems there's a potential risk from malicious browser extensions, but we'd need to confirm that and if that's so, work out blocks. See [this useful security related post](https://github.com/c9fe/22120/issues/67) for some info. + +

Top

+ +### Is this free? + +Yes this is totally free to download and use for personal non-commercial use. If you want to modify or distribute it, or use it commercially (either internally or for customer functions) you need to purchase a [Noncommercial, internal use, or SMB license](#license). + +

Top

+ +### What if it can't find my chrome? + +See this useful [issue](https://github.com/c9fe/22120/issues/68). + +

Top

+ +### What's the roadmap? + +- Full text search ✅ +- Library server to serve archive publicly. +- Distributed p2p web browser on IPFS + +

Top

+ +### What about streaming content? + +The following are probably hard (and I haven't thought much about): + +- Streaming content (audio, video) +- "Impure" request response pairs (such as if you call GET /endpoint 1 time you get "A", if you call it a second time you get "AA", and other examples like this). +- WebSockets (how to capture and replay that faithfully?) + +Probably some way to do this tho. + +

Top

+ +### Can I black list domains to not archive them? + +Yes! Put any domains into `22120-arc/no.json`\*, eg: + +```json +[ + "*.horribleplantations.com", + "*.cactusfernfurniture.com", + "*.gustymeadows.com", + "*.nytimes.com", + "*.cnn.co?" +] +``` + +Will not cache any resource with a host matching those. Wildcards: + +- `*` (0 or more anything) and +- `?` (0 or 1 anything) + +\**Note: the `no` file is per-archive. `22120-arc` is the archive root of a single archive, and by defualt it is placed in your home directory. But you can change the parent directory for `22120-arc` to have multiple archvies, and each archive requires its own `no` file, if you want a blacklist in that archive.* + +

Top

+ +### Is there a DEBUG mode for troubleshooting? + +Yes, just make sure you set an environment variable called `DEBUG_22120` to anything non empty. + +So for example in posix systems: + +```bash +export DEBUG_22120=True +``` + +

Top

+ +### Can I version the archive? + +Yes! But you need to use `git` for versioning. Just initiate a git repo in your archive repository. And when you want to save a snapshot, make a new git commit. + +

Top

+ +### Can I change the archive path? + +Yes, there's a control for changing the archive path in the control page: http://localhost:22120 + +

Top

+ +### Can I change this other thing? + +There's a few command line arguments. You'll see the format printed as the first printed line when you start the program. + +For other things you can examine the source code. + +

Top

+ diff --git a/docs/SECURITY.md b/docs/SECURITY.md new file mode 100644 index 0000000..6248bf8 --- /dev/null +++ b/docs/SECURITY.md @@ -0,0 +1,17 @@ +# Security Policy + +## Supported Versions + +Use this section to tell people about which versions of your project are +currently being supported with security updates. + +| Version | Supported | +| ------- | ------------------ | +| Latest | :white_check_mark: | + + +## Reporting a Vulnerability + +To report a vulnerability, contact: cris@dosycorp.com + +To view previous responsible disclosure vulnerability reports, mediation write ups, notes and other information, please visit the [Dosyago Responsible Dislcousre Center](https://github.com/dosyago/vulnerability-reports) diff --git a/docs/features.md b/docs/features.md new file mode 100644 index 0000000..43be1b1 --- /dev/null +++ b/docs/features.md @@ -0,0 +1,4 @@ +Cool Possible Feature Ideas + +- might be nice to have historical documents indexed as well. For example. Every time we reload a page, we could add a new copy to the index, if it's different...or we could add a new copy if it's been more than X time since the last time we added it. So 1 day , or 1 week. Then we show all results in search (maybe in an expander under the main URL, like "historical URL". So you can find a result that was on front page of HN 1 year ago or 3 weeks ago, even if you revisit and reindex HN every day. + diff --git a/docs/issues b/docs/issues new file mode 100644 index 0000000..a7fcf8c --- /dev/null +++ b/docs/issues @@ -0,0 +1,12 @@ +- ndx index seems to lose documents. + - e.g. + 1. visit goog:hell + 2. visit top link: wiki - hell + 3. visit hellomagainze.com + 4. search hell + 5. see results: goog/hell, wiki/hell, hellomag + 6. reload wiki - hell + 7. search hell + 8. see results: wiki/hell, hellomag + - WHERE THE HELL DID goog/hell go? + diff --git a/docs/todo b/docs/todo new file mode 100644 index 0000000..a0c91f7 --- /dev/null +++ b/docs/todo @@ -0,0 +1,25 @@ +- complete snippet generation + - sometimes we are not getting any segments. In that case we should just show the first part of the file. + - improve trigram segmenter: lower max segment length, increase fore and aft context +- Index.json is randomly getting clobbered sometimes. Investigate and fix. Important because this breaks the whole archive. + - No idea what's causing this after an small investigation. But I've added a log on saveIndex to see when it writes. +- publish button + - way to selectively add (bookmark mode) + - way to remove (all modes) items from index +- save trigram index to disk +- let's not reindex unless we have changed contentSignature +- let's not write FTS indexes unless we have changed them since last time (UpdatedKeys) +- result paging +- We need to not open other localhosts if we already have one open +- We need to reload on localhost 22120 if we open with that + - throttle how often this can occur per URL +- search improvements + - use different min score options for different sources (noticed URL not match meghan highlight for hello mag even tho query got megan and did match and highlight queen in url) + - get snippets earlier (before rendering in lib server) and use to add to signal + - if we have multiple query terms (multiple determined by some form of tokenization) then try to show all terms present in the snippet. even tho one term may be higher scoring. Should we do multiple passes of ukkonen distance one for whole query and one for each term? This will be easier / faster with trigrams I guess. Basically we want snippet to be a relevant summary that provides signal. + - Another way to improve snippet highlight is to 'revert back' the highlighted text, and calculate their match/ukkonen on the query term. So e.g. if we get q:'israle beverly', hl:['beverly', 'beverly'], it's good overlap, but if we get hl:['is it really'] even tho that might score ok for israle, it's not a good match. so can we 'score that back' if we go match('is it really', 'israel') and see it is low, so we exclude it? + - try an exact match on the query term if possible for highlight. first one. + - we could also add signal from the highlighting to just in time alter the order (e.g. 'hell wiki' search brings google search to top rank, but the Hell wikipedia page has more highlight visible) + - Create instant search (or at least instant queries (so search over previous queries -- not results necessarily)) + - an error in Full text search can corrupt the index and make it unrecoverable...we need to guard against this + - this is still happening. sometimes the index is not saved, even on a normal error free restart. unknown why. diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..ba0e94c --- /dev/null +++ b/package-lock.json @@ -0,0 +1,2760 @@ +{ + "name": "diskernet", + "version": "2.7.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "diskernet", + "version": "2.7.1", + "license": "PolyForm Strict 1.0", + "dependencies": { + "@667/ps-list": "^1.1.3", + "chrome-launcher": "latest", + "express": "latest", + "flexsearch": "^0.7.21", + "fz-search": "^1.0.0", + "hasha": "latest", + "natural": "^5.1.11", + "ndx": "^1.0.2", + "ndx-query": "^1.0.1", + "ndx-serializable": "^1.0.0", + "node-fetch": "latest", + "ukkonen": "^1.4.0", + "ws": "latest" + }, + "bin": { + "diskernet": "build/diskernet.cjs" + }, + "devDependencies": { + "esbuild": "0.16.17", + "eslint": "^8.4.1", + "nodemon": "latest" + } + }, + "node_modules/@667/ps-list": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@667/ps-list/-/ps-list-1.1.3.tgz", + "integrity": "sha512-WQ/PkHADBTpuYqN0CRB901lc+VBbD1DW51/u2IFwRKoWWPsclqa1a0W5U0OEDvAa4kMFs+uVjbCS3k2V81OVFA==" + }, + "node_modules/@esbuild/android-arm": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.16.17.tgz", + "integrity": "sha512-N9x1CMXVhtWEAMS7pNNONyA14f71VPQN9Cnavj1XQh6T7bskqiLLrSca4O0Vr8Wdcga943eThxnVp3JLnBMYtw==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.16.17.tgz", + "integrity": "sha512-MIGl6p5sc3RDTLLkYL1MyL8BMRN4tLMRCn+yRJJmEDvYZ2M7tmAf80hx1kbNEUX2KJ50RRtxZ4JHLvCfuB6kBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.16.17.tgz", + "integrity": "sha512-a3kTv3m0Ghh4z1DaFEuEDfz3OLONKuFvI4Xqczqx4BqLyuFaFkuaG4j2MtA6fuWEFeC5x9IvqnX7drmRq/fyAQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.16.17.tgz", + "integrity": "sha512-/2agbUEfmxWHi9ARTX6OQ/KgXnOWfsNlTeLcoV7HSuSTv63E4DqtAc+2XqGw1KHxKMHGZgbVCZge7HXWX9Vn+w==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.16.17.tgz", + "integrity": "sha512-2By45OBHulkd9Svy5IOCZt376Aa2oOkiE9QWUK9fe6Tb+WDr8hXL3dpqi+DeLiMed8tVXspzsTAvd0jUl96wmg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.16.17.tgz", + "integrity": "sha512-mt+cxZe1tVx489VTb4mBAOo2aKSnJ33L9fr25JXpqQqzbUIw/yzIzi+NHwAXK2qYV1lEFp4OoVeThGjUbmWmdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.16.17.tgz", + "integrity": "sha512-8ScTdNJl5idAKjH8zGAsN7RuWcyHG3BAvMNpKOBaqqR7EbUhhVHOqXRdL7oZvz8WNHL2pr5+eIT5c65kA6NHug==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.16.17.tgz", + "integrity": "sha512-iihzrWbD4gIT7j3caMzKb/RsFFHCwqqbrbH9SqUSRrdXkXaygSZCZg1FybsZz57Ju7N/SHEgPyaR0LZ8Zbe9gQ==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.16.17.tgz", + "integrity": "sha512-7S8gJnSlqKGVJunnMCrXHU9Q8Q/tQIxk/xL8BqAP64wchPCTzuM6W3Ra8cIa1HIflAvDnNOt2jaL17vaW+1V0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.16.17.tgz", + "integrity": "sha512-kiX69+wcPAdgl3Lonh1VI7MBr16nktEvOfViszBSxygRQqSpzv7BffMKRPMFwzeJGPxcio0pdD3kYQGpqQ2SSg==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.16.17.tgz", + "integrity": "sha512-dTzNnQwembNDhd654cA4QhbS9uDdXC3TKqMJjgOWsC0yNCbpzfWoXdZvp0mY7HU6nzk5E0zpRGGx3qoQg8T2DQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.16.17.tgz", + "integrity": "sha512-ezbDkp2nDl0PfIUn0CsQ30kxfcLTlcx4Foz2kYv8qdC6ia2oX5Q3E/8m6lq84Dj/6b0FrkgD582fJMIfHhJfSw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.16.17.tgz", + "integrity": "sha512-dzS678gYD1lJsW73zrFhDApLVdM3cUF2MvAa1D8K8KtcSKdLBPP4zZSLy6LFZ0jYqQdQ29bjAHJDgz0rVbLB3g==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.16.17.tgz", + "integrity": "sha512-ylNlVsxuFjZK8DQtNUwiMskh6nT0vI7kYl/4fZgV1llP5d6+HIeL/vmmm3jpuoo8+NuXjQVZxmKuhDApK0/cKw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.16.17.tgz", + "integrity": "sha512-gzy7nUTO4UA4oZ2wAMXPNBGTzZFP7mss3aKR2hH+/4UUkCOyqmjXiKpzGrY2TlEUhbbejzXVKKGazYcQTZWA/w==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.16.17.tgz", + "integrity": "sha512-mdPjPxfnmoqhgpiEArqi4egmBAMYvaObgn4poorpUaqmvzzbvqbowRllQ+ZgzGVMGKaPkqUmPDOOFQRUFDmeUw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.16.17.tgz", + "integrity": "sha512-/PzmzD/zyAeTUsduZa32bn0ORug+Jd1EGGAUJvqfeixoEISYpGnAezN6lnJoskauoai0Jrs+XSyvDhppCPoKOA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.16.17.tgz", + "integrity": "sha512-2yaWJhvxGEz2RiftSk0UObqJa/b+rIAjnODJgv2GbGGpRwAfpgzyrg1WLK8rqA24mfZa9GvpjLcBBg8JHkoodg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.16.17.tgz", + "integrity": "sha512-xtVUiev38tN0R3g8VhRfN7Zl42YCJvyBhRKw1RJjwE1d2emWTVToPLNEQj/5Qxc6lVFATDiy6LjVHYhIPrLxzw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.16.17.tgz", + "integrity": "sha512-ga8+JqBDHY4b6fQAmOgtJJue36scANy4l/rL97W+0wYmijhxKetzZdKOJI7olaBaMhWt8Pac2McJdZLxXWUEQw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.16.17.tgz", + "integrity": "sha512-WnsKaf46uSSF/sZhwnqE4L/F89AYNMiD4YtEcYekBt9Q7nj0DiId2XH2Ng2PHM54qi5oPrQ8luuzGszqi/veig==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.16.17.tgz", + "integrity": "sha512-y+EHuSchhL7FjHgvQL/0fnnFmO4T1bhvWANX6gcnqTjtnKWbTvUMCpGnv2+t+31d7RzyEAYAd4u2fnIhHL6N/Q==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.4.1.tgz", + "integrity": "sha512-XXrH9Uarn0stsyldqDYq8r++mROmWRI1xKMXa640Bb//SY1+ECYX6VzT6Lcx5frD0V30XieqJ0oX9I2Xj5aoMA==", + "dev": true, + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^9.4.0", + "globals": "^13.19.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@humanwhocodes/config-array": { + "version": "0.11.8", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz", + "integrity": "sha512-UybHIJzJnR5Qc/MsD9Kr+RpO2h+/P1GhOwdiLPXK5TWk5sgTdu88bTD9UP+CKbPPh5Rni1u0GjAdYQLemG8g+g==", + "dev": true, + "dependencies": { + "@humanwhocodes/object-schema": "^1.2.1", + "debug": "^4.1.1", + "minimatch": "^3.0.5" + }, + "engines": { + "node": ">=10.10.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/object-schema": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz", + "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", + "dev": true + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@types/node": { + "version": "18.11.18", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.18.tgz", + "integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==" + }, + "node_modules/abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "dev": true + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/acorn": { + "version": "8.8.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz", + "integrity": "sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/afinn-165": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/afinn-165/-/afinn-165-1.0.4.tgz", + "integrity": "sha512-7+Wlx3BImrK0HiG6y3lU4xX7SpBPSSu8T9iguPMlaueRFxjbYwAQrp9lqZUuFikqKbd/en8lVREILvP2J80uJA==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/apparatus": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/apparatus/-/apparatus-0.0.10.tgz", + "integrity": "sha512-KLy/ugo33KZA7nugtQ7O0E1c8kQ52N3IvD/XgIh4w/Nr28ypfkwDfA67F1ev4N1m5D+BOk1+b2dEJDfpj/VvZg==", + "dependencies": { + "sylvester": ">= 0.0.8" + }, + "engines": { + "node": ">=0.2.6" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "node_modules/binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/body-parser": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", + "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.1", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/body-parser/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/body-parser/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "dependencies": { + "fill-range": "^7.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dependencies": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + ], + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/chokidar/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/chrome-launcher": { + "version": "0.15.1", + "resolved": "https://registry.npmjs.org/chrome-launcher/-/chrome-launcher-0.15.1.tgz", + "integrity": "sha512-UugC8u59/w2AyX5sHLZUHoxBAiSiunUhZa3zZwMH6zPVis0C3dDKiRWyUGIo14tTbZHGVviWxv3PQWZ7taZ4fg==", + "dependencies": { + "@types/node": "*", + "escape-string-regexp": "^4.0.0", + "is-wsl": "^2.2.0", + "lighthouse-logger": "^1.0.0" + }, + "bin": { + "print-chrome-path": "bin/print-chrome-path.js" + }, + "engines": { + "node": ">=12.13.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", + "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" + }, + "node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "engines": { + "node": ">= 12" + } + }, + "node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" + }, + "node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/esbuild": { + "version": "0.16.17", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.16.17.tgz", + "integrity": "sha512-G8LEkV0XzDMNwXKgM0Jwu3nY3lSTwSGY6XbxM9cr9+s0T/qSV1q1JVPBGzm3dcjhCic9+emZDmMffkwgPeOeLg==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/android-arm": "0.16.17", + "@esbuild/android-arm64": "0.16.17", + "@esbuild/android-x64": "0.16.17", + "@esbuild/darwin-arm64": "0.16.17", + "@esbuild/darwin-x64": "0.16.17", + "@esbuild/freebsd-arm64": "0.16.17", + "@esbuild/freebsd-x64": "0.16.17", + "@esbuild/linux-arm": "0.16.17", + "@esbuild/linux-arm64": "0.16.17", + "@esbuild/linux-ia32": "0.16.17", + "@esbuild/linux-loong64": "0.16.17", + "@esbuild/linux-mips64el": "0.16.17", + "@esbuild/linux-ppc64": "0.16.17", + "@esbuild/linux-riscv64": "0.16.17", + "@esbuild/linux-s390x": "0.16.17", + "@esbuild/linux-x64": "0.16.17", + "@esbuild/netbsd-x64": "0.16.17", + "@esbuild/openbsd-x64": "0.16.17", + "@esbuild/sunos-x64": "0.16.17", + "@esbuild/win32-arm64": "0.16.17", + "@esbuild/win32-ia32": "0.16.17", + "@esbuild/win32-x64": "0.16.17" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "8.31.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.31.0.tgz", + "integrity": "sha512-0tQQEVdmPZ1UtUKXjX7EMm9BlgJ08G90IhWh0PKDCb3ZLsgAOHI8fYSIzYVZej92zsgq+ft0FGsxhJ3xo2tbuA==", + "dev": true, + "dependencies": { + "@eslint/eslintrc": "^1.4.1", + "@humanwhocodes/config-array": "^0.11.8", + "@humanwhocodes/module-importer": "^1.0.1", + "@nodelib/fs.walk": "^1.2.8", + "ajv": "^6.10.0", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.3.2", + "doctrine": "^3.0.0", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^7.1.1", + "eslint-utils": "^3.0.0", + "eslint-visitor-keys": "^3.3.0", + "espree": "^9.4.0", + "esquery": "^1.4.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^6.0.1", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "globals": "^13.19.0", + "grapheme-splitter": "^1.0.4", + "ignore": "^5.2.0", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "is-path-inside": "^3.0.3", + "js-sdsl": "^4.1.4", + "js-yaml": "^4.1.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.1", + "regexpp": "^3.2.0", + "strip-ansi": "^6.0.1", + "strip-json-comments": "^3.1.0", + "text-table": "^0.2.0" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-scope": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", + "integrity": "sha512-QKQM/UXpIiHcLqJ5AOyIW7XZmzjkzQXYE54n1++wb0u9V/abW3l9uQnxX8Z5Xd18xyKIMTUAyQ0k1e8pz6LUrw==", + "dev": true, + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/eslint-utils": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz", + "integrity": "sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA==", + "dev": true, + "dependencies": { + "eslint-visitor-keys": "^2.0.0" + }, + "engines": { + "node": "^10.0.0 || ^12.0.0 || >= 14.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/mysticatea" + }, + "peerDependencies": { + "eslint": ">=5" + } + }, + "node_modules/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", + "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz", + "integrity": "sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA==", + "dev": true, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/espree": { + "version": "9.4.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.1.tgz", + "integrity": "sha512-XwctdmTO6SIvCzd9810yyNzIrOrqNYV9Koizx4C/mRhf9uq0o4yHoCEU/670pOxOL/MSraektvSAji79kX90Vg==", + "dev": true, + "dependencies": { + "acorn": "^8.8.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^3.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", + "integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==", + "dev": true, + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.18.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", + "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.1", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.5.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.2.0", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.11.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/express/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/express/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true + }, + "node_modules/fastq": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", + "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", + "dev": true, + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/file-entry-cache": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", + "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", + "dev": true, + "dependencies": { + "flat-cache": "^3.0.4" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/finalhandler": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/finalhandler/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/finalhandler/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz", + "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==", + "dev": true, + "dependencies": { + "flatted": "^3.1.0", + "rimraf": "^3.0.2" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/flatted": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.7.tgz", + "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==", + "dev": true + }, + "node_modules/flexsearch": { + "version": "0.7.31", + "resolved": "https://registry.npmjs.org/flexsearch/-/flexsearch-0.7.31.tgz", + "integrity": "sha512-XGozTsMPYkm+6b5QL3Z9wQcJjNYxp0CYn3U1gO7dwD6PAqU1SVWZxI9CCg3z+ml3YfqdPnrBehaBrnH2AGKbNA==" + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "node_modules/fz-search": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fz-search/-/fz-search-1.0.0.tgz", + "integrity": "sha512-zP5tvpXQ7JifsUZfUgySP2ZbQmFq20/R3Njw/n1+JGnu6lD7E6jbz7fFkkayfl3902KNYwZcYvN4hvAwWc5uBw==" + }, + "node_modules/get-intrinsic": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", + "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==", + "dependencies": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/globals": { + "version": "13.19.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.19.0.tgz", + "integrity": "sha512-dkQ957uSRWHw7CFXLUtUHQI3g3aWApYhfNR2O6jn/907riyTYKVBmxYVROkBcY614FSSeSJh7Xm7SrUWCxvJMQ==", + "dev": true, + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/grapheme-splitter": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz", + "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==", + "dev": true + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasha": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/hasha/-/hasha-5.2.2.tgz", + "integrity": "sha512-Hrp5vIK/xr5SkeN2onO32H0MgNZ0f17HRNH39WfL0SYUNOTZ5Lz1TJ8Pajo/87dYGEFlLMm7mIc/k/s6Bvz9HQ==", + "dependencies": { + "is-stream": "^2.0.0", + "type-fest": "^0.8.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/hasha/node_modules/type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", + "engines": { + "node": ">=8" + } + }, + "node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ignore": { + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", + "integrity": "sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/ignore-by-default": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", + "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==", + "dev": true + }, + "node_modules/import-fresh": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "dev": true, + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "dependencies": { + "is-docker": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true + }, + "node_modules/js-sdsl": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.2.0.tgz", + "integrity": "sha512-dyBIzQBDkCqCu+0upx25Y2jGdbTGxE9fshMsCdK0ViOongpV+n5tXRcZY9v7CaVQ79AGS9KA1KHtojxiM7aXSQ==", + "dev": true, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/lighthouse-logger": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/lighthouse-logger/-/lighthouse-logger-1.3.0.tgz", + "integrity": "sha512-BbqAKApLb9ywUli+0a+PcV04SyJ/N1q/8qgCNe6U97KbPCS1BTksEuHFLYdvc8DltuhfxIUBqDZsC0bBGtl3lA==", + "dependencies": { + "debug": "^2.6.9", + "marky": "^1.2.2" + } + }, + "node_modules/lighthouse-logger/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/lighthouse-logger/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true + }, + "node_modules/marky": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/marky/-/marky-1.2.5.tgz", + "integrity": "sha512-q9JtQJKjpsVxCRVgQ+WapguSbKC3SQ5HEzFGPAJMStgh3QjCawp00UKv3MTTAArTmGmmPUvllHZoNbZ3gs0I+Q==" + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/natural": { + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/natural/-/natural-5.2.4.tgz", + "integrity": "sha512-hstvCeNO63go8GMwZIO/wrmcj2oh0Wom2aN7QFF+1I7EJX/NoyM3jWKDbEoCkkQ7CCqqCn0+QdP9Bj2jkD731A==", + "dependencies": { + "afinn-165": "^1.0.2", + "apparatus": "^0.0.10", + "safe-stable-stringify": "^2.2.0", + "sylvester": "^0.0.12", + "underscore": "^1.9.1", + "wordnet-db": "^3.1.11" + }, + "engines": { + "node": ">=0.4.10" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true + }, + "node_modules/ndx": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/ndx/-/ndx-1.0.2.tgz", + "integrity": "sha512-/TbqqemJ80lGKRoRuXsz7VgA0erkIxilCUbkMfRL1h2VBGBLGvQnI+FdHvWDqJnUhgOP/T9+SYeWS84wbXGBFA==" + }, + "node_modules/ndx-query": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ndx-query/-/ndx-query-1.0.1.tgz", + "integrity": "sha512-ybm/bt2WDwDzoUDXKrqW+oHKPV9qF9E8ICqZUWZDYgPvogMZ49eaXnCJ1jP9V+bkgR98EebS7ylE1DIjwqvl4g==", + "dependencies": { + "ndx": "^1.0.2" + } + }, + "node_modules/ndx-serializable": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/ndx-serializable/-/ndx-serializable-1.0.0.tgz", + "integrity": "sha512-CViD3O8GRcWrQ2IPubwGnlmuxB81kEihjLH6SZLxUCxxL9pM6IH7RZah0SmrTuUCNx4kjiaM2S49ReaA5wiNtA==" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.0.tgz", + "integrity": "sha512-BKwRP/O0UvoMKp7GNdwPlObhYGB5DQqwhEDQlNKuoqwVYSxkSZCSbHjnFFmUEtwSKRPU4kNK8PbDYYitwaE3QA==", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/nodemon": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.20.tgz", + "integrity": "sha512-Km2mWHKKY5GzRg6i1j5OxOHQtuvVsgskLfigG25yTtbyfRGn/GNvIbRyOf1PSCKJ2aT/58TiuUsuOU5UToVViw==", + "dev": true, + "dependencies": { + "chokidar": "^3.5.2", + "debug": "^3.2.7", + "ignore-by-default": "^1.0.1", + "minimatch": "^3.1.2", + "pstree.remy": "^1.1.8", + "semver": "^5.7.1", + "simple-update-notifier": "^1.0.7", + "supports-color": "^5.5.0", + "touch": "^3.1.0", + "undefsafe": "^2.0.5" + }, + "bin": { + "nodemon": "bin/nodemon.js" + }, + "engines": { + "node": ">=8.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/nodemon" + } + }, + "node_modules/nodemon/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/nodemon/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/nodemon/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/nopt": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz", + "integrity": "sha512-NWmpvLSqUrgrAC9HCuxEvb+PSloHpqVu+FqcO4eeF2h5qYRhA7ev6KvelyQAKtegUbC6RypJnlEOhd8vloNKYg==", + "dev": true, + "dependencies": { + "abbrev": "1" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": "*" + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.12.3", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", + "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/optionator": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", + "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==", + "dev": true, + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.3" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/pstree.remy": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", + "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", + "dev": true + }, + "node_modules/punycode": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.2.0.tgz", + "integrity": "sha512-LN6QV1IJ9ZhxWTNdktaPClrNfp8xdSAYS0Zk2ddX7XsXZAxckMHPCBcHRo0cTcEIgYPRiGEkmji3Idkh2yFtYw==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", + "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/regexpp": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz", + "integrity": "sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/mysticatea" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "dev": true, + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/safe-stable-stringify": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.2.tgz", + "integrity": "sha512-gMxvPJYhP0O9n2pvcfYfIuYgbledAOJFcqRThtPRmjscaipiwcwPPKLytpVzMkG2HAN87Qmo2d4PtGiri1dSLA==", + "engines": { + "node": ">=10" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/send": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/send/node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/serve-static": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", + "dependencies": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.18.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dependencies": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/simple-update-notifier": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-1.1.0.tgz", + "integrity": "sha512-VpsrsJSUcJEseSbMHkrsrAVSdvVS5I96Qo1QAQ4FxQ9wXFcB+pjj7FB7/us9+GcgfW4ziHtYMc1J0PLczb55mg==", + "dev": true, + "dependencies": { + "semver": "~7.0.0" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/simple-update-notifier/node_modules/semver": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.0.0.tgz", + "integrity": "sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/sylvester": { + "version": "0.0.12", + "resolved": "https://registry.npmjs.org/sylvester/-/sylvester-0.0.12.tgz", + "integrity": "sha512-SzRP5LQ6Ts2G5NyAa/jg16s8e3R7rfdFjizy1zeoecYWw+nGL+YA1xZvW/+iJmidBGSdLkuvdwTYEyJEb+EiUw==", + "engines": { + "node": ">=0.2.6" + } + }, + "node_modules/text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", + "dev": true + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/touch": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz", + "integrity": "sha512-WBx8Uy5TLtOSRtIq+M03/sKDrXCLHxwDcquSP2c43Le03/9serjQBIztjRz6FkJez9D/hleyAXTBGLwwZUw9lA==", + "dev": true, + "dependencies": { + "nopt": "~1.0.10" + }, + "bin": { + "nodetouch": "bin/nodetouch.js" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ukkonen": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/ukkonen/-/ukkonen-1.4.0.tgz", + "integrity": "sha512-g8SLGxflI0/VNH2C8j66KcfJXrU5StJglRQBYPNiChXFlOrqqYM1icOykOAAUgTeBpktaEuCm9hjpPinQ080PA==" + }, + "node_modules/undefsafe": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", + "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", + "dev": true + }, + "node_modules/underscore": { + "version": "1.13.6", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", + "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", + "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", + "engines": { + "node": ">= 8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wordnet-db": { + "version": "3.1.14", + "resolved": "https://registry.npmjs.org/wordnet-db/-/wordnet-db-3.1.14.tgz", + "integrity": "sha512-zVyFsvE+mq9MCmwXUWHIcpfbrHHClZWZiVOzKSxNJruIcFn2RbY55zkhiAMMxM8zCVSmtNiViq8FsAZSFpMYag==", + "engines": { + "node": ">=0.6.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, + "node_modules/ws": { + "version": "8.12.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.12.0.tgz", + "integrity": "sha512-kU62emKIdKVeEIOIKVegvqpXMSTAMLJozpHZaJNDYqBjzlSYXQGviYwN1osDLJ9av68qHd4a2oSjd7yD4pacig==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..812b4ea --- /dev/null +++ b/package.json @@ -0,0 +1,69 @@ +{ + "name": "diskernet", + "version": "2.7.1", + "type": "module", + "description": "Library server and an archivist browser controller.", + "main": "src/app.js", + "module": "dist/diskernet.mjs", + "bin": { + "diskernet": "build/diskernet.cjs" + }, + "scripts": { + "start": "node src/app.js", + "setup": "bash ./scripts/build_setup.sh", + "build": "echo Ensure you 'npm run setup' first && bash ./scripts/compile.sh", + "compile": "npm run build", + "build-only": "bash ./scripts/build_only.sh", + "clean": "rm -rf build/* bin/*", + "super-clean": "npm run clean || : && rm -rf node_modules || : && rm package-lock.json", + "test": "nodemon src/app.js", + "inspect": "node --inspect-brk=127.0.0.1:9999 src/app.js", + "save": "nodemon src/app.js DiskerNet save", + "serve": "nodemon src/app.js DiskerNet serve", + "lint": "watch -n 5 npx eslint .", + "test-hl": "node src/highlighter.js", + "prepublishOnly": "npm run build-only" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/dosyago/DiskerNet.git" + }, + "pkg": { + "scripts": "build/**/*.js", + "assets": [ + "public/**/*", + "build/vendor/**/*" + ], + "outputPath": "bin" + }, + "keywords": [ + "archivist", + "library" + ], + "author": "@dosy", + "license": "PolyForm Strict 1.0", + "bugs": { + "url": "https://github.com/dosyago/DiskerNet/issues" + }, + "homepage": "https://github.com/dosyago/DiskerNet#readme", + "dependencies": { + "@667/ps-list": "^1.1.3", + "chrome-launcher": "latest", + "express": "latest", + "flexsearch": "^0.7.21", + "fz-search": "^1.0.0", + "hasha": "latest", + "natural": "^5.1.11", + "ndx": "^1.0.2", + "ndx-query": "^1.0.1", + "ndx-serializable": "^1.0.0", + "node-fetch": "latest", + "ukkonen": "^1.4.0", + "ws": "latest" + }, + "devDependencies": { + "esbuild": "0.16.17", + "eslint": "^8.4.1", + "nodemon": "latest" + } +} diff --git a/public/find_cleaned_duplicates.mjs b/public/find_cleaned_duplicates.mjs new file mode 100755 index 0000000..17682a3 --- /dev/null +++ b/public/find_cleaned_duplicates.mjs @@ -0,0 +1,133 @@ +#!/usr/bin/env node + +import fs from 'node:fs'; +import path from 'node:path'; +import child_process from 'node:child_process'; + +import { + loadPref, + cache_file, + index_file, +} from '../src/args.js'; + +const CLEAN = true; +const CONCURRENT = 7; +const sleep = ms => new Promise(res => setTimeout(res, ms)); +const problems = new Map(); +let cleaning = false; +let made = false; + +process.on('exit', cleanup); +process.on('SIGINT', cleanup); +process.on('SIGTERM', cleanup); +process.on('SIGHUP', cleanup); +process.on('SIGUSR2', cleanup); +process.on('beforeExit', cleanup); + +console.log({Pref:loadPref(), cache_file: cache_file(), index_file: index_file()}); +make(); + +async function make() { + const indexFile = fs.readFileSync(index_file()).toString(); + JSON.parse(indexFile).map(([key, value]) => { + if ( typeof key === "number" ) return; + if ( key.startsWith('ndx') ) return; + if ( value.title === undefined ) { + console.log('no title property', {key, value}); + } + const url = key; + const title = value.title.toLocaleLowerCase(); + if ( title.length === 0 || title.includes('404') || title.includes('not found') ) { + if ( problems.has(url) ) { + console.log('Found duplicate', url, title, problems.get(url)); + } + const prob = {title, dupes:[], dupe:false}; + problems.set(url, prob); + const cleaned1 = clean(url); + if ( problems.has(cleaned1) ) { + console.log(`Found duplicate`, {url, title, cleaned1, dupeEntry:problems.get(cleaned1)}); + prob.dupe = true; + prob.dupes.push(cleaned1); + url !== cleaned1 && (problems.delete(cleaned1), prob.diff = true); + } + const cleaned2 = clean2(url); + if ( problems.has(cleaned2) ) { + console.log(`Found duplicate`, {url, title, cleaned2, dupeEntry: problems.get(cleaned2)}); + prob.dupe = true; + prob.dupes.push(cleaned2); + url !== cleaned2 && (problems.delete(cleaned2), prob.diff = true); + } + } + }); + + made = true; + + cleanup(); +} + +function cleanup() { + if ( cleaning ) return; + if ( ! made ) return; + cleaning = true; + console.log('cleanup running'); + const outData = [...problems.entries()].filter(([key, {dupe}]) => dupe); + outData.sort(([a], [b]) => a.localeCompare(b)); + fs.writeFileSync( + path.resolve('.', 'url-cleaned-dupes.json'), + JSON.stringify(outData, null, 2) + ); + const {size:bytesWritten} = fs.statSync( + path.resolve('.', 'url-cleaned-dupes.json'), + {bigint: true} + ); + console.log(`Wrote ${outData.length} dupe urls in ${bytesWritten} bytes.`); + process.exit(0); +} + +function clean(urlString) { + const url = new URL(urlString); + if ( url.hash.startsWith('#!') || url.hostname.includes('google.com') || url.hostname.includes('80s.nyc') ) { + } else { + url.hash = ''; + } + for ( const [key, value] of url.searchParams ) { + if ( key.startsWith('utm_') ) { + url.searchParams.delete(key); + } + } + url.pathname = url.pathname.replace(/\/$/, ''); + url.protocol = 'https:'; + url.pathname = url.pathname.replace(/(\.htm.?|\.php|\.asp.?)$/, ''); + if ( url.hostname.startsWith('www.') ) { + url.hostname = url.hostname.replace(/^www./, ''); + } + const key = url.toString(); + return key; +} + +function clean2(urlString) { + const url = new URL(urlString); + url.pathname = ''; + return url.toString(); +} + +function curlCommand(url) { + return `curl -k -L -s -o /dev/null -w '%{url_effective}' ${JSON.stringify(url)} \ + -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \ + -H 'Accept-Language: en,en-US;q=0.9,zh-TW;q=0.8,zh-CN;q=0.7,zh;q=0.6,ja;q=0.5' \ + -H 'Cache-Control: no-cache' \ + -H 'Connection: keep-alive' \ + -H 'DNT: 1' \ + -H 'Pragma: no-cache' \ + -H 'Sec-Fetch-Dest: document' \ + -H 'Sec-Fetch-Mode: navigate' \ + -H 'Sec-Fetch-Site: none' \ + -H 'Sec-Fetch-User: ?1' \ + -H 'Upgrade-Insecure-Requests: 1' \ + -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36' \ + -H 'sec-ch-ua: "Chromium";v="104", " Not A;Brand";v="99", "Google Chrome";v="104"' \ + -H 'sec-ch-ua-mobile: ?0' \ + -H 'sec-ch-ua-platform: "macOS"' \ + --compressed ; + `; +} diff --git a/public/find_crawlable.mjs b/public/find_crawlable.mjs new file mode 100755 index 0000000..117c4ac --- /dev/null +++ b/public/find_crawlable.mjs @@ -0,0 +1,92 @@ +#!/usr/bin/env node + +import fs from 'node:fs'; +import path from 'node:path'; +import child_process from 'node:child_process'; + +const CLEAN = false; +const CONCURRENT = 7; +const sleep = ms => new Promise(res => setTimeout(res, ms)); +const entries = []; +let cleaning = false; + +process.on('exit', cleanup); +process.on('SIGINT', cleanup); +process.on('SIGTERM', cleanup); +process.on('SIGHUP', cleanup); +process.on('SIGUSR2', cleanup); +process.on('beforeExit', cleanup); + +make(); + +async function make() { + const titlesFile = fs.readFileSync(path.resolve('.', 'topTitles.json')).toString(); + const titles = new Map(JSON.parse(titlesFile).map(([url, title]) => [url, {url,title}])); + titles.forEach(({url,title}) => { + if ( title.length === 0 && url.startsWith('https:') && !url.endsWith('.pdf') ) { + entries.push(url); + } + }); + + cleanup(); +} + +function cleanup() { + if ( cleaning ) return; + cleaning = true; + console.log('cleanup running'); + fs.writeFileSync( + path.resolve('.', 'recrawl-https-3.json'), + JSON.stringify(entries, null, 2) + ); + console.log(`Wrote recrawlable urls`); + process.exit(0); +} + +function clean(urlString) { + const url = new URL(urlString); + if ( url.hash.startsWith('#!') || url.hostname.includes('google.com') || url.hostname.includes('80s.nyc') ) { + } else { + url.hash = ''; + } + for ( const [key, value] of url.searchParams ) { + if ( key.startsWith('utm_') ) { + url.searchParams.delete(key); + } + } + url.pathname = url.pathname.replace(/\/$/, ''); + url.protocol = 'https:'; + url.pathname = url.pathname.replace(/(\.htm.?|\.php)$/, ''); + if ( url.hostname.startsWith('www.') ) { + url.hostname = url.hostname.replace(/^www./, ''); + } + const key = url.toString(); + return key; +} + +function clean2(urlString) { + const url = new URL(urlString); + url.pathname = ''; + return url.toString(); +} + +function curlCommand(url) { + return `curl -k -L -s -o /dev/null -w '%{url_effective}' ${JSON.stringify(url)} \ + -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \ + -H 'Accept-Language: en,en-US;q=0.9,zh-TW;q=0.8,zh-CN;q=0.7,zh;q=0.6,ja;q=0.5' \ + -H 'Cache-Control: no-cache' \ + -H 'Connection: keep-alive' \ + -H 'DNT: 1' \ + -H 'Pragma: no-cache' \ + -H 'Sec-Fetch-Dest: document' \ + -H 'Sec-Fetch-Mode: navigate' \ + -H 'Sec-Fetch-Site: none' \ + -H 'Sec-Fetch-User: ?1' \ + -H 'Upgrade-Insecure-Requests: 1' \ + -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36' \ + -H 'sec-ch-ua: "Chromium";v="104", " Not A;Brand";v="99", "Google Chrome";v="104"' \ + -H 'sec-ch-ua-mobile: ?0' \ + -H 'sec-ch-ua-platform: "macOS"' \ + --compressed ; + `; +} diff --git a/public/index.html b/public/index.html new file mode 100644 index 0000000..f91d953 --- /dev/null +++ b/public/index.html @@ -0,0 +1,243 @@ + + +Your HTML Library + +
+

22120 — Internet Offline Library

+
+

+ View the index +

+ +
+
+ Search your archive + + +
+
+
+
+ Save or Serve: Mode Control +

+ Control whether pages you browse are , or + your archive +
+ Pro-Tip: Serve pages when you're offline, and it will still feel like you're online +

+ + + + +

+ + +

+
+
+
+ File system path of archive +

+ Set the path to where your archive folder will go +
+ The default is your home directory +

+ +

+ + +

+
+
+
+ Publish your archive +

+ Publish a search engine from your archive +
+ This will generate a server.zip file that you can unzip and run +

+ +

+
+ diff --git a/public/injection.js b/public/injection.js new file mode 100644 index 0000000..910090b --- /dev/null +++ b/public/injection.js @@ -0,0 +1,195 @@ +import {DEBUG as debug} from '../src/common.js'; + +const DEBUG = debug || false; + +export function getInjection({sessionId}) { + // Notes: + // say() function + // why aliased? Resistant to page overwriting + // just a precaution as we are already in an isolated world here, but this makes + // this script more portable if it were introduced globally as well as robust + // against API or behaviour changes of the browser or its remote debugging protocol + // in future + return ` + { + const X = 1; + const DEBUG = ${JSON.stringify(DEBUG, null, 2)}; + const MIN_CHECK_TEXT = 3000; // min time between checking documentElement.innerText + const MIN_NOTIFY = 5000; // min time between telling controller text maybe changed + const MAX_NOTIFICATIONS = 13; // max times we will tell controller text maybe changed + const OBSERVER_OPTS = { + subtree: true, + childList: true, + characterData: true + }; + const Top = globalThis.top; + let lastInnerText; + + if ( Top === globalThis ) { + const ConsoleInfo = console.info.bind(console); + const JSONStringify = JSON.stringify.bind(JSON); + const TITLE_CHANGES = 10; + const INITIAL_CHECK_TIME = 500; + const TIME_MULTIPLIER = Math.E; + const sessionId = "${sessionId}"; + const sleep = ms => new Promise(res => setTimeout(res, ms)); + const handler = throttle(handleFrameMessage, MIN_NOTIFY); + let count = 0; + + installTop(); + + async function installTop() { + console.log("Installing in top frame..."); + self.startUrl = location.href; + say({install: { sessionId, startUrl }}); + await sleep(500); + beginTitleChecks(); + beginTextNotifications(); + console.log("Installed."); + } + + function beginTitleChecks() { + let lastTitle = null; + let checker; + let timeToNextCheck = INITIAL_CHECK_TIME; + let changesLogged = 0; + + check(); + console.log('Begun logging title changes.'); + + function check() { + clearTimeout(checker); + const currentTitle = document.title; + if ( lastTitle !== currentTitle ) { + say({titleChange: {lastTitle, currentTitle, url: location.href, sessionId}}); + lastTitle = currentTitle; + changesLogged++; + } else { + // increase check time if there's no change + timeToNextCheck *= TIME_MULTIPLIER; + } + if ( changesLogged < TITLE_CHANGES ) { + checker = setTimeout(check, timeToNextCheck); + } else { + console.log('Finished logging title changes.'); + } + } + } + + function say(thing) { + ConsoleInfo(JSONStringify(thing)); + } + + function beginTextNotifications() { + // listen for {textChange:true} messages + // throttle them + // on leading throttle edge send message to controller with + // console.info(JSON.stringify({textChange:...})); + self.addEventListener('message', messageParser); + + console.log('Begun notifying of text changes.'); + + function messageParser({data, origin}) { + let source; + try { + ({source} = data.frameTextChangeNotification); + if ( count > MAX_NOTIFICATIONS ) { + self.removeEventListener('message', messageParser); + return; + } + count++; + handler({textChange:{source}}); + } catch(e) { + DEBUG.verboseSlow && console.warn('could not parse message', data, e); + } + } + } + + function handleFrameMessage({textChange}) { + const {source} = textChange; + console.log('Telling controller that text changed'); + say({textChange:{source, sessionId, count}}); + } + } + + beginTextMutationChecks(); + + function beginTextMutationChecks() { + // create mutation observer for text + // throttle output + + const observer = new MutationObserver(throttle(check, MIN_CHECK_TEXT)); + observer.observe(document.documentElement || document, OBSERVER_OPTS); + + console.log('Begun observing text changes.'); + + function check() { + console.log('check'); + const textMutated = document.documentElement.innerText !== lastInnerText; + if ( textMutated ) { + DEBUG.verboseSlow && console.log('Text changed'); + lastInnerText = document.documentElement.innerText; + Top.postMessage({frameTextChangeNotification:{source:location.href}}, '*'); + } + } + } + + // javascript throttle function + // source: https://stackoverflow.com/a/59378445 + /* + function throttle(func, timeFrame) { + var lastTime = 0; + return function (...args) { + var now = new Date(); + if (now - lastTime >= timeFrame) { + func.apply(this, args); + lastTime = now; + } + }; + } + */ + + // alternate throttle function with trailing edge call + // source: https://stackoverflow.com/a/27078401 + ///* + // Notes + // Returns a function, that, when invoked, will only be triggered at most once + // during a given window of time. Normally, the throttled function will run + // as much as it can, without ever going more than once per \`wait\` duration; + // but if you'd like to disable the execution on the leading edge, pass + // \`{leading: false}\`. To disable execution on the trailing edge, ditto. + function throttle(func, wait, options) { + var context, args, result; + var timeout = null; + var previous = 0; + if (!options) options = {}; + var later = function() { + previous = options.leading === false ? 0 : Date.now(); + timeout = null; + result = func.apply(context, args); + if (!timeout) context = args = null; + }; + return function() { + var now = Date.now(); + if (!previous && options.leading === false) previous = now; + var remaining = wait - (now - previous); + context = this; + args = arguments; + if (remaining <= 0 || remaining > wait) { + if (timeout) { + clearTimeout(timeout); + timeout = null; + } + previous = now; + result = func.apply(context, args); + if (!timeout) context = args = null; + } else if (!timeout && options.trailing !== false) { + timeout = setTimeout(later, remaining); + } + return result; + }; + } + //*/ + } + `; +} diff --git a/public/library/README.md b/public/library/README.md new file mode 100644 index 0000000..977a8ab --- /dev/null +++ b/public/library/README.md @@ -0,0 +1,10 @@ +# ALT Default storage directory for library + +Remove `public/library/http*` and `public/library/cache.json` from `.gitignore` if you forked this repo and want to commit your library using git. + +## Clearing your cache + +To clear everything, delete all directories that start with `http` or `https` and delete cache.json + +To clear only stuff from domains you don't want, delete all directories you don't want that start with `http` or `https` and DON'T delete cache.json + diff --git a/public/make_top.mjs b/public/make_top.mjs new file mode 100755 index 0000000..b7eb40a --- /dev/null +++ b/public/make_top.mjs @@ -0,0 +1,250 @@ +#!/usr/bin/env node + +import fs from 'node:fs'; +import path from 'node:path'; +import child_process from 'node:child_process'; + +const CLEAN = false; +const CONCURRENT = 7; +const sleep = ms => new Promise(res => setTimeout(res, ms)); +const entries = []; +const counted = new Set(); +const errors = new Map(); +let counts; +let cleaning = false; + +process.on('exit', cleanup); +process.on('SIGINT', cleanup); +process.on('SIGTERM', cleanup); +process.on('SIGHUP', cleanup); +process.on('SIGUSR2', cleanup); +process.on('beforeExit', cleanup); + +make(); + +async function make() { + const titlesFile = fs.readFileSync(path.resolve('.', 'topTitles.json')).toString(); + const titles = new Map(JSON.parse(titlesFile).map(([url, title]) => [url, {url,title}])); + if ( CLEAN ) { + for ( const [url, obj] of titles ) { + const k1 = clean(url); + const k2 = clean2(url); + if ( !titles.has(k1) ) { + titles.set(k1, obj); + } + if ( !titles.has(k2) ) { + titles.set(k2, obj); + } + } + } + const remainingFile = fs.readFileSync(path.resolve('.', 'remainingFile.json')).toString(); + const remainingSet = new Set(JSON.parse(remainingFile)); + const countsFile = fs.readFileSync(path.resolve('.', 'ran-counts.json')).toString(); + counts = new Map(JSON.parse(countsFile).filter(([url, count]) => remainingSet.has(url))); + let current = 0; + for ( const [url, count] of counts ) { + let title; + let realUrl; + if ( titles.has(url) ) { + ({title} = titles.get(url)); + entries.push({ + url, + title, + count, + }); + counted.add(url); + } else { + console.log(`Curl call for ${url} in progress...`); + let notifyCurlComplete; + const curlCall = new Promise(res => notifyCurlComplete = res); + do { + await sleep(1000); + } while ( current >= CONCURRENT ); + child_process.exec(curlCommand(url), (err, stdout, stderr) => { + if ( ! err && (!stderr || stderr.length == 0)) { + realUrl = stdout; + if ( titles.has(realUrl) ) { + ({title} = titles.get(realUrl)); + entries.push({ + url, + realUrl, + title, + count, + }); + counted.add(url); + } + } else { + console.log(`Error on curl for ${url}`, {err, stderr}); + errors.set(url, {err, stderr}); + } + console.log(`Curl call for ${url} complete!`); + notifyCurlComplete(); + }); + current += 1; + curlCall.then(() => current -= 1); + } + } + cleanup(); +} + +async function make_v2() { + const titlesFile = fs.readFileSync(path.resolve('.', 'topTitles.json')).toString(); + const titles = new Map(JSON.parse(titlesFile).map(([url, title]) => [url, {url,title}])); + if ( CLEAN ) { + for ( const [url, obj] of titles ) { + const k1 = clean(url); + const k2 = clean2(url); + if ( !titles.has(k1) ) { + titles.set(k1, obj); + } + if ( !titles.has(k2) ) { + titles.set(k2, obj); + } + } + } + const countsFile = fs.readFileSync(path.resolve('.', 'ran-counts.json')).toString(); + counts = new Map(JSON.parse(countsFile)); + let current = 0; + for ( const [url, count] of counts ) { + let title; + let realUrl; + if ( titles.has(url) ) { + ({title} = titles.get(url)); + entries.push({ + url, + title, + count, + }); + counted.add(url); + } else { + console.log(`Curl call for ${url} in progress...`); + let notifyCurlComplete; + const curlCall = new Promise(res => notifyCurlComplete = res); + do { + await sleep(250); + } while ( current >= CONCURRENT ); + child_process.exec(curlCommand(url), (err, stdout, stderr) => { + if ( ! err && (!stderr || stderr.length == 0)) { + realUrl = stdout; + if ( titles.has(realUrl) ) { + ({title} = titles.get(realUrl)); + entries.push({ + url, + realUrl, + title, + count, + }); + counted.add(url); + } + } else { + console.log(`Error on curl for ${url}`, {err, stderr}); + errors.set(url, {err, stderr}); + } + console.log(`Curl call for ${url} complete!`); + notifyCurlComplete(); + }); + current += 1; + curlCall.then(() => current -= 1); + } + } + cleanup(); +} + +function cleanup() { + if ( cleaning ) return; + cleaning = true; + console.log('cleanup running'); + if ( errors.size ) { + fs.writeFileSync( + path.resolve('.', 'errorLinks4.json'), + JSON.stringify([...errors.keys()], null, 2) + ); + console.log(`Wrote errors`); + } + if ( counted.size !== counts.size ) { + counted.forEach(url => counts.delete(url)); + fs.writeFileSync( + path.resolve('.', 'noTitleFound4.json'), + JSON.stringify([...counts.keys()], null, 2) + ) + console.log(`Wrote noTitleFound`); + } + fs.writeFileSync( + path.resolve('.', 'topFrontPageLinksWithCounts4.json'), + JSON.stringify(entries, null, 2) + ); + console.log(`Wrote top links with counts`); + process.exit(0); +} + +async function make_v1() { + const titlesFile = fs.readFileSync(path.resolve('.', 'topTitles.json')).toString(); + const titles = new Map(JSON.parse(titlesFile).map(([url, title]) => [clean(url), {url,title}])); + const countsFile = fs.readFileSync(path.resolve('.', 'counts.json')).toString(); + const counts = new Map(JSON.parse(countsFile).map(([url, count]) => [clean(url), count])); + for ( const [key, count] of counts ) { + counts.set(clean2(key), count); + } + const entries = []; + for ( const [key, {url,title}] of titles ) { + entries.push({ + url, title, + count: counts.get(key) || + counts.get(url) || + counts.get(clean2(key)) || + console.log(`No count found for`, {key, url, title, c2key: clean2(key)}) + }); + } + fs.writeFileSync( + path.resolve('.', 'topFrontPageLinks.json'), + JSON.stringify(entries, null, 2) + ); +} + +function clean(urlString) { + const url = new URL(urlString); + if ( url.hash.startsWith('#!') || url.hostname.includes('google.com') || url.hostname.includes('80s.nyc') ) { + } else { + url.hash = ''; + } + for ( const [key, value] of url.searchParams ) { + if ( key.startsWith('utm_') ) { + url.searchParams.delete(key); + } + } + url.pathname = url.pathname.replace(/\/$/, ''); + url.protocol = 'https:'; + url.pathname = url.pathname.replace(/(\.htm.?|\.php)$/, ''); + if ( url.hostname.startsWith('www.') ) { + url.hostname = url.hostname.replace(/^www./, ''); + } + const key = url.toString(); + return key; +} + +function clean2(urlString) { + const url = new URL(urlString); + url.pathname = ''; + return url.toString(); +} + +function curlCommand(url) { + return `curl -k -L -s -o /dev/null -w '%{url_effective}' ${JSON.stringify(url)} \ + -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \ + -H 'Accept-Language: en,en-US;q=0.9,zh-TW;q=0.8,zh-CN;q=0.7,zh;q=0.6,ja;q=0.5' \ + -H 'Cache-Control: no-cache' \ + -H 'Connection: keep-alive' \ + -H 'DNT: 1' \ + -H 'Pragma: no-cache' \ + -H 'Sec-Fetch-Dest: document' \ + -H 'Sec-Fetch-Mode: navigate' \ + -H 'Sec-Fetch-Site: none' \ + -H 'Sec-Fetch-User: ?1' \ + -H 'Upgrade-Insecure-Requests: 1' \ + -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36' \ + -H 'sec-ch-ua: "Chromium";v="104", " Not A;Brand";v="99", "Google Chrome";v="104"' \ + -H 'sec-ch-ua-mobile: ?0' \ + -H 'sec-ch-ua-platform: "macOS"' \ + --compressed ; + `; +} diff --git a/public/none b/public/none new file mode 100644 index 0000000..6a9376f --- /dev/null +++ b/public/none @@ -0,0 +1,1440 @@ +No count found for { + key: 'https://tesla.com/modelx', + url: 'https://www.tesla.com/modelx', + title: 'Model X | Tesla', + c2key: 'https://tesla.com/' +} +No count found for { + key: 'https://web.tecgraf.puc-rio.br/~lhf/ftp/doc/hopl.pdf', + url: 'https://web.tecgraf.puc-rio.br/~lhf/ftp/doc/hopl.pdf', + title: '', + c2key: 'https://web.tecgraf.puc-rio.br/' +} +No count found for { + key: 'https://upstart.bizjournals.comnews-markets/national-news/portfolio/2008/01/14/Media-Defenders-Profile', + url: 'http://upstart.bizjournals.comnews-markets/national-news/portfolio/2008/01/14/Media-Defenders-Profile', + title: '', + c2key: 'https://upstart.bizjournals.comnews-markets/' +} +No count found for { + key: 'https://playframework.com/', + url: 'https://www.playframework.com/', + title: 'Play Framework - Build Modern & Scalable Web Apps with Java and Scala', + c2key: 'https://playframework.com/' +} +No count found for { + key: 'https://grobmeier.solutions/the-10-rules-of-a-zen-programmer-03022012', + url: 'https://grobmeier.solutions/the-10-rules-of-a-zen-programmer-03022012.html', + title: 'The 10 rules of a Zen programmer', + c2key: 'https://grobmeier.solutions/' +} +No count found for { + key: 'https://facebook.com/login?next=https%3A%2F%2Fwww.facebook.com%2Fprofile.php%3Fid%3D9445547199%26sk%3Dnotes', + url: 'https://www.facebook.com/login/?next=https%3A%2F%2Fwww.facebook.com%2Fprofile.php%3Fid%3D9445547199%26sk%3Dnotes', + title: 'Log into Facebook', + c2key: 'https://facebook.com/?next=https%3A%2F%2Fwww.facebook.com%2Fprofile.php%3Fid%3D9445547199%26sk%3Dnotes' +} +No count found for { + key: 'https://adssettings.google.com/authenticated?hl=en', + url: 'https://adssettings.google.com/authenticated?hl=en', + title: 'Ad Settings', + c2key: 'https://adssettings.google.com/?hl=en' +} +No count found for { + key: 'https://worldbank.org/en/programs/business-enabling-environment', + url: 'https://www.worldbank.org/en/programs/business-enabling-environment', + title: 'Business Enabling Environment', + c2key: 'https://worldbank.org/' +} +No count found for { + key: 'https://designit.com/journal/2012/08/the-best-interface-is-no-interface', + url: 'https://www.designit.com/journal/2012/08/the-best-interface-is-no-interface.html', + title: 'Designit – Designit', + c2key: 'https://designit.com/' +} +No count found for { + key: 'https://simcast.com/?d=arcsynthesis.org&pcid=802&rid=152&a=1', + url: 'https://simcast.com/?d=arcsynthesis.org&pcid=802&rid=152&a=1', + title: 'Simcast News portal.', + c2key: 'https://simcast.com/?d=arcsynthesis.org&pcid=802&rid=152&a=1' +} +No count found for { + key: 'https://boards.4channel.org/news?all', + url: 'https://boards.4channel.org/news/?all', + title: '/news/ - Current News - 4chan', + c2key: 'https://boards.4channel.org/?all' +} +No count found for { + key: 'https://home.work.caltech.edu/library', + url: 'https://home.work.caltech.edu/library/', + title: 'Machine Learning Video Library - Learning From Data (Abu-Mostafa)', + c2key: 'https://home.work.caltech.edu/' +} +No count found for { + key: 'https://random.waxy.org/arsdigita', + url: 'https://random.waxy.org/arsdigita/', + title: 'ArsDigita: From Start-Up to Bust-Up', + c2key: 'https://random.waxy.org/' +} +No count found for { + key: 'https://www1.telehash.org/?tm=1&subid4=1661767972.0187130000&kw=mesh+network&KW1=Full%20Mesh%20Network&KW2=Private%20Cloud%20Network&KW3=Online%20Text%20Messaging%20System&searchbox=0&domainname=0&backfill=0', + url: 'http://www1.telehash.org/?tm=1&subid4=1661767972.0187130000&kw=mesh+network&KW1=Full%20Mesh%20Network&KW2=Private%20Cloud%20Network&KW3=Online%20Text%20Messaging%20System&searchbox=0&domainname=0&backfill=0', + title: 'telehash.org', + c2key: 'https://www1.telehash.org/?tm=1&subid4=1661767972.0187130000&kw=mesh+network&KW1=Full%20Mesh%20Network&KW2=Private%20Cloud%20Network&KW3=Online%20Text%20Messaging%20System&searchbox=0&domainname=0&backfill=0' +} +No count found for { + key: 'https://tesla.com/blog/secret-tesla-motors-master-plan-just-between-you-and-me', + url: 'https://www.tesla.com/blog/secret-tesla-motors-master-plan-just-between-you-and-me', + title: 'The Secret Tesla Motors Master Plan (just between you and me) | Tesla', + c2key: 'https://tesla.com/' +} +No count found for { + key: 'https://jamstack.org/generators', + url: 'https://jamstack.org/generators/', + title: 'Static Site Generators - Top Open Source SSGs | Jamstack', + c2key: 'https://jamstack.org/' +} +No count found for { + key: 'https://scottaaronson.blog/?p=4229', + url: 'https://scottaaronson.blog/?p=4229', + title: 'Shtetl-Optimized » Blog Archive » Sensitivity Conjecture resolved', + c2key: 'https://scottaaronson.blog/?p=4229' +} +No count found for { + key: 'https://newscientist.com/article/dn25723-massive-ocean-discovered-towards-earths-core?ignored=irrelevant', + url: 'https://www.newscientist.com/article/dn25723-massive-ocean-discovered-towards-earths-core/?ignored=irrelevant', + title: "Massive 'ocean' discovered towards Earth's core | New Scientist", + c2key: 'https://newscientist.com/?ignored=irrelevant' +} +No count found for { + key: 'https://vulkan.org/', + url: 'https://www.vulkan.org/', + title: 'Home | Vulkan | Cross platform 3D Graphics', + c2key: 'https://vulkan.org/' +} +No count found for { + key: 'https://michaelfranz.com/', + url: 'http://www.michaelfranz.com/', + title: 'Home Page of Professor Michael Franz, University of California, Irvine', + c2key: 'https://michaelfranz.com/' +} +No count found for { + key: 'https://gamedeveloper.com/design/how-i-used-eve-online-to-predict-the-great-recession', + url: 'https://www.gamedeveloper.com/design/how-i-used-eve-online-to-predict-the-great-recession', + title: 'How I Used EVE Online to Predict the Great Recession', + c2key: 'https://gamedeveloper.com/' +} +No count found for { + key: 'https://forbes.com/sites/victoriabarret/2011/10/18/dropbox-the-inside-story-of-techs-hottest-startup?sh=17ca4956437b', + url: 'https://www.forbes.com/sites/victoriabarret/2011/10/18/dropbox-the-inside-story-of-techs-hottest-startup/?sh=17ca4956437b', + title: "Dropbox: The Inside Story Of Tech's Hottest Startup", + c2key: 'https://forbes.com/?sh=17ca4956437b' +} +No count found for { + key: 'https://forbes.com/sites/cameronkeng/2014/06/22/employees-that-stay-in-companies-longer-than-2-years-get-paid-50-less?sh=b8cf086e07fa', + url: 'https://www.forbes.com/sites/cameronkeng/2014/06/22/employees-that-stay-in-companies-longer-than-2-years-get-paid-50-less/?sh=b8cf086e07fa', + title: 'Employees Who Stay In Companies Longer Than Two Years Get Paid 50% Less', + c2key: 'https://forbes.com/?sh=b8cf086e07fa' +} +No count found for { + key: 'https://exurbe.com/the-shape-of-rome', + url: 'https://www.exurbe.com/the-shape-of-rome/', + title: 'The Shape of Rome – Ex Urbe', + c2key: 'https://exurbe.com/' +} +No count found for { + key: 'https://ecb.torontomu.ca/~elf/hack/recovery', + url: 'https://www.ecb.torontomu.ca/~elf/hack/recovery.html', + title: '', + c2key: 'https://ecb.torontomu.ca/' +} +No count found for { + key: 'https://dropbox.com/login?cont=https%3A%2F%2Fwww.dropbox.com%2Fteam', + url: 'https://www.dropbox.com/login?cont=https%3A%2F%2Fwww.dropbox.com%2Fteam', + title: 'Login - Dropbox', + c2key: 'https://dropbox.com/?cont=https%3A%2F%2Fwww.dropbox.com%2Fteam' +} +No count found for { + key: 'https://buzzfeednews.com/article/peteraldhous/spies-in-the-skies', + url: 'https://www.buzzfeednews.com/article/peteraldhous/spies-in-the-skies', + title: 'See Maps Showing Where FBI Planes Are Watching From Above', + c2key: 'https://buzzfeednews.com/' +} +No count found for { + key: 'https://techtldr.com/google-may-be-stealing-your-mobile-traffic', + url: 'https://techtldr.com/google-may-be-stealing-your-mobile-traffic/', + title: 'Google May Be Stealing Your Mobile Traffic - Tech TLDR;', + c2key: 'https://techtldr.com/' +} +No count found for { + key: 'https://archief.ntr.nl/tuinderlusten/en', + url: 'https://archief.ntr.nl/tuinderlusten/en.html', + title: 'Jheronimus Bosch - The Garden of Earthly Delights', + c2key: 'https://archief.ntr.nl/' +} +No count found for { + key: 'https://rfc-editor.org/rfc/rfc1925', + url: 'https://www.rfc-editor.org/rfc/rfc1925', + title: 'RFC 1925: The Twelve Networking Truths', + c2key: 'https://rfc-editor.org/' +} +No count found for { + key: 'https://rfc-editor.org/rfc/rfc7258', + url: 'https://www.rfc-editor.org/rfc/rfc7258', + title: 'RFC 7258: Pervasive Monitoring Is an Attack', + c2key: 'https://rfc-editor.org/' +} +No count found for { + key: 'https://docs.microsoft.com/en-us/previous-versions/technet-magazine/cc565089(v=msdn.10)?redirectedfrom=MSDN', + url: 'https://docs.microsoft.com/en-us/previous-versions/technet-magazine/cc565089(v=msdn.10)?redirectedfrom=MSDN', + title: 'Raymond Chen discusses Microsoft Bob | Microsoft Docs', + c2key: 'https://docs.microsoft.com/?redirectedfrom=MSDN' +} +No count found for { + key: 'https://susam.net/blog/lisp-in-vim', + url: 'https://susam.net/blog/lisp-in-vim.html', + title: 'Lisp in Vim - Susam Pal', + c2key: 'https://susam.net/' +} +No count found for { + key: 'chrome-error://chromewebdata', + url: 'chrome-error://chromewebdata/', + title: 'www.pawfal.org', + c2key: 'chrome-error://chromewebdata' +} +No count found for { + key: 'https://pinpayments.com/', + url: 'https://pinpayments.com/', + title: 'Secure online payments that work for you | Online payments | Pin Payments', + c2key: 'https://pinpayments.com/' +} +No count found for { + key: 'https://mega.io/?nz=1', + url: 'https://mega.io/?nz=1', + title: '', + c2key: 'https://mega.io/?nz=1' +} +No count found for { + key: 'https://mega.io/', + url: 'https://mega.io/', + title: 'The Most Trusted, Best-Protected Cloud Storage - MEGA', + c2key: 'https://mega.io/' +} +No count found for { + key: 'https://cantorsparadise.com/the-riemann-hypothesis-explained-fa01c1f75d3f?gi=e8129d9b771', + url: 'https://www.cantorsparadise.com/the-riemann-hypothesis-explained-fa01c1f75d3f?gi=e8129d9b771', + title: 'The Riemann Hypothesis, explained | by Jørgen Veisdal | Cantor’s Paradise', + c2key: 'https://cantorsparadise.com/?gi=e8129d9b771' +} +No count found for { + key: 'https://sive.rs/zipit', + url: 'https://sive.rs/zipit', + title: 'Announcing your plans makes you less motivated to accomplish them | Derek Sivers', + c2key: 'https://sive.rs/' +} +No count found for { + key: 'https://sive.rs/mistake', + url: 'https://sive.rs/mistake', + title: 'My $3.3M mistake | Derek Sivers', + c2key: 'https://sive.rs/' +} +No count found for { + key: 'https://blog.shiftyjelly.com/2011/08/02/amazon-app-store-rotten-to-the-core', + url: 'https://blog.shiftyjelly.com/2011/08/02/amazon-app-store-rotten-to-the-core/', + title: "Amazon App Store: Rotten To The Core | Shifty Jelly's blog of mystery", + c2key: 'https://blog.shiftyjelly.com/' +} +No count found for { + key: 'https://theintercept.com/', + url: 'https://theintercept.com/', + title: 'The Intercept', + c2key: 'https://theintercept.com/' +} +No count found for { + key: 'https://scobleizer.blog/2008/07/26/the-silicon-valley-vc-disease', + url: 'https://scobleizer.blog/2008/07/26/the-silicon-valley-vc-disease/', + title: 'The Silicon Valley VC Disease – Scobleizer', + c2key: 'https://scobleizer.blog/' +} +No count found for { + key: 'https://engineering.fb.com/2014/02/20/web/an-analysis-of-facebook-photo-caching', + url: 'https://engineering.fb.com/2014/02/20/web/an-analysis-of-facebook-photo-caching/', + title: 'An analysis of Facebook photo caching - Engineering at Meta', + c2key: 'https://engineering.fb.com/' +} +No count found for { + key: 'https://archive.org/details/byte-magazine-1983-08/1983_08_BYTE_08-08_The_C_Language?view=theater', + url: 'https://archive.org/details/byte-magazine-1983-08/1983_08_BYTE_08-08_The_C_Language?view=theater', + title: 'Byte Magazine Volume 08 Number 08 - The C Language : Free Download, Borrow, and Streaming : Internet Archive', + c2key: 'https://archive.org/?view=theater' +} +No count found for { + key: 'https://archive.org/details/byte-magazine-1983-08/mode/2up?view=theater', + url: 'https://archive.org/details/byte-magazine-1983-08/mode/2up?view=theater', + title: 'Byte Magazine Volume 08 Number 08 - The C Language : Free Download, Borrow, and Streaming : Internet Archive', + c2key: 'https://archive.org/?view=theater' +} +No count found for { + key: 'https://rjlipton.wpcomstaging.com/2010/08/11/deolalikar-responds-to-issues-about-his-p%E2%89%A0np-proof', + url: 'https://rjlipton.wpcomstaging.com/2010/08/11/deolalikar-responds-to-issues-about-his-p%E2%89%A0np-proof/', + title: "Deolalikar Responds To Issues About His P≠NP Proof | Gödel's Lost Letter and P=NP", + c2key: 'https://rjlipton.wpcomstaging.com/' +} +No count found for { + key: 'https://replit.com/', + url: 'https://replit.com/', + title: 'The collaborative browser based IDE - Replit', + c2key: 'https://replit.com/' +} +No count found for { + key: 'https://microsoft.com/en-us/research/people/simonpj?from=https%3A%2F%2Fresearch.microsoft.com%2Fen-us%2Fum%2Fpeople%2Fsimonpj%2Fpapers%2Fgiving-a-talk%2Fwriting-a-paper-slides.pdf', + url: 'https://www.microsoft.com/en-us/research/people/simonpj/?from=https%3A%2F%2Fresearch.microsoft.com%2Fen-us%2Fum%2Fpeople%2Fsimonpj%2Fpapers%2Fgiving-a-talk%2Fwriting-a-paper-slides.pdf', + title: 'Simon Peyton Jones at Microsoft Research', + c2key: 'https://microsoft.com/?from=https%3A%2F%2Fresearch.microsoft.com%2Fen-us%2Fum%2Fpeople%2Fsimonpj%2Fpapers%2Fgiving-a-talk%2Fwriting-a-paper-slides.pdf' +} +No count found for { + key: 'https://dev.realworldocaml.org/', + url: 'https://dev.realworldocaml.org/', + title: 'Real World OCaml', + c2key: 'https://dev.realworldocaml.org/' +} +No count found for { + key: 'https://archive.nytimes.com/open.blogs.nytimes.com/2015/04/09/extracting-structured-data-from-recipes-using-conditional-random-fields', + url: 'https://archive.nytimes.com/open.blogs.nytimes.com/2015/04/09/extracting-structured-data-from-recipes-using-conditional-random-fields/', + title: 'Extracting Structured Data From Recipes Using Conditional Random Fields - The New York Times', + c2key: 'https://archive.nytimes.com/' +} +No count found for { + key: 'https://archive.nytimes.com/open.blogs.nytimes.com/2015/04/09/extracting-structured-data-from-recipes-using-conditional-random-fields?mtrref=undefined&gwh=5E043617DAA0EB411C6FFFB46E9EACEF&gwt=pay&assetType=PAYWALL', + url: 'https://archive.nytimes.com/open.blogs.nytimes.com/2015/04/09/extracting-structured-data-from-recipes-using-conditional-random-fields/?mtrref=undefined&gwh=5E043617DAA0EB411C6FFFB46E9EACEF&gwt=pay&assetType=PAYWALL', + title: 'Extracting Structured Data From Recipes Using Conditional Random Fields - The New York Times', + c2key: 'https://archive.nytimes.com/?mtrref=undefined&gwh=5E043617DAA0EB411C6FFFB46E9EACEF&gwt=pay&assetType=PAYWALL' +} +No count found for { + key: 'https://news.stanford.edu/news/2005/june15/jobs-061505', + url: 'https://news.stanford.edu/news/2005/june15/jobs-061505.html', + title: "Text of Steve Jobs' Commencement address (2005)", + c2key: 'https://news.stanford.edu/' +} +No count found for { + key: 'https://domainr.com/hacks/csshttprequest', + url: 'https://domainr.com/hacks/csshttprequest', + title: 'Not Found · Domainr', + c2key: 'https://domainr.com/' +} +No count found for { + key: 'https://vice.com/en/article/xygykj/my-year-in-san-franciscos-2-million-secret-society-startup', + url: 'https://www.vice.com/en/article/xygykj/my-year-in-san-franciscos-2-million-secret-society-startup', + title: 'My Year in San Francisco’s $2 Million Secret Society Startup', + c2key: 'https://vice.com/' +} +No count found for { + key: 'https://mosh.org/', + url: 'https://mosh.org/', + title: 'Mosh: the mobile shell', + c2key: 'https://mosh.org/' +} +No count found for { + key: 'https://color.adobe.com/create', + url: 'https://color.adobe.com/create', + title: '', + c2key: 'https://color.adobe.com/' +} +No count found for { + key: 'https://color.adobe.com/create/color-wheel', + url: 'https://color.adobe.com/create/color-wheel', + title: 'Color wheel, a color palette generator | Adobe Color', + c2key: 'https://color.adobe.com/' +} +No count found for { + key: 'https://webamp.org/', + url: 'https://webamp.org/', + title: 'Webamp · Winamp 2 in your browser', + c2key: 'https://webamp.org/' +} +No count found for { + key: 'https://two.js.org/', + url: 'https://two.js.org/', + title: 'Two.js • Homepage', + c2key: 'https://two.js.org/' +} +No count found for { + key: 'https://archive.jlongster.com/How-I-Became-Better-Programmer', + url: 'https://archive.jlongster.com/How-I-Became-Better-Programmer', + title: 'How I Became a Better Programmer', + c2key: 'https://archive.jlongster.com/' +} +No count found for { + key: 'https://websitepolicies.com/?utm_campaign=redirect', + url: 'https://www.websitepolicies.com/?utm_medium=redirect&utm_campaign=redirect&utm_source=headjs.com', + title: 'WebsitePolicies: Compliance Solutions for Online Businesses', + c2key: 'https://websitepolicies.com/?utm_campaign=redirect' +} +No count found for { + key: 'https://haskell.org/platform', + url: 'https://www.haskell.org/platform/', + title: 'Haskell Platform', + c2key: 'https://haskell.org/' +} +No count found for { + key: 'https://go.dev/doc/articles/wiki', + url: 'https://go.dev/doc/articles/wiki/', + title: 'Writing Web Applications - The Go Programming Language', + c2key: 'https://go.dev/' +} +No count found for { + key: 'https://github.blog/2009-10-20-how-we-made-github-fast', + url: 'https://github.blog/2009-10-20-how-we-made-github-fast/', + title: 'How We Made GitHub Fast | The GitHub Blog', + c2key: 'https://github.blog/' +} +No count found for { + key: 'https://gamedeveloper.com/programming/in-depth-functional-programming-in-c-', + url: 'https://www.gamedeveloper.com/programming/in-depth-functional-programming-in-c-', + title: 'In-depth: Functional programming in C++', + c2key: 'https://gamedeveloper.com/' +} +No count found for { + key: 'https://flutter.dev/', + url: 'https://flutter.dev/', + title: 'Flutter - Build apps for any screen', + c2key: 'https://flutter.dev/' +} +No count found for { + key: 'https://orbitalquark.github.io/textadept', + url: 'https://orbitalquark.github.io/textadept/', + title: 'Textadept', + c2key: 'https://orbitalquark.github.io/' +} +No count found for { + key: 'https://faculty.econ.ucsb.edu/~doug', + url: 'https://faculty.econ.ucsb.edu/~doug/', + title: '', + c2key: 'https://faculty.econ.ucsb.edu/' +} +No count found for { + key: 'https://seriouseats.com/challenges-of-opening-a-brewery-job-advice-beer-industry-collin-mcdonnell-henhouse', + url: 'https://www.seriouseats.com/challenges-of-opening-a-brewery-job-advice-beer-industry-collin-mcdonnell-henhouse', + title: 'So You Think You Want to Open a Brewery...', + c2key: 'https://seriouseats.com/' +} +No count found for { + key: 'https://manybutfinite.com/post/intel-cpu-caches', + url: 'https://manybutfinite.com/post/intel-cpu-caches/', + title: 'Cache: a place for concealment and safekeeping | Many But Finite', + c2key: 'https://manybutfinite.com/' +} +No count found for { + key: 'https://berthub.eu/articles/posts/amazing-dna', + url: 'https://berthub.eu/articles/posts/amazing-dna/', + title: "DNA seen through the eyes of a coder (or, If you are a hammer, everything looks like a nail) - Bert Hubert's writings", + c2key: 'https://berthub.eu/' +} +No count found for { + key: 'https://cxl.com/blog/pricing-experiments-you-might-not-know-but-can-learn-from', + url: 'https://cxl.com/blog/pricing-experiments-you-might-not-know-but-can-learn-from/', + title: 'Pricing Experiments You Might Not Know, But Can Learn From', + c2key: 'https://cxl.com/' +} +No count found for { + key: 'https://namecheap.com/blog/2011/12/26/godaddy-transfer-update', + url: 'https://www.namecheap.com/blog/2011/12/26/godaddy-transfer-update', + title: 'Page not found - Namecheap Blog', + c2key: 'https://namecheap.com/' +} +No count found for { + key: 'https://parenscript.common-lisp.dev/', + url: 'https://parenscript.common-lisp.dev/', + title: 'Parenscript', + c2key: 'https://parenscript.common-lisp.dev/' +} +No count found for { + key: 'https://developer.android.com/?csw=1', + url: 'https://developer.android.com/?csw=1', + title: 'Android Mobile App Developer Tools – Android Developers', + c2key: 'https://developer.android.com/?csw=1' +} +No count found for { + key: 'https://cbea.ms/git-commit', + url: 'https://cbea.ms/git-commit/', + title: 'How to Write a Git Commit Message', + c2key: 'https://cbea.ms/' +} +No count found for { + key: 'https://docs.microsoft.com/en-us/events/pdc/pdc-1996/pdc-1996-keynote-with-bob-muglia-and-steve-jobs', + url: 'https://docs.microsoft.com/en-us/events/pdc/pdc-1996/pdc-1996-keynote-with-bob-muglia-and-steve-jobs', + title: '404 - Content Not Found | Microsoft Docs', + c2key: 'https://docs.microsoft.com/' +} +No count found for { + key: 'https://silktide.com/', + url: 'https://silktide.com/', + title: 'Silktide - Measure and improve your websites', + c2key: 'https://silktide.com/' +} +No count found for { + key: 'https://freshdesk.com/general/the-freshdesk-story-blog', + url: 'https://freshdesk.com/general/the-freshdesk-story-blog/', + title: 'The Freshdesk Story of Where and How it All Started', + c2key: 'https://freshdesk.com/' +} +No count found for { + key: 'https://fastmail.blog/open-technologies/jmap-a-better-way-to-email', + url: 'https://fastmail.blog/open-technologies/jmap-a-better-way-to-email/', + title: 'Dec 23: JMAP — A better way to email', + c2key: 'https://fastmail.blog/' +} +No count found for { + key: 'https://typeof.net/Iosevka', + url: 'https://typeof.net/Iosevka/', + title: 'Iosevka', + c2key: 'https://typeof.net/' +} +No count found for { + key: 'https://facebook.com/login?next=https%3A%2F%2Fapps.facebook.com%2Fimessenger', + url: 'https://www.facebook.com/login/?next=https%3A%2F%2Fapps.facebook.com%2Fimessenger', + title: 'Log into Facebook', + c2key: 'https://facebook.com/?next=https%3A%2F%2Fapps.facebook.com%2Fimessenger' +} +No count found for { + key: 'https://statmodeling.stat.columbia.edu/2012/11/16808', + url: 'https://statmodeling.stat.columbia.edu/2012/11/16808', + title: 'Page not found | Statistical Modeling, Causal Inference, and Social Science', + c2key: 'https://statmodeling.stat.columbia.edu/' +} +No count found for { + key: 'https://alumnit.ca/', + url: 'http://alumnit.ca/', + title: 'The Lumnit – Resource focused on open source', + c2key: 'https://alumnit.ca/' +} +No count found for { + key: 'https://patterns.dev/posts/classic-design-patterns', + url: 'https://www.patterns.dev/posts/classic-design-patterns/', + title: 'Learning JavaScript Design Patterns', + c2key: 'https://patterns.dev/' +} +No count found for { + key: 'https://signalvnoise.com/posts/3024-questions-i-ask-when-reviewing-a-design', + url: 'https://signalvnoise.com/posts/3024-questions-i-ask-when-reviewing-a-design', + title: 'Questions I ask when reviewing a design – Signal v. Noise', + c2key: 'https://signalvnoise.com/' +} +No count found for { + key: 'https://signalvnoise.com/archives2/dont_scale_99999_uptime_is_for_walmart.php', + url: 'https://signalvnoise.com/archives2/dont_scale_99999_uptime_is_for_walmart.php', + title: "Don't scale: 99.999% uptime is for Wal-Mart - Signal vs. Noise (by 37signals)", + c2key: 'https://signalvnoise.com/' +} +No count found for { + key: 'https://techtarget.com/', + url: 'https://www.techtarget.com/', + title: 'Purchase Intent Data for Enterprise Tech Sales and Marketing - TechTarget', + c2key: 'https://techtarget.com/' +} +No count found for { + key: 'https://cardinalalumni.stanford.edu/get/page/magazine/article?article_id=41260', + url: 'https://cardinalalumni.stanford.edu/get/page/magazine/article?article_id=41260', + title: 'Stanford Magazine - Article', + c2key: 'https://cardinalalumni.stanford.edu/?article_id=41260' +} +No count found for { + key: 'https://stanfordmag.org/contents/how-my-start-up-failed', + url: 'https://stanfordmag.org/contents/how-my-start-up-failed', + title: 'How My Start-Up Failed | STANFORD magazine', + c2key: 'https://stanfordmag.org/' +} +No count found for { + key: 'https://apps.ankiweb.net/', + url: 'https://apps.ankiweb.net/', + title: 'Anki - powerful, intelligent flashcards', + c2key: 'https://apps.ankiweb.net/' +} +No count found for { + key: 'https://ww1.apirocks.com/', + url: 'http://ww1.apirocks.com/', + title: '', + c2key: 'https://ww1.apirocks.com/' +} +No count found for { + key: 'https://login.microsoftonline.com/44467e6f-462c-4ea2-823f-7800de5434e3/saml2?SAMLRequest=fVLLbtswEPwVgXeJMqVYEmE5cGMUNZA2RqT0kEtAU8uYAB8uH2n796VfQXrJeWdndmZ2cftHq%2BwNnJfW9GhWlCgDw%2B0kzWuPnsaveYtulwvPtCIHuophbx7hVwQfsrRoPD1PehSdoZZ56alhGjwNnA6r7%2FeUFCU9OBsstwplK%2B%2FBhSR1Z42PGtwA7k1yeHq879E%2BhIOnGPu93O2sgrAvpFLSWOkLmCKW0wEnKiEV4NMl%2BKhA8PZhGPEwPKBsne6ShoWTlyudsq%2FSFFpyZ70VwZpECQW3Gtd1PW9gLvJ6TnheAyN5SyqRN21ZTnBTVzVU%2BGQQZZt1j16abiY6UrXzqROiK1kjGJlueCeasiG7qkkw7yNsjA%2FMhB6RkpC8bHPSjTNCSUlnKY6uekbZ9hLJF2nOUX%2BW3%2B4M8vTbOG7zo12U%2FbxWlgDoUhA9qbuPzXxOzK51oOURphkHKlP7WltDo4z8mPsCfyR%2F%2F4UfiW2z3lol%2Bd9spZT9feeABehRcBEQXl72%2Fv%2Ba5T8%3D&RelayState=e1s1', + url: 'https://login.microsoftonline.com/44467e6f-462c-4ea2-823f-7800de5434e3/saml2?SAMLRequest=fVLLbtswEPwVgXeJMqVYEmE5cGMUNZA2RqT0kEtAU8uYAB8uH2n796VfQXrJeWdndmZ2cftHq%2BwNnJfW9GhWlCgDw%2B0kzWuPnsaveYtulwvPtCIHuophbx7hVwQfsrRoPD1PehSdoZZ56alhGjwNnA6r7%2FeUFCU9OBsstwplK%2B%2FBhSR1Z42PGtwA7k1yeHq879E%2BhIOnGPu93O2sgrAvpFLSWOkLmCKW0wEnKiEV4NMl%2BKhA8PZhGPEwPKBsne6ShoWTlyudsq%2FSFFpyZ70VwZpECQW3Gtd1PW9gLvJ6TnheAyN5SyqRN21ZTnBTVzVU%2BGQQZZt1j16abiY6UrXzqROiK1kjGJlueCeasiG7qkkw7yNsjA%2FMhB6RkpC8bHPSjTNCSUlnKY6uekbZ9hLJF2nOUX%2BW3%2B4M8vTbOG7zo12U%2FbxWlgDoUhA9qbuPzXxOzK51oOURphkHKlP7WltDo4z8mPsCfyR%2F%2F4UfiW2z3lol%2Bd9spZT9feeABehRcBEQXl72%2Fv%2Ba5T8%3D&RelayState=e1s1', + title: 'Sign in to your account', + c2key: 'https://login.microsoftonline.com/?SAMLRequest=fVLLbtswEPwVgXeJMqVYEmE5cGMUNZA2RqT0kEtAU8uYAB8uH2n796VfQXrJeWdndmZ2cftHq%2BwNnJfW9GhWlCgDw%2B0kzWuPnsaveYtulwvPtCIHuophbx7hVwQfsrRoPD1PehSdoZZ56alhGjwNnA6r7%2FeUFCU9OBsstwplK%2B%2FBhSR1Z42PGtwA7k1yeHq879E%2BhIOnGPu93O2sgrAvpFLSWOkLmCKW0wEnKiEV4NMl%2BKhA8PZhGPEwPKBsne6ShoWTlyudsq%2FSFFpyZ70VwZpECQW3Gtd1PW9gLvJ6TnheAyN5SyqRN21ZTnBTVzVU%2BGQQZZt1j16abiY6UrXzqROiK1kjGJlueCeasiG7qkkw7yNsjA%2FMhB6RkpC8bHPSjTNCSUlnKY6uekbZ9hLJF2nOUX%2BW3%2B4M8vTbOG7zo12U%2FbxWlgDoUhA9qbuPzXxOzK51oOURphkHKlP7WltDo4z8mPsCfyR%2F%2F4UfiW2z3lol%2Bd9spZT9feeABehRcBEQXl72%2Fv%2Ba5T8%3D&RelayState=e1s1' +} +No count found for { + key: 'https://sound-effects.bbcrewind.co.uk/', + url: 'https://sound-effects.bbcrewind.co.uk/', + title: 'BBC Rewind - Sound Effects', + c2key: 'https://sound-effects.bbcrewind.co.uk/' +} +No count found for { + key: 'https://adamnash.blog/2011/10/10/steve-jobs-bmw-ebay', + url: 'https://adamnash.blog/2011/10/10/steve-jobs-bmw-ebay/', + title: 'Steve Jobs, BMW & eBay | Psychohistory', + c2key: 'https://adamnash.blog/' +} +No count found for { + key: 'https://docker.com/blog', + url: 'https://www.docker.com/blog/', + title: 'Docker Blog - Docker', + c2key: 'https://docker.com/' +} +No count found for { + key: 'https://booked.net/objectmentor', + url: 'https://www.booked.net/objectmentor', + title: 'Vacation Rental Hacks', + c2key: 'https://booked.net/' +} +No count found for { + key: 'https://blog.parse.ly/1691lucene', + url: 'https://blog.parse.ly/1691lucene', + title: 'Page not found | Parse.ly', + c2key: 'https://blog.parse.ly/' +} +No count found for { + key: 'https://blogs.harvard.edu/sj/2011/07/24/aaron-swartz-v-united-states', + url: 'http://blogs.harvard.edu/sj/2011/07/24/aaron-swartz-v-united-states/', + title: 'The Longest Now', + c2key: 'https://blogs.harvard.edu/' +} +No count found for { + key: 'https://devblogs.microsoft.com/ericlippert/what-would-feynman-do.aspx', + url: 'https://devblogs.microsoft.com/ericlippert/what-would-feynman-do.aspx', + title: '403 Forbidden', + c2key: 'https://devblogs.microsoft.com/' +} +No count found for { + key: 'https://devblogs.microsoft.com/jw_on_tech/why-i-left-google.aspx', + url: 'https://devblogs.microsoft.com/jw_on_tech/why-i-left-google.aspx', + title: '403 Forbidden', + c2key: 'https://devblogs.microsoft.com/' +} +No count found for { + key: 'https://devblogs.microsoft.com/ntdebugging/understanding-arm-assembly-part-1.aspx', + url: 'https://devblogs.microsoft.com/ntdebugging/understanding-arm-assembly-part-1.aspx', + title: '403 Forbidden', + c2key: 'https://devblogs.microsoft.com/' +} +No count found for { + key: 'https://devblogs.microsoft.com/oldnewthing/10378851.aspx', + url: 'https://devblogs.microsoft.com/oldnewthing/10378851.aspx', + title: '403 Forbidden', + c2key: 'https://devblogs.microsoft.com/' +} +No count found for { + key: 'https://nuxeo.com/blog/speeding-up-the-android-emulator', + url: 'https://www.nuxeo.com/blog/speeding-up-the-android-emulator.html', + title: '404 | Nuxeo', + c2key: 'https://nuxeo.com/' +} +No count found for { + key: 'https://wsj.com/articles/BL-232B-2715', + url: 'https://www.wsj.com/articles/BL-232B-2715', + title: 'Jessica Livingston: Why Startups Need to Focus on Sales, Not Marketing - WSJ', + c2key: 'https://wsj.com/' +} +No count found for { + key: 'https://cloud.google.com/appengine/docs/legacy/standard/python/python25?csw=1', + url: 'https://cloud.google.com/appengine/docs/legacy/standard/python/python25?csw=1', + title: 'Python 2.5  |  App Engine standard environment for Python 2  |  Google Cloud', + c2key: 'https://cloud.google.com/?csw=1' +} +No count found for { + key: 'https://mcmillen.dev/language_checklist', + url: 'https://www.mcmillen.dev/language_checklist.html', + title: 'Programming Language Checklist | Colin McMillen', + c2key: 'https://mcmillen.dev/' +} +No count found for { + key: 'https://toptal.com/designers/colourcode', + url: 'https://www.toptal.com/designers/colourcode', + title: 'ColourCode: Color Palette Generator | Toptal®', + c2key: 'https://toptal.com/' +} +No count found for { + key: 'https://noteapp.com/', + url: 'https://noteapp.com/', + title: 'NoteApp - Simple, Collaborative Notetaking | https://noteapp.com', + c2key: 'https://noteapp.com/' +} +No count found for { + key: 'https://deceptive.design/', + url: 'https://www.deceptive.design/', + title: 'Deceptive Design - user interfaces crafted to trick you', + c2key: 'https://deceptive.design/' +} +No count found for { + key: 'https://docker.com/', + url: 'https://www.docker.com/', + title: 'Home - Docker', + c2key: 'https://docker.com/' +} +No count found for { + key: 'https://orcid.org/signin', + url: 'https://orcid.org/signin', + title: 'ORCID', + c2key: 'https://orcid.org/' +} +No count found for { + key: 'https://johnresig.com/blog/ocr-and-neural-nets-in-javascript', + url: 'https://johnresig.com/blog/ocr-and-neural-nets-in-javascript/', + title: 'John Resig - OCR and Neural Nets in JavaScript', + c2key: 'https://johnresig.com/' +} +No count found for { + key: 'https://johnresig.com/blog/how-javascript-timers-work', + url: 'https://johnresig.com/blog/how-javascript-timers-work/', + title: 'John Resig - How JavaScript Timers Work', + c2key: 'https://johnresig.com/' +} +No count found for { + key: 'https://steinway.com/misc/etude', + url: 'https://www.steinway.com/misc/etude', + title: 'Etude - Steinway & Sons', + c2key: 'https://steinway.com/' +} +No count found for { + key: 'https://simcast.com/?d=flexboxin5.com&pcid=802&rid=152&a=1', + url: 'https://simcast.com/?d=flexboxin5.com&pcid=802&rid=152&a=1', + title: 'Simcast News portal.', + c2key: 'https://simcast.com/?d=flexboxin5.com&pcid=802&rid=152&a=1' +} +No count found for { + key: 'https://freakonomics.com/2009/10/do-we-need-a-37-cent-coin', + url: 'https://freakonomics.com/2009/10/do-we-need-a-37-cent-coin/', + title: 'Do We Need a 37-Cent Coin? - Freakonomics', + c2key: 'https://freakonomics.com/' +} +No count found for { + key: 'https://services.github.com/', + url: 'https://services.github.com/', + title: 'GitHub Professional Services | From idea to implementation, our experts are ready to help your team get wherever you want to go. Start a conversation with us about how we can bring your goals to life.', + c2key: 'https://services.github.com/' +} +No count found for { + key: 'https://ai.googleblog.com/2006/06/extra-extra-read-all-about-it-nearly', + url: 'https://ai.googleblog.com/2006/06/extra-extra-read-all-about-it-nearly.html', + title: 'Google AI Blog: Extra, Extra - Read All About It: Nearly All Binary Searches and Mergesorts are Broken', + c2key: 'https://ai.googleblog.com/' +} +No count found for { + key: 'https://haldean.org/vim-problems', + url: 'https://haldean.org/vim-problems/', + title: '', + c2key: 'https://haldean.org/' +} +No count found for { + key: 'https://wiki.hackerspaces.org/List_of_Hacker_Spaces', + url: 'https://wiki.hackerspaces.org/List_of_Hacker_Spaces', + title: 'List of Hacker Spaces - HackerspaceWiki', + c2key: 'https://wiki.hackerspaces.org/' +} +No count found for { + key: 'https://icps.icube.unistra.fr/img_auth.php/d/db/ModernC.pdf', + url: 'https://icps.icube.unistra.fr/img_auth.php/d/db/ModernC.pdf', + title: 'Access denied', + c2key: 'https://icps.icube.unistra.fr/' +} +No count found for { + key: 'https://instagram-engineering.com/what-powers-instagram-hundreds-of-instances-dozens-of-technologies-adf2e22da2ad?gi=106df3be0fc4', + url: 'https://instagram-engineering.com/what-powers-instagram-hundreds-of-instances-dozens-of-technologies-adf2e22da2ad?gi=106df3be0fc4', + title: 'What Powers Instagram: Hundreds of Instances, Dozens of Technologies | by Instagram Engineering | Instagram Engineering', + c2key: 'https://instagram-engineering.com/?gi=106df3be0fc4' +} +No count found for { + key: 'https://instagram-engineering.com/what-powers-instagram-hundreds-of-instances-dozens-of-technologies-adf2e22da2ad', + url: 'https://instagram-engineering.com/what-powers-instagram-hundreds-of-instances-dozens-of-technologies-adf2e22da2ad', + title: 'What Powers Instagram: Hundreds of Instances, Dozens of Technologies | by Instagram Engineering | Instagram Engineering', + c2key: 'https://instagram-engineering.com/' +} +No count found for { + key: 'https://runestone.academy/runestone/static/pythonds/index', + url: 'https://runestone.academy/runestone/static/pythonds/index.html', + title: '404 Not Found', + c2key: 'https://runestone.academy/' +} +No count found for { + key: 'https://ipfs.tech/', + url: 'https://ipfs.tech/', + title: '', + c2key: 'https://ipfs.tech/' +} +No count found for { + key: 'https://crockford.com/little', + url: 'https://www.crockford.com/little.html', + title: 'The Little JavaScripter', + c2key: 'https://crockford.com/' +} +No count found for { + key: 'https://julesjacobs.com/2015/08/17/bayesian-scoring-of-ratings', + url: 'https://julesjacobs.com/2015/08/17/bayesian-scoring-of-ratings.html', + title: 'Bayesian ranking of items with up and downvotes or 5 star ratings', + c2key: 'https://julesjacobs.com/' +} +No count found for { + key: 'https://kotlinlang.org/', + url: 'https://kotlinlang.org/', + title: 'Kotlin Programming Language', + c2key: 'https://kotlinlang.org/' +} +No count found for { + key: 'https://blog.loveconquersallgames.com/post/2350461718/fuck-the-super-game-boy-introduction', + url: 'https://blog.loveconquersallgames.com/post/2350461718/fuck-the-super-game-boy-introduction', + title: 'Fuck the Super Game Boy: Introduction', + c2key: 'https://blog.loveconquersallgames.com/' +} +No count found for { + key: 'https://wordpress.com/typo?subdomain=mattmaroon', + url: 'https://wordpress.com/typo/?subdomain=mattmaroon', + title: 'WordPress.com', + c2key: 'https://wordpress.com/?subdomain=mattmaroon' +} +No count found for { + key: 'https://zoom.us/j/542614774', + url: 'https://zoom.us/j/542614774#success', + title: 'Launch Meeting - Zoom', + c2key: 'https://zoom.us/' +} +No count found for { + key: 'https://blogs.opera.com/news', + url: 'https://blogs.opera.com/news/', + title: 'The Opera Blog - News | Opera', + c2key: 'https://blogs.opera.com/' +} +No count found for { + key: 'https://f5.com/company/news/press-releases', + url: 'https://www.f5.com/company/news/press-releases', + title: 'Press Releases', + c2key: 'https://f5.com/' +} +No count found for { + key: 'https://ww38.nlpwp.org/book', + url: 'http://ww38.nlpwp.org/book', + title: 'nlpwp.org', + c2key: 'https://ww38.nlpwp.org/' +} +No count found for { + key: 'https://wsj.com/articles/SB124648494429082661', + url: 'https://www.wsj.com/articles/SB124648494429082661', + title: 'Two Centuries On, a Cryptologist Cracks a Presidential Code - WSJ', + c2key: 'https://wsj.com/' +} +No count found for { + key: 'https://wsj.com/articles/SB121124460502305693', + url: 'https://www.wsj.com/articles/SB121124460502305693', + title: "You Can't Soak the Rich - WSJ", + c2key: 'https://wsj.com/' +} +No count found for { + key: 'https://wsj.com/articles/SB125875892887958111', + url: 'https://www.wsj.com/articles/SB125875892887958111', + title: 'The Henry Ford of Heart Surgery - WSJ', + c2key: 'https://wsj.com/' +} +No count found for { + key: 'https://draft.blogger.com/blogin.g?blogspotURL=https://paultyma.blogspot.com/2007/03/howto-pass-silicon-valley-software.html&type=blog', + url: 'https://draft.blogger.com/blogin.g?blogspotURL=https://paultyma.blogspot.com/2007/03/howto-pass-silicon-valley-software.html&type=blog', + title: 'Permission denied', + c2key: 'https://draft.blogger.com/?blogspotURL=https://paultyma.blogspot.com/2007/03/howto-pass-silicon-valley-software.html&type=blog' +} +No count found for { + key: 'https://beyondgrep.com/', + url: 'https://beyondgrep.com/', + title: 'Beyond grep: ack v3.6.0', + c2key: 'https://beyondgrep.com/' +} +No count found for { + key: 'https://taorminahotels.org/refactormycodecom', + url: 'http://www.taorminahotels.org/refactormycodecom/', + title: 'Recent codes - RefactorMyCode.com', + c2key: 'https://taorminahotels.org/' +} +No count found for { + key: 'https://blog.bradfieldcs.com/you-are-not-google-84912cf44afb?gi=9ecc3e5c11eb', + url: 'https://blog.bradfieldcs.com/you-are-not-google-84912cf44afb?gi=9ecc3e5c11eb', + title: 'You Are Not Google. Software engineers go crazy for the… | by Oz Nova | Bradfield', + c2key: 'https://blog.bradfieldcs.com/?gi=9ecc3e5c11eb' +} +No count found for { + key: 'https://abevoelker.com/2019-03-06/on-the-death-of-my-familys-dairy-farm', + url: 'https://abevoelker.com/2019-03-06/on-the-death-of-my-familys-dairy-farm/', + title: 'On the death of my family’s dairy farm - Abe Voelker', + c2key: 'https://abevoelker.com/' +} +No count found for { + key: 'https://discord.com/blog', + url: 'https://discord.com/blog', + title: 'Discord Blog', + c2key: 'https://discord.com/' +} +No count found for { + key: 'https://engineering.fb.com/2014/01/07/core-data/scaling-mercurial-at-facebook', + url: 'https://engineering.fb.com/2014/01/07/core-data/scaling-mercurial-at-facebook/', + title: 'Scaling Mercurial at Facebook - Engineering at Meta', + c2key: 'https://engineering.fb.com/' +} +No count found for { + key: 'https://web.dev/', + url: 'https://web.dev/', + title: 'web.dev', + c2key: 'https://web.dev/' +} +No count found for { + key: 'https://education.github.com/', + url: 'https://education.github.com/', + title: 'Engaged students are the result of using real-world tools - GitHub Education', + c2key: 'https://education.github.com/' +} +No count found for { + key: 'https://paper.dropbox.com/hackpad', + url: 'https://paper.dropbox.com/hackpad/', + title: 'Dropbox Paper', + c2key: 'https://paper.dropbox.com/' +} +No count found for { + key: 'https://sive.rs/itunes', + url: 'https://sive.rs/itunes', + title: 'The day Steve Jobs dissed me in a keynote | Derek Sivers', + c2key: 'https://sive.rs/' +} +No count found for { + key: 'https://sive.rs/kimo', + url: 'https://sive.rs/kimo', + title: 'There’s no speed limit | Derek Sivers', + c2key: 'https://sive.rs/' +} +No count found for { + key: 'https://jmeiners.com/lc3-vm', + url: 'https://www.jmeiners.com/lc3-vm/', + title: 'Write your Own Virtual Machine', + c2key: 'https://jmeiners.com/' +} +No count found for { + key: 'https://objective-see.org/products/lulu', + url: 'https://objective-see.org/products/lulu.html', + title: 'Objective-See: LuLu', + c2key: 'https://objective-see.org/' +} +No count found for { + key: 'https://pippinbarr.com/itisasifyouweredoingwork', + url: 'https://pippinbarr.com/itisasifyouweredoingwork/', + title: 'It is as if you were doing work', + c2key: 'https://pippinbarr.com/' +} +No count found for { + key: 'https://hugedomains.com/domain_profile.cfm?d=spottedsun.com', + url: 'https://www.hugedomains.com/domain_profile.cfm?d=spottedsun.com', + title: 'SpottedSun.com is for sale | HugeDomains', + c2key: 'https://hugedomains.com/?d=spottedsun.com' +} +No count found for { + key: 'https://ytdl-org.github.io/youtube-dl/index', + url: 'https://ytdl-org.github.io/youtube-dl/index.html', + title: 'youtube-dl', + c2key: 'https://ytdl-org.github.io/' +} +No count found for { + key: 'https://root.cern/cling', + url: 'https://root.cern/cling/', + title: 'Cling - ROOT', + c2key: 'https://root.cern/' +} +No count found for { + key: 'https://blog.sanctum.geek.nz/series/unix-as-ide', + url: 'https://blog.sanctum.geek.nz/series/unix-as-ide/', + title: 'Series: Unix as IDE « Arabesque', + c2key: 'https://blog.sanctum.geek.nz/' +} +No count found for { + key: 'https://standardnotes.com/', + url: 'https://standardnotes.com/', + title: 'Standard Notes | End-To-End Encrypted Notes App', + c2key: 'https://standardnotes.com/' +} +No count found for { + key: 'https://ffc2016.startupnotes.org/', + url: 'http://ffc2016.startupnotes.org/', + title: 'Startup Notes', + c2key: 'https://ffc2016.startupnotes.org/' +} +No count found for { + key: 'https://equity.ltse.com/calculators/tldr-stock-options', + url: 'https://equity.ltse.com/calculators/tldr-stock-options', + title: 'TLDR Options', + c2key: 'https://equity.ltse.com/' +} +No count found for { + key: 'https://toptal.com/designers/subtlepatterns', + url: 'https://www.toptal.com/designers/subtlepatterns/', + title: 'Subtle Patterns | Free textures for your next web project', + c2key: 'https://toptal.com/' +} +No count found for { + key: 'https://expeditedsecurity.com/aws-in-plain-english', + url: 'https://expeditedsecurity.com/aws-in-plain-english/', + title: 'Amazon Web Services In Plain English', + c2key: 'https://expeditedsecurity.com/' +} +No count found for { + key: 'https://gamedeveloper.com/programming/making-a-game-boy-game-in-2017-a-quot-sheep-it-up-quot-post-mortem-part-1-2-', + url: 'https://www.gamedeveloper.com/programming/making-a-game-boy-game-in-2017-a-quot-sheep-it-up-quot-post-mortem-part-1-2-', + title: 'Making a Game Boy game in 2017: A "Sheep It Up!" Post-Mortem', + c2key: 'https://gamedeveloper.com/' +} +No count found for { + key: 'https://forbes.com/sites/kristinakillgrove/2018/05/11/meet-the-worst-businessman-of-the-18th-century?sh=3592efaa2d5d', + url: 'https://www.forbes.com/sites/kristinakillgrove/2018/05/11/meet-the-worst-businessman-of-the-18th-century/?sh=3592efaa2d5d', + title: 'Meet The Worst Businessman Of The 18th Century BC', + c2key: 'https://forbes.com/?sh=3592efaa2d5d' +} +No count found for { + key: 'https://isik.dev/posts/Eigentechno', + url: 'https://www.isik.dev/posts/Eigentechno.html', + title: 'Eigentechno – Umut Isik', + c2key: 'https://isik.dev/' +} +No count found for { + key: 'https://science.org/content/article/scientists-pull-living-microbes-100-million-years-beneath-sea', + url: 'https://www.science.org/content/article/scientists-pull-living-microbes-100-million-years-beneath-sea', + title: 'Scientists pull living microbes, possibly 100 million years old, from beneath the sea | Science | AAAS', + c2key: 'https://science.org/' +} +No count found for { + key: 'https://science.org/content/article/these-120000-year-old-footprints-offer-early-evidence-humans-arabia', + url: 'https://www.science.org/content/article/these-120000-year-old-footprints-offer-early-evidence-humans-arabia', + title: 'These 120,000-year-old footprints offer early evidence for humans in Arabia | Science | AAAS', + c2key: 'https://science.org/' +} +No count found for { + key: 'https://unison-lang.org/', + url: 'https://www.unison-lang.org/', + title: 'The Unison language', + c2key: 'https://unison-lang.org/' +} +No count found for { + key: 'https://puredanger.github.io/tech.puredanger.com/2011/10/20/real-world-clojure', + url: 'https://puredanger.github.io/tech.puredanger.com/2011/10/20/real-world-clojure/', + title: 'Real world Clojure', + c2key: 'https://puredanger.github.io/' +} +No count found for { + key: 'https://netflixtechblog.com/linux-performance-analysis-in-60-000-milliseconds-accc10403c55?gi=acfa90ef312a', + url: 'https://netflixtechblog.com/linux-performance-analysis-in-60-000-milliseconds-accc10403c55?gi=acfa90ef312a', + title: 'Linux Performance Analysis in 60,000 Milliseconds | by Netflix Technology Blog | Netflix TechBlog', + c2key: 'https://netflixtechblog.com/?gi=acfa90ef312a' +} +No count found for { + key: 'https://netflixtechblog.com/linux-performance-analysis-in-60-000-milliseconds-accc10403c55', + url: 'https://netflixtechblog.com/linux-performance-analysis-in-60-000-milliseconds-accc10403c55', + title: 'Linux Performance Analysis in 60,000 Milliseconds | by Netflix Technology Blog | Netflix TechBlog', + c2key: 'https://netflixtechblog.com/' +} +No count found for { + key: 'https://signalvnoise.com/posts/753-ask-37signals-how-do-you-process-credit-cards', + url: 'https://signalvnoise.com/posts/753-ask-37signals-how-do-you-process-credit-cards', + title: 'Ask 37signals: How do you process credit cards? – Signal v. Noise', + c2key: 'https://signalvnoise.com/' +} +No count found for { + key: 'https://imperial.ac.uk/news/149087/scientists-discover-turn-light-into-matter', + url: 'https://www.imperial.ac.uk/news/149087/scientists-discover-turn-light-into-matter/', + title: 'Scientists discover how to turn light into matter after 80-year quest | Imperial News | Imperial College London', + c2key: 'https://imperial.ac.uk/' +} +No count found for { + key: 'https://aloneonahill.com/blog/if-php-were-british', + url: 'https://aloneonahill.com/blog/if-php-were-british/', + title: 'If PHP Were British | Alone On A Hill', + c2key: 'https://aloneonahill.com/' +} +No count found for { + key: 'https://adamtornhill.com/articles/lispweb', + url: 'http://www.adamtornhill.com/articles/lispweb.htm', + title: 'Lisp for the Web', + c2key: 'https://adamtornhill.com/' +} +No count found for { + key: 'https://web.archive.org/web/20220607102703/https://www.advogato.org/person/apenwarr/diary/371', + url: 'https://web.archive.org/web/20220607102703/https://www.advogato.org/person/apenwarr/diary/371.html', + title: '', + c2key: 'https://web.archive.org/' +} +No count found for { + key: 'https://andromeda.com/?f', + url: 'http://www.andromeda.com/?f', + title: 'andromeda.com', + c2key: 'https://andromeda.com/?f' +} +No count found for { + key: 'https://support.apple.com/en-us/HT204204', + url: 'https://support.apple.com/en-us/HT204204', + title: 'Update your iPhone, iPad, or iPod touch - Apple Support', + c2key: 'https://support.apple.com/' +} +No count found for { + key: 'https://archive.boston.com/bostonglobe/ideas/articles/2011/03/06/the_power_of_lonely?page=full', + url: 'http://archive.boston.com/bostonglobe/ideas/articles/2011/03/06/the_power_of_lonely/?page=full', + title: 'The power of lonely - The Boston Globe', + c2key: 'https://archive.boston.com/?page=full' +} +No count found for { + key: 'https://tildesites.bowdoin.edu/~ltoma/teaching/cs340/spring05/coursestuff/Bentley_BumperSticker.pdf', + url: 'https://tildesites.bowdoin.edu/~ltoma/teaching/cs340/spring05/coursestuff/Bentley_BumperSticker.pdf', + title: '', + c2key: 'https://tildesites.bowdoin.edu/' +} +No count found for { + key: 'https://blog.codinghorror.com/the-ten-commandments-of-egoless-programming', + url: 'https://blog.codinghorror.com/the-ten-commandments-of-egoless-programming/', + title: 'The Ten Commandments of Egoless Programming', + c2key: 'https://blog.codinghorror.com/' +} +No count found for { + key: 'https://simplethread.com/The-Programmer-Dress-Code', + url: 'https://www.simplethread.com/The-Programmer-Dress-Code', + title: 'Page not found - Simple Thread', + c2key: 'https://simplethread.com/' +} +No count found for { + key: 'https://blog.codinghorror.com/paying-down-your-technical-debt', + url: 'https://blog.codinghorror.com/paying-down-your-technical-debt/', + title: 'Paying Down Your Technical Debt', + c2key: 'https://blog.codinghorror.com/' +} +No count found for { + key: 'https://okcupid.com/', + url: 'https://www.okcupid.com/', + title: 'OkCupid', + c2key: 'https://okcupid.com/' +} +No count found for { + key: 'https://charlesrosenberg.com/', + url: 'http://www.charlesrosenberg.com/', + title: '** Home Page for Charles Rosenberg **', + c2key: 'https://charlesrosenberg.com/' +} +No count found for { + key: 'https://people.eecs.berkeley.edu/~bh/hacker', + url: 'https://people.eecs.berkeley.edu/~bh/hacker.html', + title: 'What is a Hacker?', + c2key: 'https://people.eecs.berkeley.edu/' +} +No count found for { + key: 'https://di.ku.dk/hjemmesider/ansatte/torbenm/Basics', + url: 'https://di.ku.dk/hjemmesider/ansatte/torbenm/Basics', + title: 'Siden blev ikke fundet (404) – Københavns Universitet', + c2key: 'https://di.ku.dk/' +} +No count found for { + key: 'https://forbes.com/sites/andygreenberg/2010/08/24/full-body-scan-technology-deployed-in-street-roving-vans?sh=212861fd42e3', + url: 'https://www.forbes.com/sites/andygreenberg/2010/08/24/full-body-scan-technology-deployed-in-street-roving-vans/?sh=212861fd42e3', + title: 'Full-Body Scan Technology Deployed In Street-Roving Vans', + c2key: 'https://forbes.com/?sh=212861fd42e3' +} +No count found for { + key: 'https://subscribe.forteantimes.com/features/articles/1302/lost_in_space', + url: 'https://subscribe.forteantimes.com/features/articles/1302/lost_in_space.html', + title: 'Nothing found for Features Articles 1302 Lost_In_Space', + c2key: 'https://subscribe.forteantimes.com/' +} +No count found for { + key: 'https://ye.gg/blog/2010/06/paths-to-5m-for-a-startup-founder', + url: 'https://ye.gg/blog/2010/06/paths-to-5m-for-a-startup-founder.html', + title: '404 - Page not found', + c2key: 'https://ye.gg/' +} +No count found for { + key: 'https://ye.gg/startupswiki/Ask_YC_Archive', + url: 'https://ye.gg/startupswiki/Ask_YC_Archive', + title: '404 - Page not found', + c2key: 'https://ye.gg/' +} +No count found for { + key: 'https://ads.google.com/home', + url: 'https://ads.google.com/home/', + title: '気軽に利用できるオンライン広告で顧客を増やしましょう | Google 広告', + c2key: 'https://ads.google.com/' +} +No count found for { + key: 'https://trends.google.com/trends?geo=JP', + url: 'https://trends.google.com/trends/?geo=JP', + title: 'Google Trends', + c2key: 'https://trends.google.com/?geo=JP' +} +No count found for { + key: 'https://sijinjoseph.com/programmer-competency-matrix', + url: 'https://sijinjoseph.com/programmer-competency-matrix/', + title: 'Programmer Competency Matrix | Sijin Joseph', + c2key: 'https://sijinjoseph.com/' +} +No count found for { + key: 'https://janestreet.com/minsky_weeks-jfp_18.pdf', + url: 'https://www.janestreet.com/minsky_weeks-jfp_18.pdf', + title: 'Page not found :: Jane Street', + c2key: 'https://janestreet.com/' +} +No count found for { + key: 'https://lessig.org/index.php?page=404', + url: 'https://lessig.org/index.php?page=404', + title: '404 - Not Found.', + c2key: 'https://lessig.org/?page=404' +} +No count found for { + key: 'https://let.rug.nl/bos/lpn//lpnpage.php?pageid=online', + url: 'http://www.let.rug.nl/bos/lpn//lpnpage.php?pageid=online', + title: 'Learn Prolog Now!', + c2key: 'https://let.rug.nl/?pageid=online' +} +No count found for { + key: 'https://lettersofnote.com/?p=826', + url: 'https://lettersofnote.com/?p=826', + title: 'Page not found – Letters of Note', + c2key: 'https://lettersofnote.com/?p=826' +} +No count found for { + key: 'https://makelinux.github.io/kernel/map', + url: 'https://makelinux.github.io/kernel/map/', + title: 'Interactive map of Linux kernel', + c2key: 'https://makelinux.github.io/' +} +No count found for { + key: 'https://people.math.umass.edu/~lavine/Book/book', + url: 'https://people.math.umass.edu/~lavine/Book/book.html', + title: '', + c2key: 'https://people.math.umass.edu/' +} +No count found for { + key: 'https://people.tamu.edu/~huafei-yan//Rota/mitless', + url: 'https://people.tamu.edu/~huafei-yan//Rota/mitless.html', + title: 'Ten lessons', + c2key: 'https://people.tamu.edu/' +} +No count found for { + key: 'https://news.microsoft.com/', + url: 'https://news.microsoft.com/', + title: 'Stories | Microsoft news, features, events, and press materials', + c2key: 'https://news.microsoft.com/' +} +No count found for { + key: 'https://calligraphr.com/en?rtom=myscriptfont', + url: 'https://www.calligraphr.com/en/?rtom=myscriptfont', + title: 'Calligraphr - Draw your own fonts.', + c2key: 'https://calligraphr.com/?rtom=myscriptfont' +} +No count found for { + key: 'https://news.northwestern.edu/stories/2015/03/creative-genius-driven-by-distraction', + url: 'https://news.northwestern.edu/stories/2015/03/creative-genius-driven-by-distraction.html', + title: 'Creative Genius Driven by Distraction - Northwestern Now', + c2key: 'https://news.northwestern.edu/' +} +No count found for { + key: 'https://wiki.openmoko.org/wiki/Main_Page', + url: 'http://wiki.openmoko.org/wiki/Main_Page', + title: 'Openmoko', + c2key: 'https://wiki.openmoko.org/' +} +No count found for { + key: 'https://userpage.fu-berlin.de/~ram/pub/pub_jf47ht81Ht/doc_kay_oop_en', + url: 'http://userpage.fu-berlin.de/~ram/pub/pub_jf47ht81Ht/doc_kay_oop_en', + title: 'Dr. Alan Kay on the Meaning of "Object-Oriented Programming"', + c2key: 'https://userpage.fu-berlin.de/' +} +No count found for { + key: 'https://simcast.com/?d=rafekettler.com&pcid=802&rid=152&a=1', + url: 'https://simcast.com/?d=rafekettler.com&pcid=802&rid=152&a=1', + title: 'Simcast News portal.', + c2key: 'https://simcast.com/?d=rafekettler.com&pcid=802&rid=152&a=1' +} +No count found for { + key: 'https://randomhacks.net.s3-website-us-east-1.amazonaws.com/2007/02/22/bayes-rule-and-drug-tests', + url: 'http://www.randomhacks.net.s3-website-us-east-1.amazonaws.com/2007/02/22/bayes-rule-and-drug-tests/', + title: "Bayes' rule in Haskell, or why drug tests don't work | Random Hacks", + c2key: 'https://randomhacks.net.s3-website-us-east-1.amazonaws.com/' +} +No count found for { + key: 'https://siegemedia.com/team/ross-hudgens', + url: 'https://www.siegemedia.com/team/ross-hudgens', + title: 'Ross Hudgens - Siege Media', + c2key: 'https://siegemedia.com/' +} +No count found for { + key: 'https://scottaaronson.blog/?p=304', + url: 'https://scottaaronson.blog/?p=304', + title: 'Shtetl-Optimized » Blog Archive » Ten Signs a Claimed Mathematical Breakthrough is Wrong', + c2key: 'https://scottaaronson.blog/?p=304' +} +No count found for { + key: 'https://robwalling.com/2010/12/09/how-to-detect-a-toxic-customer', + url: 'https://robwalling.com/2010/12/09/how-to-detect-a-toxic-customer/', + title: 'How to Detect a Toxic Customer | Rob Walling - Serial Entrepreneur', + c2key: 'https://robwalling.com/' +} +No count found for { + key: 'https://toptal.com/developers/sorting-algorithms', + url: 'https://www.toptal.com/developers/sorting-algorithms', + title: 'Sorting Algorithms Animations | Toptal®', + c2key: 'https://toptal.com/' +} +No count found for { + key: 'https://webspace.science.uu.nl/~gadda001/goodtheorist', + url: 'https://webspace.science.uu.nl/~gadda001/goodtheorist/', + title: 'How to become a GOOD Theoretical Physicist', + c2key: 'https://webspace.science.uu.nl/' +} +No count found for { + key: 'https://smh.com.au/story/1848433/the-ocean-is-broken', + url: 'https://www.smh.com.au/story/1848433/the-ocean-is-broken', + title: 'Error', + c2key: 'https://smh.com.au/' +} +No count found for { + key: 'https://theregister.com/2001/05/15/could_bill_gates_write_code', + url: 'https://www.theregister.com/2001/05/15/could_bill_gates_write_code', + title: 'Could Bill Gates write code? • The Register', + c2key: 'https://theregister.com/' +} +No count found for { + key: 'https://weegen.home.xs4all.nl/eelis/analogliterals.xhtml', + url: 'https://weegen.home.xs4all.nl/eelis/analogliterals.xhtml', + title: '404 Not Found', + c2key: 'https://weegen.home.xs4all.nl/' +} +No count found for { + key: 'https://google.com/sorry/index?continue=https://www.youtube.com/watch%3Fv%3D4XpnKHJAok8&q=EhAkAIkCAAAAAPA8k__-ebNIGI7ls5gGIhAzUhbf-JaZapASy1H_urZoMgFy', + url: 'https://www.google.com/sorry/index?continue=https://www.youtube.com/watch%3Fv%3D4XpnKHJAok8&q=EhAkAIkCAAAAAPA8k__-ebNIGI7ls5gGIhAzUhbf-JaZapASy1H_urZoMgFy', + title: 'https://www.youtube.com/watch?v=4XpnKHJAok8', + c2key: 'https://google.com/?continue=https://www.youtube.com/watch%3Fv%3D4XpnKHJAok8&q=EhAkAIkCAAAAAPA8k__-ebNIGI7ls5gGIhAzUhbf-JaZapASy1H_urZoMgFy' +} +No count found for { + key: 'https://google.com/sorry/index?continue=https://www.youtube.com/watch%3Fv%3Dz6hoPw5hItY&q=EhAkAIkCAAAAAPA8k__-ebNIGI_ls5gGIhAStzDTA8hAkZxBtvd-7494MgFy', + url: 'https://www.google.com/sorry/index?continue=https://www.youtube.com/watch%3Fv%3Dz6hoPw5hItY&q=EhAkAIkCAAAAAPA8k__-ebNIGI_ls5gGIhAStzDTA8hAkZxBtvd-7494MgFy', + title: 'https://www.youtube.com/watch?v=z6hoPw5hItY', + c2key: 'https://google.com/?continue=https://www.youtube.com/watch%3Fv%3Dz6hoPw5hItY&q=EhAkAIkCAAAAAPA8k__-ebNIGI_ls5gGIhAStzDTA8hAkZxBtvd-7494MgFy' +} +No count found for { + key: 'https://dan.com/buy-domain/%E2%9E%A1.ws?redirected=true', + url: 'https://dan.com/buy-domain/%E2%9E%A1.ws?redirected=true', + title: 'Buy and Sell Domain Names | Dan.com', + c2key: 'https://dan.com/?redirected=true' +} +No count found for { + key: 'https://google.com/sorry/index?continue=https://youtube.com/watch%3Fv%3DD1R-jKKp3NA&q=EhAkAIkCAAAAAPA8k__-ebNIGP3ms5gGIhBa2zvTMOF3FV_ML1zDIDqKMgFy', + url: 'https://www.google.com/sorry/index?continue=https://youtube.com/watch%3Fv%3DD1R-jKKp3NA&q=EhAkAIkCAAAAAPA8k__-ebNIGP3ms5gGIhBa2zvTMOF3FV_ML1zDIDqKMgFy', + title: 'https://youtube.com/watch?v=D1R-jKKp3NA', + c2key: 'https://google.com/?continue=https://youtube.com/watch%3Fv%3DD1R-jKKp3NA&q=EhAkAIkCAAAAAPA8k__-ebNIGP3ms5gGIhBa2zvTMOF3FV_ML1zDIDqKMgFy' +} +No count found for { + key: 'https://clarle.github.io/yui3/crockford', + url: 'https://clarle.github.io/yui3/crockford', + title: 'Page not found · GitHub Pages', + c2key: 'https://clarle.github.io/' +} +No count found for { + key: 'https://humprog.org/~stephen/blog/2014/10/07', + url: 'https://www.humprog.org/~stephen/blog/2014/10/07/', + title: 'Rambles around computer science', + c2key: 'https://humprog.org/' +} +No count found for { + key: 'https://enso.org/', + url: 'https://enso.org/', + title: 'Enso | Get insights you can rely on. In real time.', + c2key: 'https://enso.org/' +} +No count found for { + key: 'https://cs1.tf.fau.de/research/system-security-group/tresor-trevisor-armored', + url: 'https://www.cs1.tf.fau.de/research/system-security-group/tresor-trevisor-armored/', + title: 'CPU-bound Encryption (TRESOR, TreVisor, ARMORED) › IT Security Infrastructures Lab', + c2key: 'https://cs1.tf.fau.de/' +} +No count found for { + key: 'https://mailchimp.com/resources', + url: 'https://mailchimp.com/resources/', + title: 'Marketing Resources | Mailchimp', + c2key: 'https://mailchimp.com/' +} +No count found for { + key: 'https://manybutfinite.com/post/anatomy-of-a-program-in-memory', + url: 'https://manybutfinite.com/post/anatomy-of-a-program-in-memory/', + title: 'Anatomy of a Program in Memory | Many But Finite', + c2key: 'https://manybutfinite.com/' +} +No count found for { + key: 'https://danbenjamin.com/', + url: 'https://danbenjamin.com/', + title: 'Dan Benjamin', + c2key: 'https://danbenjamin.com/' +} +No count found for { + key: 'https://npopov.com/2012/06/15/The-true-power-of-regular-expressions', + url: 'https://www.npopov.com/2012/06/15/The-true-power-of-regular-expressions.html', + title: 'The true power of regular expressions', + c2key: 'https://npopov.com/' +} +No count found for { + key: 'https://coveryourtracks.eff.org/', + url: 'https://coveryourtracks.eff.org/', + title: 'Cover Your Tracks', + c2key: 'https://coveryourtracks.eff.org/' +} +No count found for { + key: 'https://google.com/sorry/index?continue=https://www.youtube.com/watch%3Fv%3Dtc4ROCJYbm0&q=EhAkAIkCAAAAAPA8k__-ebNIGLr4s5gGIhBl65vidhJMTKGIAA9WmktwMgFy', + url: 'https://www.google.com/sorry/index?continue=https://www.youtube.com/watch%3Fv%3Dtc4ROCJYbm0&q=EhAkAIkCAAAAAPA8k__-ebNIGLr4s5gGIhBl65vidhJMTKGIAA9WmktwMgFy', + title: 'https://www.youtube.com/watch?v=tc4ROCJYbm0', + c2key: 'https://google.com/?continue=https://www.youtube.com/watch%3Fv%3Dtc4ROCJYbm0&q=EhAkAIkCAAAAAPA8k__-ebNIGLr4s5gGIhBl65vidhJMTKGIAA9WmktwMgFy' +} +No count found for { + key: 'https://khoury.northeastern.edu/404.php', + url: 'https://www.khoury.northeastern.edu/404.php', + title: 'Page not found – Khoury College Development', + c2key: 'https://khoury.northeastern.edu/' +} +No count found for { + key: 'https://bizjournals.com/news/technology/startups', + url: 'https://www.bizjournals.com/news/technology/startups', + title: 'Startups News - The Business Journals', + c2key: 'https://bizjournals.com/' +} +No count found for { + key: 'https://crockford.com/dec64', + url: 'https://www.crockford.com/dec64.html', + title: 'DEC64: Decimal Floating Point', + c2key: 'https://crockford.com/' +} +No count found for { + key: 'https://forge.ocamlcore.org/projects/ocamlunix', + url: 'https://forge.ocamlcore.org/projects/ocamlunix/', + title: 'ocamlunix', + c2key: 'https://forge.ocamlcore.org/' +} +No count found for { + key: 'https://ww17.blog.searchyc.com/', + url: 'http://ww17.blog.searchyc.com/', + title: '', + c2key: 'https://ww17.blog.searchyc.com/' +} +No count found for { + key: 'https://johnresig.com/apps/learn', + url: 'https://johnresig.com/apps/learn/', + title: 'Learning Advanced JavaScript', + c2key: 'https://johnresig.com/' +} +No count found for { + key: 'https://dropcatch.ai/domains/ramtin-amin.fr', + url: 'https://www.dropcatch.ai/domains/ramtin-amin.fr', + title: 'ramtin-amin.fr is for sale! - DropCatch.ai', + c2key: 'https://dropcatch.ai/' +} +No count found for { + key: 'https://githubstatus.com/', + url: 'https://www.githubstatus.com/', + title: 'GitHub Status', + c2key: 'https://githubstatus.com/' +} +No count found for { + key: 'https://health.aws.amazon.com/health/status', + url: 'https://health.aws.amazon.com/health/status', + title: 'AWS Health Dashboard - Aug 29, 2022 PDT', + c2key: 'https://health.aws.amazon.com/' +} diff --git a/public/problem_find.mjs b/public/problem_find.mjs new file mode 100755 index 0000000..25ac067 --- /dev/null +++ b/public/problem_find.mjs @@ -0,0 +1,117 @@ +#!/usr/bin/env node + +import fs from 'node:fs'; +import path from 'node:path'; +import child_process from 'node:child_process'; + +import { + loadPref, + cache_file, + index_file, +} from '../src/args.js'; + +const CLEAN = false; +const CONCURRENT = 7; +const sleep = ms => new Promise(res => setTimeout(res, ms)); +const problems = new Map(); +let cleaning = false; +let made = false; + +process.on('exit', cleanup); +process.on('SIGINT', cleanup); +process.on('SIGTERM', cleanup); +process.on('SIGHUP', cleanup); +process.on('SIGUSR2', cleanup); +process.on('beforeExit', cleanup); + +console.log({Pref:loadPref(), cache_file: cache_file(), index_file: index_file()}); +make(); + +async function make() { + const indexFile = fs.readFileSync(index_file()).toString(); + JSON.parse(indexFile).map(([key, value]) => { + if ( typeof key === "number" ) return; + if ( key.startsWith('ndx') ) return; + if ( value.title === undefined ) { + console.log('no title property', {key, value}); + } + const url = key; + const title = value.title.toLocaleLowerCase(); + if ( title.length === 0 || title.includes('404') || title.includes('not found') ) { + if ( problems.has(url) ) { + console.log('Found duplicate', url, title, problems.get(url)); + } + problems.set(url, title); + } + }); + + made = true; + + cleanup(); +} + +function cleanup() { + if ( cleaning ) return; + if ( ! made ) return; + cleaning = true; + console.log('cleanup running'); + const outData = [...problems.entries()]; + fs.writeFileSync( + path.resolve('.', 'url-problems.json'), + JSON.stringify(outData, null, 2) + ); + const {size:bytesWritten} = fs.statSync( + path.resolve('.', 'url-problems.json'), + {bigint: true} + ); + console.log(`Wrote ${outData.length} problem urls in ${bytesWritten} bytes.`); + process.exit(0); +} + +function clean(urlString) { + const url = new URL(urlString); + if ( url.hash.startsWith('#!') || url.hostname.includes('google.com') || url.hostname.includes('80s.nyc') ) { + } else { + url.hash = ''; + } + for ( const [key, value] of url.searchParams ) { + if ( key.startsWith('utm_') ) { + url.searchParams.delete(key); + } + } + url.pathname = url.pathname.replace(/\/$/, ''); + url.protocol = 'https:'; + url.pathname = url.pathname.replace(/(\.htm.?|\.php)$/, ''); + if ( url.hostname.startsWith('www.') ) { + url.hostname = url.hostname.replace(/^www./, ''); + } + const key = url.toString(); + return key; +} + +function clean2(urlString) { + const url = new URL(urlString); + url.pathname = ''; + return url.toString(); +} + +function curlCommand(url) { + return `curl -k -L -s -o /dev/null -w '%{url_effective}' ${JSON.stringify(url)} \ + -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \ + -H 'Accept-Language: en,en-US;q=0.9,zh-TW;q=0.8,zh-CN;q=0.7,zh;q=0.6,ja;q=0.5' \ + -H 'Cache-Control: no-cache' \ + -H 'Connection: keep-alive' \ + -H 'DNT: 1' \ + -H 'Pragma: no-cache' \ + -H 'Sec-Fetch-Dest: document' \ + -H 'Sec-Fetch-Mode: navigate' \ + -H 'Sec-Fetch-Site: none' \ + -H 'Sec-Fetch-User: ?1' \ + -H 'Upgrade-Insecure-Requests: 1' \ + -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36' \ + -H 'sec-ch-ua: "Chromium";v="104", " Not A;Brand";v="99", "Google Chrome";v="104"' \ + -H 'sec-ch-ua-mobile: ?0' \ + -H 'sec-ch-ua-platform: "macOS"' \ + --compressed ; + `; +} diff --git a/public/redirector.html b/public/redirector.html new file mode 100644 index 0000000..f1289a0 --- /dev/null +++ b/public/redirector.html @@ -0,0 +1,21 @@ + + +

About to index archive and index

+ diff --git a/public/style.css b/public/style.css new file mode 100644 index 0000000..ce20e72 --- /dev/null +++ b/public/style.css @@ -0,0 +1,101 @@ + + :root { + font-family: sans-serif; + background: lavenderblush; + } + body { + display: table; + margin: 0 auto 0 min(10%, 5em); + background: white; + padding: 0.5em; + border-bottom: 1px solid purple; + max-width: min(777px, 80%); + } + header { + font-size: smaller; + margin-bottom: 3em; + } + form { + margin-bottom: 2em; + } + label.cmd { + border-bottom: thin solid dodgerblue; + cursor: default; + font-style: italic; + } + :is(form, label)[disabled] { + color: grey; + } + em.caps { + font-style: normal; + font-variant: small-caps; + } + :is(form:hover, form:active) em.caps { + background: yellow; + } + + legend { + font-weight: 600; + } + fieldset { + border: thin solid transparent; + } + fieldset.search { + display: flex; + } + button, input, output { + } + button { + } + form .long { + width: 100%; + min-width: 250px; + } + output { + font-size: smaller; + color: purple; + } + h1 { + margin: 0; + } + h2 { + margin-top: 0; + } + small.url { + word-break: break-all; + } + .small { + font-size: smaller; + } + + label small { + font-style: italic; + color: darkslategrey; + } + + .units { + color: grey; + font-size: smaller; + } + + input[type="number"] { + text-align: right; + ] + + input.search { + flex-grow: 1; + padding: 0.25em 0.5em; + } + input.search, + input.search + button { + font-size: 1em; + } + ol.results { + list-style-type: none; + } + .cent { + text-align: center; + } + .grey { + color: grey; + } diff --git a/public/test-injection.html b/public/test-injection.html new file mode 100644 index 0000000..e4d25a8 --- /dev/null +++ b/public/test-injection.html @@ -0,0 +1 @@ + diff --git a/public/top.html b/public/top.html new file mode 100644 index 0000000..2c0547b --- /dev/null +++ b/public/top.html @@ -0,0 +1,3 @@ + diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..14c31a4 --- /dev/null +++ b/run.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +phys=$(free -t -m | grep -oP '\d+' | sed '10!d') +alloc=$(echo "$phys * 90/100" | bc ) +echo $alloc +node --max-old-space-size=$alloc src/app.js diff --git a/scripts/build_only.sh b/scripts/build_only.sh new file mode 100755 index 0000000..52505a6 --- /dev/null +++ b/scripts/build_only.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +./node_modules/.bin/esbuild src/app.js --bundle --outfile=dist/diskernet.mjs --format=esm --platform=node --minify --analyze +./node_modules/.bin/esbuild src/app.js --bundle --outfile=build/out.cjs --platform=node --minify --analyze +./node_modules/.bin/esbuild src/app.js --bundle --outfile=build/test.cjs --platform=node +echo "#!/usr/bin/env node" > build/diskernet.cjs +cat build/out.cjs >> build/diskernet.cjs +chmod +x build/diskernet.cjs + diff --git a/scripts/build_setup.sh b/scripts/build_setup.sh new file mode 100755 index 0000000..b7198d1 --- /dev/null +++ b/scripts/build_setup.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +source $HOME/.nvm/nvm.sh + +echo "Making build directories..." + +mkdir -p dist/ +mkdir -p bin/ +mkdir -p build/ + +echo "Setting node to lts/*..." +nvm use --lts + +echo "Installing pkg..." + +which pkg || npm i -g pkg + +echo "Installing esbuild..." +npm install --save-exact esbuild + +echo "Done" + diff --git a/scripts/compile.sh b/scripts/compile.sh new file mode 100755 index 0000000..b99eb47 --- /dev/null +++ b/scripts/compile.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +./scripts/build_only.sh + +unset npm_config_prefix +source $HOME/.nvm/nvm.sh +nvm use --lts + +pkg --compress GZip . diff --git a/scripts/dl-node.sh b/scripts/dl-node.sh new file mode 100755 index 0000000..d0aa0da --- /dev/null +++ b/scripts/dl-node.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +unset npm_config_prefix +source $HOME/.nvm/nvm.sh +. $HOME/.profile + +nvm install --lts +nvm use --lts + +pkg ./src/hello.js + +rm -rf hello-* + diff --git a/scripts/go_build.sh b/scripts/go_build.sh new file mode 100755 index 0000000..9f0aff5 --- /dev/null +++ b/scripts/go_build.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +cp ./.package.build.json ./package.json +cp ./src/.common.build.js ./src/common.js + diff --git a/scripts/go_dev.sh b/scripts/go_dev.sh new file mode 100755 index 0000000..5569800 --- /dev/null +++ b/scripts/go_dev.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +gut "Just built" +cp ./.package.dev.json ./package.json +cp ./src/.common.dev.js ./src/common.js + diff --git a/scripts/old_compile.sh b/scripts/old_compile.sh new file mode 100755 index 0000000..3bea6db --- /dev/null +++ b/scripts/old_compile.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +unset npm_config_prefix +source $HOME/.nvm/nvm.sh +. $HOME/.profile +nvm use --lts + +echo "Cleaning old build and dist files..." + +rm -rf build/* dist/* + +echo "Setting build (CJS) mode..." +./scripts/go_build.sh + +patch_required=$(grep -ER "require\([\"'](node:)?stream/web[\"']\)" node_modules/*) +files=$(grep -rlER "require\([\"'](node:)?stream/web[\"']\)" node_modules/*) +if [[ ! -z "$patch_required" ]]; then + while IFS= read -r file; do + #echo '--->' $file + #grep -q $file package.json + #if [ $? == 1 ]; then + echo '--->' $file "UNPATCHED!" + echo "Found an error!" + echo "Found something you need to patch before building" + echo "See: https://github.com/vercel/pkg/issues/1451" + echo + echo "$patch_required" + echo + echo "You need to add all these to pkg.patches to replace with require('stream').web" + ./scripts/go_dev.sh + exit 1 + #fi + #echo "OK" + done <<< $files +fi + +echo "Bundling javascript..." +export NODE_ENV='production' +npx webpack +chmod +x ./build/22120.js +echo "Building for windows nix and macos..." +npx pkg --compress Gzip . + +echo "Restoring dev (ES module) mode..." +./scripts/go_dev.sh + +echo "Rebundling an es module for npm es module import..." +npm run bundle diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh new file mode 100755 index 0000000..ca9c1f1 --- /dev/null +++ b/scripts/postinstall.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +which brew || /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +which mkcert || brew install mkcert +mkdir -p $HOME/local-sslcerts +cd $HOME/local-sslcerts + +mkcert -key-file privkey.pem -cert-file fullchain.pem localhost +mkcert -install + diff --git a/scripts/publish.sh b/scripts/publish.sh new file mode 100755 index 0000000..0a336a2 --- /dev/null +++ b/scripts/publish.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +./scripts/go_build.sh +gpush minor "$@" +./scripts/go_dev.sh + diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 0000000..e888e6e --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +#./scripts/compile.sh +description=$1 +latest_tag=$(git describe --abbrev=0) +grel release -u crisdosyago -r 22120 --tag $latest_tag --name "New release" --description '"'"$description"'"' +grel upload -u crisdosyago -r 22120 --tag $latest_tag --name "diskernet-win.exe" --file bin/diskernet-win.exe +grel upload -u crisdosyago -r 22120 --tag $latest_tag --name "diskernet-linux" --file bin/diskernet-linux +grel upload -u crisdosyago -r 22120 --tag $latest_tag --name "diskernet-macos" --file bin/diskernet-macos + + + diff --git a/src/app.js b/src/app.js new file mode 100644 index 0000000..f42569c --- /dev/null +++ b/src/app.js @@ -0,0 +1,167 @@ +import fs from 'fs'; +import ChildProcess from 'child_process'; +import readline from 'readline'; +import util from 'util'; +import {stdin as input, stdout as output} from 'process'; + +import ChromeLauncher from 'chrome-launcher'; +import psList from '@667/ps-list'; + +import {DEBUG, sleep, NO_SANDBOX, GO_SECURE} from './common.js'; + +import {Archivist} from './archivist.js'; +import LibraryServer from './libraryServer.js'; +import args from './args.js'; + +const {server_port, mode, chrome_port} = args; +const CHROME_OPTS = !NO_SANDBOX ? [ + /*'--restore-last-session',*/ + `--disk-cache-dir=${args.temp_browser_cache()}`, + `--aggressive-cache-discard` +] : [ + /*'--restore-last-session',*/ + `--disk-cache-dir=${args.temp_browser_cache()}`, + `--aggressive-cache-discard`, + '--no-sandbox', +]; +const LAUNCH_OPTS = { + logLevel: DEBUG ? 'verbose' : 'silent', + port: chrome_port, + chromeFlags:CHROME_OPTS, + userDataDir:false, + startingUrl: `${GO_SECURE ? 'https' : 'http'}://localhost:${args.server_port}`, + ignoreDefaultFlags: true +} +const KILL_ON = { + win32: 'taskkill /IM chrome.exe /F', + darwin: 'kill $(pgrep Chrome)', + freebsd: 'pkill -15 chrome', + linux: 'pkill -15 chrome', +}; + +let quitting; + +start(); + +async function start() { + console.log(`Running in node...`); + + process.on('error', cleanup); + process.on('unhandledRejection', cleanup); + process.on('uncaughtException', cleanup); + process.on('SIGHUP', cleanup); + process.on('beforeExit', cleanup); + process.on('SIGINT', code => cleanup(code, 'signal', {exit:true})); + process.on('SIGTERM', code => cleanup(code, 'signal', {exit:true})); + process.on('SIGQUIT', code => cleanup(code, 'signal', {exit:true})); + process.on('SIGBREAK', code => cleanup(code, 'signal', {exit:true})); + process.on('SIGABRT', code => cleanup(code, 'signal', {exit:true})); + + console.log(`Importing dependencies...`); + const {launch:ChromeLaunch} = ChromeLauncher; + + let chromeOpen = false; + + const list = await psList(); + + chromeOpen = list.some(({name,cmd}) => name?.match?.(/chrome/g) || cmd?.match?.(/chrome/g)); + + if ( chromeOpen ) { + console.info(`Seems Chrome is open`); + if ( DEBUG.askFirst ) { + const rl = readline.createInterface({input, output}); + const question = util.promisify(rl.question).bind(rl); + console.info(`\nIf you don't shut down Chrome and restart it under DiskerNet control + you will not be able to save or serve your archives.\n`); + const answer = await question("Would you like to shutdown Chrome browser now (y/N) ? "); + if ( answer?.match(/^y/i) ) { + await killChrome(); + } else { + console.log(`OK, not shutting it!\n`); + if ( chromeOpen ) { + process.exit(0); + } + } + } else { + await killChrome(); + } + } + + console.log(`Removing 22120's existing temporary browser cache if it exists...`); + if ( fs.existsSync(args.temp_browser_cache()) ) { + console.log(`Temp browser cache directory (${args.temp_browser_cache()}) exists, deleting...`); + fs.rmdirSync(args.temp_browser_cache(), {recursive:true}); + console.log(`Deleted.`); + } + console.log(`Launching library server...`); + await LibraryServer.start({server_port}); + console.log(`Library server started.`); + + console.log(`Waiting 1 second...`); + await sleep(1000); + + console.log(`Launching chrome...`); + try { + await ChromeLaunch(LAUNCH_OPTS); + } catch(e) { + console.log(`Could not launch chrome.`); + DEBUG.verboseSlow && console.info('Chrome launch error:', e); + process.exit(1); + } + console.log(`Chrome started.`); + + console.log(`Waiting 1 second...`); + await sleep(1000); + console.log(`Launching archivist and connecting to browser...`); + await Archivist.collect({chrome_port, mode}); + console.log(`System ready.`); +} + +async function killChrome(wait = true) { + try { + if ( process.platform in KILL_ON ) { + console.log(`Attempting to shut running chrome...`); + const [err] = (await new Promise( + res => ChildProcess.exec(KILL_ON[process.platform], (...a) => res(a)) + )); + if ( err ) { + console.log(`There was no running chrome.`); + DEBUG.verboseSlow && console.warn("Error closing existing chrome", err); + } else { + console.log(`Running chrome shut down.`); + if ( wait ) { + console.log(`Waiting 1 second...`); + await sleep(1000); + } + } + } else { + console.warn(`If you have chrome running, you may need to shut it down manually and restart 22120.`); + } + } catch(e) { + console.warn("in kill chrome", e); + } +} + +async function cleanup(reason, err, {exit = false} = {}) { + console.log(`Cleanup called on reason: ${reason}`, err); + + if ( quitting ) { + console.log(`Cleanup already called so not running again.`); + return; + } + quitting = true; + + Archivist.shutdown(); + + LibraryServer.stop(); + + killChrome(false); + + if ( exit ) { + console.log(`Take a breath. Everything's done. DiskerNet is exiting in 3 seconds...`); + + await sleep(3000); + + process.exit(0); + } +} diff --git a/src/archivist.js b/src/archivist.js new file mode 100644 index 0000000..7741b76 --- /dev/null +++ b/src/archivist.js @@ -0,0 +1,1911 @@ +// Licenses + // FlexSearch is Apache-2.0 licensed + // Source: https://github.com/nextapps-de/flexsearch/blob/bffb255b7904cb7f79f027faeb963ecef0a85dba/LICENSE + // NDX is MIT licensed + // Source: https://github.com/ndx-search/ndx/blob/cc9ec2780d88918338d4edcfca2d4304af9dc721/LICENSE + +// module imports + import hasha from 'hasha'; + import {URL} from 'url'; + import Path from 'path'; + import os from 'os'; + import Fs from 'fs'; + import {stdin as input, stdout as output} from 'process'; + import util from 'util'; + import readline from 'readline'; + + // search related + import FlexSearch from 'flexsearch'; + const {Index: FTSIndex} = FlexSearch; + //const {Index: FTSIndex} = require('flexsearch'); + import { + createIndex as NDX, + addDocumentToIndex as ndx, + removeDocumentFromIndex, + vacuumIndex + } from 'ndx'; + import { query as NDXQuery } from 'ndx-query'; + import { toSerializable, fromSerializable } from 'ndx-serializable'; + //import { DocumentIndex } from 'ndx'; + import Fuzzy from 'fz-search'; + //import * as _Fuzzy from './lib/fz.js'; + import Nat from 'natural'; + + import args from './args.js'; + import { + GO_SECURE, + untilTrue, + sleep, DEBUG as debug, + BATCH_SIZE, + MIN_TIME_PER_PAGE, + MAX_TIME_PER_PAGE, + MAX_TITLE_LENGTH, + MAX_URL_LENGTH, + clone, + CHECK_INTERVAL, TEXT_NODE, FORBIDDEN_TEXT_PARENT, + RichError + } from './common.js'; + import {connect} from './protocol.js'; + import {BLOCKED_CODE, BLOCKED_HEADERS} from './blockedResponse.js'; + import {getInjection} from '../public/injection.js'; + import {hasBookmark, bookmarkChanges} from './bookmarker.js'; + +// search related state: constants and variables + const DEBUG = debug || false; + // common + /* eslint-disable no-control-regex */ + const STRIP_CHARS = /[\u0001-\u001a\0\v\f\r\t\n]/g; + /* eslint-enable no-control-regex */ + //const Fuzzy = globalThis.FuzzySearch; + const NDX_OLD = false; + const USE_FLEX = true; + const FTS_INDEX_DIR = args.fts_index_dir; + const URI_SPLIT = /[/.]/g; + const NDX_ID_KEY = 'ndx_id'; + const INDEX_HIDDEN_KEYS = new Set([ + NDX_ID_KEY + ]); + const hiddenKey = key => key.startsWith('ndx') || INDEX_HIDDEN_KEYS.has(key); + let Id; + + // natural (NLP tools -- stemmers and tokenizers, etc) + const {WordTokenizer, PorterStemmer} = Nat; + const Tokenizer = new WordTokenizer(); + const Stemmer = PorterStemmer; + const words = Tokenizer.tokenize.bind(Tokenizer); + const termFilter = Stemmer.stem.bind(Stemmer); + //const termFilter = s => s.toLocaleLowerCase(); + + // FlexSearch + const FLEX_OPTS = { + charset: "utf8", + context: true, + language: "en", + tokenize: "reverse" + }; + let Flex = new FTSIndex(FLEX_OPTS); + DEBUG.verboseSlow && console.log({Flex}); + + // NDX + const NDXRemoved = new Set(); + const REMOVED_CAP_TO_VACUUM_NDX = 10; + const NDX_FIELDS = ndxDocFields(); + let NDX_FTSIndex = new NDXIndex(NDX_FIELDS); + let NDXId; + DEBUG.verboseSlow && console.log({NDX_FTSIndex}); + + // fuzzy (maybe just for queries ?) + const REGULAR_SEARCH_OPTIONS_FUZZY = { + minimum_match: 1.0 + }; + const HIGHLIGHT_OPTIONS_FUZZY = { + minimum_match: 2.0 // or 3.0 seems to be good + }; + const FUZZ_OPTS = { + keys: ndxDocFields({namesOnly:true}) + }; + const Docs = new Map(); + let fuzzy = new Fuzzy({source: [...Docs.values()], keys: FUZZ_OPTS.keys}); + +// module state: constants and variables + // cache is a simple map + // that holds the serialized requests + // that are saved on disk + const Status = { + loaded: false + }; + const FrameNodes = new Map(); + const Targets = new Map(); + const UpdatedKeys = new Set(); + const Cache = new Map(); + const Index = new Map(); + const Indexing = new Set(); + const CrawlIndexing = new Set(); + const CrawlTargets = new Set(); + const CrawlData = new Map(); + const Q = new Set(); + const Sessions = new Map(); + const Installations = new Set(); + const ConfirmedInstalls = new Set(); + const BLANK_STATE = { + Targets, + Sessions, + Installations, + ConfirmedInstalls, + FrameNodes, + Docs, + Indexing, + CrawlIndexing, + CrawlData, + CrawlTargets, + Cache, + Index, + NDX_FTSIndex, + Flex, + SavedCacheFilePath: null, + SavedIndexFilePath: null, + SavedFTSIndexDirPath: null, + SavedFuzzyIndexDirPath: null, + saver: null, + indexSaver: null, + ftsIndexSaver: null, + saveInProgress: false, + ftsSaveInProgress: false + }; + const State = Object.assign({}, BLANK_STATE); + export const Archivist = { + NDX_OLD, + USE_FLEX, + collect, getMode, changeMode, shutdown, + beforePathChanged, + afterPathChanged, + saveIndex, + getIndex, + deleteFromIndexAndSearch, + search, + getDetails, + isReady, + findOffsets, + archiveAndIndexURL + } + const BODYLESS = new Set([ + 301, + 302, + 303, + 307 + ]); + const NEVER_CACHE = new Set([ + `${GO_SECURE ? 'https' : 'http'}://localhost:${args.server_port}`, + `http://localhost:${args.chrome_port}` + ]); + const SORT_URLS = ([urlA],[urlB]) => urlA < urlB ? -1 : 1; + const CACHE_FILE = args.cache_file; + const INDEX_FILE = args.index_file; + const NO_FILE = args.no_file; + const TBL = /:\/\//g; + const HASH_OPTS = {algorithm: 'sha1'}; + const UNCACHED_BODY = b64('We have not saved this data'); + const UNCACHED_CODE = 404; + const UNCACHED_HEADERS = [ + { name: 'Content-type', value: 'text/plain' }, + { name: 'Content-length', value: '26' } + ]; + const UNCACHED = { + body:UNCACHED_BODY, responseCode:UNCACHED_CODE, responseHeaders:UNCACHED_HEADERS + } + let Mode, Close; + +// shutdown and cleanup + // handle writing out indexes and closing browser connection when resetting under nodemon + process.once('SIGUSR2', function () { + shutdown(function () { + process.kill(process.pid, 'SIGUSR2'); + }); + }); + +// logging + let logName; + let logStream; + +// main + async function collect({chrome_port:port, mode} = {}) { + const {library_path} = args; + const exitHandlers = []; + process.on('SIGUSR2', runHandlers); + process.on('beforeExit', runHandlers); + process.on('exit', code => runHandlers(code, 'exit', {exit: true})); + State.connection = State.connection || await connect({port}); + State.onExit = { + addHandler(h) { + exitHandlers.push(h); + } + }; + const {send, on, close} = State.connection; + //const DELAY = 100; // 500 ? + Close = close; + + let requestStage; + + await loadFiles(); + + clearSavers(); + + Mode = mode; + console.log({Mode}); + if ( Mode == 'save' || Mode == 'select' ) { + requestStage = "Response"; + // in case we get a updateBasePath call before an interval + // and we don't clear it in time, leading us to erroneously save the old + // cache to the new path, we always used our saved copy + State.saver = setInterval(() => saveCache(State.SavedCacheFilePath), 17000); + // we use timeout because we can trigger this ourself + // so in order to not get a race condition (overlapping calls) we ensure + // only 1 call at 1 time + State.indexSaver = setTimeout(() => saveIndex(State.SavedIndexFilePath), 11001); + State.ftsIndexSaver = setTimeout(() => saveFTS(State.SavedFTSIndexDirPath), 31001); + } else if ( Mode == 'serve' ) { + requestStage = "Request"; + clearSavers(); + } else { + throw new TypeError(`Must specify mode, and must be one of: save, serve, select`); + } + + on("Target.targetInfoChanged", attachToTarget); + on("Target.targetInfoChanged", updateTargetInfo); + on("Target.targetInfoChanged", indexURL); + on("Target.attachedToTarget", installForSession); + on("Page.loadEventFired", reloadIfNotLive); + on("Fetch.requestPaused", cacheRequest); + on("Runtime.consoleAPICalled", handleMessage); + + await send("Target.setDiscoverTargets", {discover:true}); + await send("Target.setAutoAttach", {autoAttach:true, waitForDebuggerOnStart:false, flatten: true}); + await send("Security.setIgnoreCertificateErrors", {ignore:true}); + await send("Fetch.enable", { + patterns: [ + { + urlPattern: "http*://*", + requestStage + } + ], + }); + + const {targetInfos:targets} = await send("Target.getTargets", {}); + const pageTargets = targets.filter(({type}) => type == 'page').map(targetInfo => ({targetInfo})); + await Promise.all(pageTargets.map(attachToTarget)); + sleep(5000).then(() => Promise.all(pageTargets.map(reloadIfNotLive))); + + State.bookmarkObserver = State.bookmarkObserver || startObservingBookmarkChanges(); + + Status.loaded = true; + + return Status.loaded; + + async function runHandlers(reason, err, {exit = false} = {}) { + console.log('before exit running', exitHandlers, {reason, err}); + while(exitHandlers.length) { + const h = exitHandlers.shift(); + try { + h(); + } catch(e) { + console.warn(`Error in exit handler`, h, e); + } + } + if ( exit ) { + console.log(`Exiting in 3 seconds...`); + await sleep(3000); + process.exit(0); + } + } + + function handleMessage(args) { + const {type, args:[{value:strVal}]} = args; + if ( type == 'info' ) { + try { + const val = JSON.parse(strVal); + // possible messages + const {install, titleChange, textChange} = val; + switch(true) { + case !!install: { + confirmInstall({install}); + } break; + case !!titleChange: { + reindexOnContentChange({titleChange}); + } break; + case !!textChange: { + reindexOnContentChange({textChange}); + } break; + default: { + if ( DEBUG ) { + console.warn(`Unknown message`, strVal); + } + } break; + } + } catch(e) { + DEBUG.verboseSlow && console.info('Not the message we expected to confirm install. This is OK.', {originalMessage:args}); + } + } + } + + function confirmInstall({install}) { + const {sessionId} = install; + if ( ! State.ConfirmedInstalls.has(sessionId) ) { + State.ConfirmedInstalls.add(sessionId); + DEBUG.verboseSlow && console.log({confirmedInstall:install}); + } + } + + async function reindexOnContentChange({titleChange, textChange}) { + const data = titleChange || textChange; + if ( data ) { + const {sessionId} = data; + const latestTargetInfo = clone(await untilHas(Targets, sessionId)); + if ( titleChange ) { + const {currentTitle} = titleChange; + DEBUG.verboseSlow && console.log('Received titleChange', titleChange); + latestTargetInfo.title = currentTitle; + Targets.set(sessionId, latestTargetInfo); + DEBUG.verboseSlow && console.log('Updated stored target info', latestTargetInfo); + } else { + DEBUG.verboseSlow && console.log('Received textChange', textChange); + } + if ( ! dontCache(latestTargetInfo) ) { + DEBUG.verboseSlow && console.log( + `Will reindex because we were told ${titleChange ? 'title' : 'text'} content maybe changed.`, + data + ); + indexURL({targetInfo:latestTargetInfo}); + } + } + } + + function updateTargetInfo({targetInfo}) { + if ( targetInfo.type === 'page' ) { + const sessionId = State.Sessions.get(targetInfo.targetId); + DEBUG.verboseSlow && console.log('Updating target info', targetInfo, sessionId); + if ( sessionId ) { + const existingTargetInfo = Targets.get(sessionId); + // if we have an existing target info for this URL and have saved an updated title + DEBUG.verboseSlow && console.log('Existing target info', existingTargetInfo); + if ( existingTargetInfo && existingTargetInfo.url === targetInfo.url ) { + // keep that title (because targetInfo does not reflect the latest title) + if ( existingTargetInfo.title !== existingTargetInfo.url ) { + DEBUG.verboseSlow && console.log('Setting title to existing', existingTargetInfo); + targetInfo.title = existingTargetInfo.title; + } + } + Targets.set(sessionId, clone(targetInfo)); + } + } + } + + async function reloadIfNotLive({targetInfo, sessionId} = {}) { + if ( Mode == 'serve' ) return; + if ( !targetInfo && !!sessionId ) { + targetInfo = Targets.get(sessionId); + console.log(targetInfo); + } + if ( neverCache(targetInfo?.url) ) return; + const {attached, type} = targetInfo; + if ( attached && type == 'page' ) { + const {url, targetId} = targetInfo; + const sessionId = State.Sessions.get(targetId); + if ( !!sessionId && !State.ConfirmedInstalls.has(sessionId) ) { + DEBUG.verboseSlow && console.log({ + reloadingAsNotConfirmedInstalled:{ + url, + sessionId + }, + confirmedInstalls: State.ConfirmedInstalls + }); + await sleep(600); + send("Page.stopLoading", {}, sessionId); + send("Page.reload", {}, sessionId); + } + } + } + + function neverCache(url) { + try { + url = new URL(url); + return url?.href == "about:blank" || url?.href?.startsWith('chrome') || NEVER_CACHE.has(url.origin); + } catch(e) { + DEBUG.debug && console.warn('Could not form url', url, e); + return true; + } + } + + async function installForSession({sessionId, targetInfo, waitingForDebugger}) { + if ( waitingForDebugger ) { + console.warn(targetInfo); + throw new TypeError(`Target not ready for install`); + } + if ( ! sessionId ) { + throw new TypeError(`installForSession needs a sessionId`); + } + + const {targetId, url} = targetInfo; + + const installUneeded = dontInstall(targetInfo) || + State.Installations.has(sessionId) + ; + + if ( installUneeded ) return; + + DEBUG.verboseSlow && console.log("installForSession running on target " + targetId); + + State.Sessions.set(targetId, sessionId); + Targets.set(sessionId, clone(targetInfo)); + + if ( Mode == 'save' || Mode == 'select' ) { + send("Network.setCacheDisabled", {cacheDisabled:true}, sessionId); + send("Network.setBypassServiceWorker", {bypass:true}, sessionId); + + await send("Runtime.enable", {}, sessionId); + await send("Page.enable", {}, sessionId); + await send("DOMSnapshot.enable", {}, sessionId); + + on("Page.frameNavigated", updateFrameNode); + on("Page.frameAttached", addFrameNode); + // on("Page.frameDetached", updateFrameNodes); // necessary? maybe not + + await send("Page.addScriptToEvaluateOnNewDocument", { + source: getInjection({sessionId}), + worldName: "Context-22120-Indexing" + }, sessionId); + + DEBUG.verboseSlow && console.log("Just request install", targetId, url); + } + + State.Installations.add(sessionId); + + DEBUG.verboseSlow && console.log('Installed sessionId', sessionId); + if ( Mode == 'save' ) { + indexURL({targetInfo}); + } + } + + async function indexURL({targetInfo:info = {}, sessionId, waitingForDebugger} = {}) { + if ( waitingForDebugger ) { + console.warn(info); + throw new TypeError(`Target not ready for install`); + } + if ( Mode == 'serve' ) return; + if ( info.type != 'page' ) return; + if ( ! info.url || info.url == 'about:blank' ) return; + if ( info.url.startsWith('chrome') ) return; + if ( dontCache(info) ) return; + + DEBUG.verboseSlow && console.log('Index URL', info); + + DEBUG.verboseSlow && console.log('Index URL called', info); + + if ( State.Indexing.has(info.targetId) ) return; + State.Indexing.add(info.targetId); + + if ( ! sessionId ) { + sessionId = await untilHas( + State.Sessions, info.targetId, + {timeout: State.crawling && State.crawlTimeout} + ); + } + + if ( !State.Installations.has(sessionId) ) { + await untilHas( + State.Installations, sessionId, + {timeout: State.crawling && State.crawlTimeout} + ); + } + + send("DOMSnapshot.enable", {}, sessionId); + + await sleep(500); + + const flatDoc = await send("DOMSnapshot.captureSnapshot", { + computedStyles: [], + }, sessionId); + const pageText = processDoc(flatDoc).replace(STRIP_CHARS, ' '); + + if ( State.crawling ) { + const has = await untilTrue(() => State.CrawlData.has(info.targetId)); + + const {url} = Targets.get(sessionId); + if ( ! dontCache({url}) ) { + if ( has ) { + const {depth,links} = State.CrawlData.get(info.targetId); + DEBUG.verboseSlow && console.log(info, {depth,links}); + + const {result:{value:{title,links:crawlLinks}}} = await send("Runtime.evaluate", { + expression: `(function () { + return { + links: Array.from( + document.querySelectorAll('a[href].titlelink') + ).map(a => a.href), + title: document.title + }; + }())`, + returnByValue: true + }, sessionId); + + if ( (depth + 1) <= State.crawlDepth ) { + links.length = 0; + links.push(...crawlLinks.map(url => ({url,depth:depth+1}))); + } + if ( logStream ) { + console.log(`Writing ${links.length} entries to ${logName}`); + logStream.cork(); + links.forEach(url => { + logStream.write(`${url}\n`); + }); + logStream.uncork(); + } + console.log(`Just crawled: ${title} (${info.url})`); + } + + if ( ! State.titles ) { + State.titles = new Map(); + State.onExit.addHandler(() => { + Fs.writeFileSync( + Path.resolve(args.CONFIG_DIR, `titles-${(new Date).toISOString()}.txt`), + JSON.stringify([...State.titles.entries()], null, 2) + '\n' + ); + }); + } + + const {result:{value:data}} = await send("Runtime.evaluate", + { + expression: `(function () { + return { + url: document.location.href, + title: document.title, + }; + }())`, + returnByValue: true + }, + sessionId + ); + + State.titles.set(data.url, data.title); + console.log(`Saved ${State.titles.size} titles`); + + if ( State.program && ! dontCache(info) ) { + const targetInfo = info; + const fs = Fs; + const path = Path; + try { + await sleep(500); + await eval(`(async () => { + try { + ${State.program} + } catch(e) { + console.warn('Error in program', e, State.program); + } + })();`); + await sleep(500); + } catch(e) { + console.warn(`Error evaluate program`, e); + } + } + } + } + + const {title, url} = Targets.get(sessionId); + let id, ndx_id; + if ( State.Index.has(url) ) { + ({ndx_id, id} = State.Index.get(url)); + } else { + Id++; + id = Id; + } + const doc = toNDXDoc({id, url, title, pageText}); + State.Index.set(url, {date:Date.now(),id:doc.id, ndx_id:doc.ndx_id, title}); + State.Index.set(doc.id, url); + State.Index.set('ndx'+doc.ndx_id, url); + + const contentSignature = getContentSig(doc); + + //Flex code + Flex.update(doc.id, contentSignature); + + //New NDX code + NDX_FTSIndex.update(doc, ndx_id); + + // Fuzzy + // eventually we can use this update logic for everyone + let updateFuzz = true; + if ( State.Docs.has(url) ) { + const current = State.Docs.get(url); + if ( current.contentSignature === contentSignature ) { + updateFuzz = false; + } + } + if ( updateFuzz ) { + doc.contentSignature = contentSignature; + fuzzy.add(doc); + State.Docs.set(url, doc); + DEBUG.verboseSlow && console.log({updateFuzz: {doc,url}}); + } + + DEBUG.verboseSlow && console.log("NDX updated", doc.ndx_id); + + UpdatedKeys.add(url); + + DEBUG.verboseSlow && console.log({id: doc.id, title, url, indexed: true}); + + State.Indexing.delete(info.targetId); + State.CrawlIndexing.delete(info.targetId); + } + + async function attachToTarget({targetInfo}) { + if ( dontInstall(targetInfo) ) return; + const {url} = targetInfo; + if ( url && targetInfo.type == 'page' ) { + if ( ! targetInfo.attached ) { + const {sessionId} = await send("Target.attachToTarget", { + targetId: targetInfo.targetId, + flatten: true + }); + State.Sessions.set(targetInfo.targetId, sessionId); + } + } + } + + async function cacheRequest(pausedRequest) { + const { + requestId, request, resourceType, + frameId, + responseStatusCode, responseHeaders, responseErrorReason + } = pausedRequest; + const isNavigationRequest = resourceType == "Document"; + const isFont = resourceType == "Font"; + + if ( dontCache(request) ) { + DEBUG.verboseSlow && console.log("Not caching", request.url); + return send("Fetch.continueRequest", {requestId}); + } + const key = serializeRequestKey(request); + if ( Mode == 'serve' ) { + if ( State.Cache.has(key) ) { + let {body, responseCode, responseHeaders} = await getResponseData(State.Cache.get(key)); + responseCode = responseCode || 200; + //DEBUG.verboseSlow && console.log("Fulfilling", key, responseCode, responseHeaders, body.slice(0,140)); + DEBUG.verboseSlow && console.log("Fulfilling", key, responseCode, body.slice(0,140)); + await send("Fetch.fulfillRequest", { + requestId, body, responseCode, responseHeaders + }); + } else { + DEBUG.verboseSlow && console.log("Sending cache stub", key); + await send("Fetch.fulfillRequest", { + requestId, ...UNCACHED + }); + } + } else { + let saveIt = false; + if ( Mode == 'select' ) { + const rootFrameURL = getRootFrameURL(frameId); + const frameDescendsFromBookmarkedURLFrame = hasBookmark(rootFrameURL); + saveIt = frameDescendsFromBookmarkedURLFrame; + DEBUG.verboseSlow && console.log({rootFrameURL, frameId, mode, saveIt}); + } else if ( Mode == 'save' ) { + saveIt = true; + } + if ( saveIt ) { + const response = {key, responseCode: responseStatusCode, responseHeaders}; + const resp = await getBody({requestId, responseStatusCode}); + if ( resp ) { + let {body, base64Encoded} = resp; + if ( ! base64Encoded ) { + body = b64(body); + } + response.body = body; + const responsePath = await saveResponseData(key, request.url, response); + State.Cache.set(key, responsePath); + } else { + DEBUG.verboseSlow && console.warn("get response body error", key, responseStatusCode, responseHeaders, pausedRequest.responseErrorReason); + response.body = ''; + } + //await sleep(DELAY); + if ( !isFont && responseErrorReason ) { + if ( isNavigationRequest ) { + await send("Fetch.fulfillRequest", { + requestId, + responseHeaders: BLOCKED_HEADERS, + responseCode: BLOCKED_CODE, + body: Buffer.from(responseErrorReason).toString("base64"), + }, + ); + } else { + await send("Fetch.failRequest", { + requestId, + errorReason: responseErrorReason + }, + ); + } + return; + } + } + try { + await send("Fetch.continueRequest", { + requestId, + }, + ); + } catch(e) { + console.warn("Issue with continuing request", e); + } + } + } + + async function getBody({requestId, responseStatusCode}) { + let resp; + if ( ! BODYLESS.has(responseStatusCode) ) { + resp = await send("Fetch.getResponseBody", {requestId}); + } else { + resp = {body:'', base64Encoded:true}; + } + return resp; + } + + function dontInstall(targetInfo) { + return targetInfo.type !== 'page'; + } + + async function getResponseData(path) { + try { + return JSON.parse(await Fs.promises.readFile(path)); + } catch(e) { + console.warn(`Error with ${path}`, e); + return UNCACHED; + } + } + + async function saveResponseData(key, url, response) { + const origin = (new URL(url).origin); + let originDir = State.Cache.get(origin); + if ( ! originDir ) { + originDir = Path.resolve(library_path(), origin.replace(TBL, '_')); + try { + await Fs.promises.mkdir(originDir, {recursive:true}); + } catch(e) { + console.warn(`Issue with origin directory ${Path.dirname(responsePath)}`, e); + } + State.Cache.set(origin, originDir); + } + + const fileName = `${await hasha(key, HASH_OPTS)}.json`; + + const responsePath = Path.resolve(originDir, fileName); + await Fs.promises.writeFile(responsePath, JSON.stringify(response,null,2)); + + return responsePath; + } + + function serializeRequestKey(request) { + const {url, /*urlFragment,*/ method, /*headers, postData, hasPostData*/} = request; + + /** + let sortedHeaders = ''; + for( const key of Object.keys(headers).sort() ) { + sortedHeaders += `${key}:${headers[key]}/`; + } + **/ + + return `${method}${url}`; + //return `${url}${urlFragment}:${method}:${sortedHeaders}:${postData}:${hasPostData}`; + } + + async function startObservingBookmarkChanges() { + console.info("Not observing"); + return; + for await ( const change of bookmarkChanges() ) { + if ( Mode == 'select' ) { + switch(change.type) { + case 'new': { + DEBUG.verboseSlow && console.log(change); + archiveAndIndexURL(change.url); + } break; + case 'delete': { + DEBUG.verboseSlow && console.log(change); + deleteFromIndexAndSearch(change.url); + } break; + default: { + console.log(`We don't do anything about this bookmark change, currently`, change); + } break; + } + } + } + } + } + +// helpers + function neverCache(url) { + return url == "about:blank" || url?.startsWith('chrome') || NEVER_CACHE.has(url); + } + + function dontCache(request) { + if ( ! request.url ) return true; + if ( neverCache(request.url) ) return true; + if ( Mode == 'select' && ! hasBookmark(request.url) ) return true; + const url = new URL(request.url); + return NEVER_CACHE.has(url.origin) || !!(State.No && State.No.test(url.host)); + } + + function processDoc({documents, strings}) { + /* + Info + Implementation Notes + + 1. Code uses spec at: + https://chromedevtools.github.io/devtools-protocol/tot/DOMSnapshot/#type-NodeTreeSnapshot + + 2. Note that so far the below will NOT produce text for and therefore we will NOT + index textarea or input elements. We can access those by using the textValue and + inputValue array properties of the doc, if we want to implement that. + */ + + const texts = []; + for( const doc of documents) { + const textIndices = doc.nodes.nodeType.reduce((Indices, type, index) => { + if ( type === TEXT_NODE ) { + const parentIndex = doc.nodes.parentIndex[index]; + const forbiddenParent = parentIndex >= 0 && + FORBIDDEN_TEXT_PARENT.has(strings[ + doc.nodes.nodeName[ + parentIndex + ] + ]) + if ( ! forbiddenParent ) { + Indices.push(index); + } + } + return Indices; + }, []); + textIndices.forEach(index => { + const stringsIndex = doc.nodes.nodeValue[index]; + if ( stringsIndex >= 0 ) { + const text = strings[stringsIndex]; + texts.push(text); + } + }); + } + + const pageText = texts.filter(t => t.trim()).join(' '); + DEBUG.verboseSlow && console.log('Page text>>>', pageText); + return pageText; + } + + async function isReady() { + return await untilHas(Status, 'loaded'); + } + + async function loadFuzzy({fromMemOnly: fromMemOnly = false} = {}) { + if ( ! fromMemOnly ) { + const fuzzyDocs = Fs.readFileSync(getFuzzyPath()).toString(); + State.Docs = new Map(JSON.parse(fuzzyDocs).map(doc => { + doc.i_url = getURI(doc.url); + doc.contentSignature = getContentSig(doc); + return [doc.url, doc]; + })); + } + State.Fuzzy = fuzzy = new Fuzzy({source: [...State.Docs.values()], keys: FUZZ_OPTS.keys}); + DEBUG.verboseSlow && console.log('Fuzzy loaded'); + } + + function getContentSig(doc) { + return doc.title + ' ' + doc.title + ' ' + doc.content + ' ' + getURI(doc.url); + } + + function getURI(url) { + return url.split(URI_SPLIT).join(' '); + } + + function saveFuzzy(basePath) { + const docs = [...State.Docs.values()] + .map(({url, title, content, id}) => ({url, title, content, id})); + if ( docs.length === 0 ) return; + const path = getFuzzyPath(basePath); + Fs.writeFileSync( + path, + JSON.stringify(docs, null, 2) + ); + DEBUG.verboseSlow && console.log(`Wrote fuzzy to ${path}`); + } + + function clearSavers() { + if ( State.saver ) { + clearInterval(State.saver); + State.saver = null; + } + + if ( State.indexSaver ) { + clearTimeout(State.indexSaver); + State.indexSaver = null; + } + + if ( State.ftsIndexSaver ) { + clearTimeout(State.ftsIndexSaver); + State.ftsIndexSaver = null; + } + } + + async function loadFiles() { + let cacheFile = CACHE_FILE(); + let indexFile = INDEX_FILE(); + let ftsDir = FTS_INDEX_DIR(); + let someError = false; + + try { + State.Cache = new Map(JSON.parse(Fs.readFileSync(cacheFile))); + } catch(e) { + console.warn(e+''); + State.Cache = new Map(); + someError = true; + } + + try { + State.Index = new Map(JSON.parse(Fs.readFileSync(indexFile))); + } catch(e) { + console.warn(e+''); + State.Index = new Map(); + someError = true; + } + + try { + const flexBase = getFlexBase(); + Fs.readdirSync(flexBase, {withFileTypes:true}).forEach(dirEnt => { + if ( dirEnt.isFile() ) { + const content = Fs.readFileSync(Path.resolve(flexBase, dirEnt.name)).toString(); + Flex.import(dirEnt.name, JSON.parse(content)); + } + }); + DEBUG.verboseSlow && console.log('Flex loaded'); + } catch(e) { + console.warn(e+''); + someError = true; + } + + try { + loadNDXIndex(NDX_FTSIndex); + } catch(e) { + console.warn(e+''); + someError = true; + } + + try { + await loadFuzzy(); + } catch(e) { + console.warn(e+''); + someError = true; + } + + if ( someError ) { + const rl = readline.createInterface({input, output}); + const question = util.promisify(rl.question).bind(rl); + console.warn('Error reading archive file. Your archive directory is corrupted. We will attempt to patch it so you can use it going forward, but because we replace a missing or corrupt index, cache, or full-text search index files with new blank copies, existing resources already indexed and cached may become inaccessible from your new index. A future version of this software should be able to more completely repair your archive directory, reconnecting and re-existing all cached resources and notifying you about and purging from the index any missing resources.\n'); + console.log('Sorry about this, we are not sure why this happened, but we know this must be very distressing for you.\n'); + console.log(`For your information, the corruped archive directory is at: ${args.getBasePath()}\n`); + console.info('Because this repair as described above is not a perfect solution, we will give you a choice of how to proceed. You have two options: 1) attempt a basic repair that may leave some resources inaccessible from the repaired archive, or 2) do not touch the corrupted archive, but instead create a new fresh blank archive to begin saving to. Which option would you like to proceed with?'); + console.log('1) Basic repair with possible inaccessible pages'); + console.log('2) Leave the corrupt archive untouched, start a new archive'); + let correctAnswer = false; + let newBasePath = ''; + while(!correctAnswer) { + let answer = await question('Which option would you like (1 or 2)? '); + answer = parseInt(answer); + switch(answer) { + case 1: { + console.log('Alright, selecting option 1. Using the existing archive and patching a simple repair.'); + newBasePath = args.getBasePath(); + correctAnswer = true; + } break; + case 2: { + console.log('Alright, selection option 2. Leaving the existing archive along and creating a new, fresh, blank archive.'); + let correctAnswer2 = false; + while( ! correctAnswer2 ) { + try { + newBasePath = Path.resolve(os.homedir(), await question( + 'Please enter a directory name for your new archive.\n' + + `${os.homedir()}/` + )); + correctAnswer2 = true; + } catch(e2) { + console.warn(e2); + console.info('Sorry that was not a valid directory name.'); + await question('enter to continue'); + } + } + correctAnswer = true; + } break; + default: { + correctAnswer = false; + console.log('Sorry, that was not a valid option. Please input 1 or 2.'); + } break; + } + } + console.log('Resetting base path', newBasePath); + args.updateBasePath(newBasePath, {force:true, before: [ + () => Archivist.beforePathChanged(newBasePath, {force:true}) + ]}); + saveFiles({forceSave:true}); + } + + Id = Math.round(State.Index.size / 2) + 3; + NDXId = State.Index.has(NDX_ID_KEY) ? State.Index.get(NDX_ID_KEY) + 1003000 : (Id + 1000000); + if ( !Number.isInteger(NDXId) ) NDXId = Id; + DEBUG.verboseSlow && console.log({firstFreeId: Id, firstFreeNDXId: NDXId}); + + State.SavedCacheFilePath = cacheFile; + State.SavedIndexFilePath = indexFile; + State.SavedFTSIndexDirPath = ftsDir; + DEBUG.verboseSlow && console.log(`Loaded cache key file ${cacheFile}`); + DEBUG.verboseSlow && console.log(`Loaded index file ${indexFile}`); + DEBUG.verboseSlow && console.log(`Need to load FTS index dir ${ftsDir}`); + + try { + if ( !Fs.existsSync(NO_FILE()) ) { + DEBUG.verboseSlow && console.log(`The 'No file' (${NO_FILE()}) does not exist, ignoring...`); + State.No = null; + } else { + State.No = new RegExp(JSON.parse(Fs.readFileSync(NO_FILE)) + .join('|') + .replace(/\./g, '\\.') + .replace(/\*/g, '.*') + .replace(/\?/g, '.?') + ); + } + } catch(e) { + DEBUG.verboseSlow && console.warn('Error compiling regex from No file', e); + State.No = null; + } + } + + function getMode() { return Mode; } + + function saveFiles({useState: useState = false, forceSave:forceSave = false} = {}) { + if ( State.Index.size === 0 ) return; + clearSavers(); + State.Index.set(NDX_ID_KEY, NDXId); + if ( useState ) { + // saves the old cache path + saveCache(State.SavedCacheFilePath); + saveIndex(State.SavedIndexFilePath); + saveFTS(State.SavedFTSIndexDirPath, {forceSave}); + } else { + saveCache(); + saveIndex(); + saveFTS(null, {forceSave}); + } + } + + async function changeMode(mode) { + saveFiles({forceSave:true}); + Mode = mode; + await collect({chrome_port:args.chrome_port, mode}); + DEBUG.verboseSlow && console.log('Mode changed', Mode); + } + + function getDetails(id) { + const url = State.Index.get(id); + const {title} = State.Index.get(url); + const {content} = State.Docs.get(url); + return {url, title, id, content}; + } + + function findOffsets(query, doc, maxLength = 0) { + if ( maxLength ) { + doc = Array.from(doc).slice(0, maxLength).join(''); + } + Object.assign(fuzzy.options, HIGHLIGHT_OPTIONS_FUZZY); + const hl = fuzzy.highlight(doc); + DEBUG.verboseSlow && console.log(query, hl, maxLength); + return hl; + } + + function beforePathChanged(new_path, {force: force = false} = {}) { + const currentBasePath = args.getBasePath(); + if ( !force && (currentBasePath == new_path) ) { + return false; + } + saveFiles({useState:true, forceSave:true}); + // clear all memory cache, index and full text indexes + State.Index.clear(); + State.Cache.clear(); + State.Docs.clear(); + State.NDX_FTSIndex = NDX_FTSIndex = new NDXIndex(NDX_FIELDS); + State.Flex = Flex = new FTSIndex(FLEX_OPTS); + State.fuzzy = fuzzy = new Fuzzy({source: [...State.Docs.values()], keys: FUZZ_OPTS.keys}); + return true; + } + + async function afterPathChanged() { + DEBUG.verboseSlow && console.log({libraryPathChange:args.library_path()}); + saveFiles({useState:true, forceSave:true}); + // reloads from new path and updates Saved FilePaths + await loadFiles(); + } + + function saveCache(path) { + //DEBUG.verboseSlow && console.log("Writing to", path || CACHE_FILE()); + if ( State.Cache.size === 0 ) return; + Fs.writeFileSync(path || CACHE_FILE(), JSON.stringify([...State.Cache.entries()],null,2)); + } + + function saveIndex(path) { + if ( State.saveInProgress || Mode == 'serve' ) return; + if ( State.Index.size === 0 ) return; + State.saveInProgress = true; + + clearTimeout(State.indexSaver); + + DEBUG.verboseSlow && console.log( + `INDEXLOG: Writing Index (size: ${State.Index.size}) to`, path || INDEX_FILE() + ); + //DEBUG.verboseSlow && console.log([...State.Index.entries()].sort(SORT_URLS)); + Fs.writeFileSync( + path || INDEX_FILE(), + JSON.stringify([...State.Index.entries()].sort(SORT_URLS),null,2) + ); + + State.indexSaver = setTimeout(saveIndex, 11001); + + State.saveInProgress = false; + } + + function getIndex() { + const idx = JSON.parse(Fs.readFileSync(INDEX_FILE())) + .filter(([key]) => typeof key === 'string' && !hiddenKey(key)) + .sort(([,{date:a}], [,{date:b}]) => b-a); + DEBUG.verboseSlow && console.log(idx); + return idx; + } + + async function deleteFromIndexAndSearch(url) { + if ( State.Index.has(url) ) { + const {id, ndx_id, title, /*date,*/} = State.Index.get(url); + // delete index entries + State.Index.delete(url); + State.Index.delete(id); + State.Index.delete('ndx'+ndx_id); + // delete FTS entries (where we can) + State.NDX_FTSIndex.remove(ndx_id); + State.Flex.remove(id); + State.Docs.delete(url); + // save it all (to ensure we don't load data from disk that contains delete entries) + saveFiles({forceSave:true}); + // and just rebuild the whole FTS index (where we must) + await loadFuzzy({fromMemOnly:true}); + return {title}; + } + } + + async function search(query) { + const flex = (await Flex.searchAsync(query, args.results_per_page)) + .map(id=> ({id, url: State.Index.get(id)})); + const ndx = NDX_FTSIndex.search(query) + .map(r => ({ + ndx_id: r.key, + url: State.Index.get('ndx'+r.key), + score: r.score + })); + Object.assign(fuzzy.options, REGULAR_SEARCH_OPTIONS_FUZZY); + const fuzzRaw = fuzzy.search(query); + const fuzz = processFuzzResults(fuzzRaw); + + const results = combineResults({flex, ndx, fuzz}); + //console.log({flex,ndx,fuzz}); + const ids = new Set(results); + + const HL = new Map(); + const highlights = fuzzRaw.filter(({id}) => ids.has(id)).map(obj => { + const title = State.Index.get(obj.url)?.title; + return { + id: obj.id, + url: Archivist.findOffsets(query, obj.url, MAX_URL_LENGTH) || obj.url, + title: Archivist.findOffsets(query, title, MAX_TITLE_LENGTH) || title, + }; + }); + highlights.forEach(hl => HL.set(hl.id, hl)); + + return {query,results, HL}; + } + + function combineResults({flex,ndx,fuzz}) { + DEBUG.verboseSlow && console.log({flex,ndx,fuzz}); + const score = {}; + flex.forEach(countRank(score)); + ndx.forEach(countRank(score)); + fuzz.forEach(countRank(score)); + DEBUG.verboseSlow && console.log(score); + + const results = [...Object.values(score)].map(obj => { + try { + const {id} = State.Index.get(obj.url); + obj.id = id; + return obj; + } catch(e) { + console.log({obj, index:State.Index, e, ndx, flex, fuzz}); + return obj; + } + }); + results.sort(({score:scoreA}, {score:scoreB}) => scoreB-scoreA); + DEBUG.verboseSlow && console.log(results); + const resultIds = results.map(({id}) => id).filter(v => !!v); + return resultIds; + } + + function countRank(record, weight = 1.0) { + return ({url, score:res_score = 1.0}, rank, all) => { + let result = record[url]; + if ( ! result ) { + result = record[url] = { + url, + score: 0 + }; + } + + result.score += res_score*weight*(all.length - rank)/all.length + }; + } + + function processFuzzResults(docs) { + const docIds = docs.map(({id}) => id); + const uniqueIds = new Set(docIds); + return [...uniqueIds.keys()].map(id => ({id, url:State.Index.get(id)})); + } + + async function saveFTS(path = undefined, {forceSave:forceSave = false} = {}) { + if ( State.ftsSaveInProgress || Mode == 'serve' ) return; + State.ftsSaveInProgress = true; + + clearTimeout(State.ftsIndexSaver); + + DEBUG.verboseSlow && console.log("Writing FTS index to", path || FTS_INDEX_DIR()); + const dir = path || FTS_INDEX_DIR(); + + if ( forceSave || UpdatedKeys.size ) { + DEBUG.verboseSlow && console.log(`${UpdatedKeys.size} keys updated since last write`); + const flexBase = getFlexBase(dir); + Flex.export((key, data) => { + key = key.split('.').pop(); + try { + Fs.writeFileSync( + Path.resolve(flexBase, key), + JSON.stringify(data, null, 2) + ); + } catch(e) { + console.error('Error writing full text search index', e); + } + }); + DEBUG.verboseSlow && console.log(`Wrote Flex to ${flexBase}`); + NDX_FTSIndex.save(dir); + saveFuzzy(dir); + UpdatedKeys.clear(); + } else { + DEBUG.verboseSlow && console.log("No FTS keys updated, no writes needed this time."); + } + + State.ftsIndexSaver = setTimeout(saveFTS, 31001); + State.ftsSaveInProgress = false; + } + + function shutdown(then) { + DEBUG.verboseSlow && console.log(`Archivist shutting down...`); + saveFiles({forceSave:true}); + Close && Close(); + DEBUG.verboseSlow && console.log(`Archivist shut down.`); + return then && then(); + } + + function b64(s) { + return Buffer.from(s).toString('base64'); + } + + function NDXIndex(fields) { + let retVal; + + // source: + // adapted from: + // https://github.com/ndx-search/docs/blob/94530cbff6ae8ea66c54bba4c97bdd972518b8b4/README.md#creating-a-simple-indexer-with-a-search-function + + if ( ! new.target ) { throw `NDXIndex must be called with 'new'`; } + + // `createIndex()` creates an index data structure. + // First argument specifies how many different fields we want to index. + const index = NDX(fields.length); + // `fieldAccessors` is an array with functions that used to retrieve data from different fields. + const fieldAccessors = fields.map(f => doc => doc[f.name]); + const fieldBoostFactors = fields.map(f => f.boost); + + retVal = { + index, + // `add()` function will add documents to the index. + add: doc => ndx( + retVal.index, + fieldAccessors, + // Tokenizer is a function that breaks text into words, phrases, symbols, or other meaningful elements + // called tokens. + // Lodash function `words()` splits string into an array of its words, see https://lodash.com/docs/#words for + // details. + words, + // Filter is a function that processes tokens and returns terms, terms are used in Inverted Index to + // index documents. + termFilter, + // Document key, it can be a unique document id or a refernce to a document if you want to store all documents + // in memory. + doc.ndx_id, + // Document. + doc, + ), + remove: id => { + removeDocumentFromIndex(retVal.index, NDXRemoved, id); + maybeClean(); + }, + update: (doc, old_id) => { + retVal.remove(old_id); + retVal.add(doc); + }, + // `search()` function will be used to perform queries. + search: q => NDXQuery( + retVal.index, + fieldBoostFactors, + // BM25 ranking function constants: + 1.2, // BM25 k1 constant, controls non-linear term frequency normalization (saturation). + 0.75, // BM25 b constant, controls to what degree document length normalizes tf values. + words, + termFilter, + // Set of removed documents, in this example we don't want to support removing documents from the index, + // so we can ignore it by specifying this set as `undefined` value. + NDXRemoved, + q, + ), + save: (basePath) => { + maybeClean(true); + const obj = toSerializable(retVal.index); + const objStr = JSON.stringify(obj, null, 2); + const path = getNDXPath(basePath); + Fs.writeFileSync( + path, + objStr + ); + DEBUG.verboseSlow && console.log("Write NDX to ", path); + }, + load: newIndex => { + retVal.index = newIndex; + } + }; + + DEBUG.verboseSlow && console.log('ndx setup', {retVal}); + return retVal; + + function maybeClean(doIt = false) { + if ( (doIt && NDXRemoved.size) || NDXRemoved.size >= REMOVED_CAP_TO_VACUUM_NDX ) { + vacuumIndex(retVal.index, NDXRemoved); + } + } + } + + function loadNDXIndex(ndxFTSIndex) { + if ( Fs.existsSync(getNDXPath()) ) { + const indexContent = Fs.readFileSync(getNDXPath()).toString(); + const index = fromSerializable(JSON.parse(indexContent)); + ndxFTSIndex.load(index); + } + DEBUG.verboseSlow && console.log('NDX loaded'); + } + + function toNDXDoc({id, url, title, pageText}) { + // use existing defined id or a new one + return { + id, + ndx_id: NDXId++, + url, + i_url: getURI(url), + title, + content: pageText + }; + } + + function ndxDocFields({namesOnly:namesOnly = false} = {}) { + if ( !namesOnly && !NDX_OLD ) { + /* old format (for newer ndx >= v1 ) */ + return [ + /* we index over the special indexable url field, not the regular url field */ + { name: "title", boost: 1.3 }, + { name: "i_url", boost: 1.15 }, + { name: "content", boost: 1.0 }, + ]; + } else { + /* new format (for older ndx ~ v0.4 ) */ + return [ + "title", + "i_url", + "content" + ]; + } + } + + async function untilHas(thing, key, {timeout: timeout = false} = {}) { + if ( thing instanceof Map ) { + if ( thing.has(key) ) { + return thing.get(key); + } else { + let resolve; + const pr = new Promise(res => resolve = res); + const then = Date.now(); + const checker = setInterval(() => { + const now = Date.now(); + if ( thing.has(key) || (timeout && (now-then) >= timeout) ) { + clearInterval(checker); + resolve(thing.get(key)); + } else { + DEBUG.verboseSlow && console.log(thing, "not have", key); + } + }, CHECK_INTERVAL); + + return pr; + } + } else if ( thing instanceof Set ) { + if ( thing.has(key) ) { + return true; + } else { + let resolve; + const pr = new Promise(res => resolve = res); + const then = Date.now(); + const checker = setInterval(() => { + const now = Date.now(); + if ( thing.has(key) || (timeout && (now-then) >= timeout) ) { + clearInterval(checker); + resolve(true); + } else { + DEBUG.verboseSlow && console.log(thing, "not have", key); + } + }, CHECK_INTERVAL); + + return pr; + } + } else if ( typeof thing === "object" ) { + if ( thing[key] ) { + return true; + } else { + let resolve; + const pr = new Promise(res => resolve = res); + const then = Date.now(); + const checker = setInterval(() => { + const now = Date.now(); + if ( thing[key] || (timeout && (now-then) >= timeout) ) { + clearInterval(checker); + resolve(true); + } else { + DEBUG.verboseSlow && console.log(thing, "not have", key); + } + }, CHECK_INTERVAL); + + return pr; + } + } else { + throw new TypeError(`untilHas with thing of type ${thing} is not yet implemented!`); + } + } + + function getNDXPath(basePath) { + return Path.resolve(args.ndx_fts_index_dir(basePath), 'index.ndx'); + } + + function getFuzzyPath(basePath) { + return Path.resolve(args.fuzzy_fts_index_dir(basePath), 'docs.fzz'); + } + + function getFlexBase(basePath) { + return args.flex_fts_index_dir(basePath); + } + + function addFrameNode(observedFrame) { + const {frameId, parentFrameId} = observedFrame; + const node = { + id: frameId, + parentId: parentFrameId, + parent: State.FrameNodes.get(parentFrameId) + }; + + DEBUG.verboseSlow && console.log({observedFrame}); + + State.FrameNodes.set(node.id, node); + + return node; + } + + function updateFrameNode(frameNavigated) { + const { + frame: { + id: frameId, + parentId, url: rawUrl, urlFragment, + /* + domainAndRegistry, unreachableUrl, + adFrameStatus + */ + } + } = frameNavigated; + const url = urlFragment?.startsWith(rawUrl.slice(0,4)) ? urlFragment : rawUrl; + let frameNode = State.FrameNodes.get(frameId); + + DEBUG.verboseSlow && console.log({frameNavigated}); + + if ( ! frameNode ) { + // Note + // This is not actually a panic because + // it can happen. It may just mean + // this isn't a sub frame. + // So rather than panicing: + /* + throw new TypeError( + `Sanity check failed: frameId ${ + frameId + } is not in our FrameNodes data, which currently has ${ + State.FrameNodes.size + } entries.` + ); + */ + // We do this instead (just add it): + frameNode = addFrameNode({frameId, parentFrameId: parentId}); + } + + if ( frameNode.id !== frameId ) { + throw new TypeError( + `Sanity check failed: Child frameId ${ + frameNode.frameId + } was supposed to be ${ + frameId + }` + ); + } + + // Note: + // use the urlFragment (a URL + the hash fragment identifier) + // only if it's actually a URL + + // Update frame node url (and possible parent) + frameNode.url = url; + if ( parentId !== frameNode.parentId ) { + console.info(`Interesting. Frame parent changed from ${frameNode.parentId} to ${parentId}`); + frameNode.parentId = parentId; + frameNode.parent = State.FrameNodes.get(parentId); + if ( parentId && !frameNode.parent ) { + throw new TypeError( + `!! FrameNode ${ + frameId + } uses parentId ${ + parentId + } but we don't have any record of ${ + parentId + } in out FrameNodes data` + ); + } + } + + // comment out these details but reserve for possible future use + /* + frameNode.detail = { + unreachableUrl, urlFragment, + domainAndRegistry, adFrameStatus + }; + */ + } + + /* + function removeFrameNode(frameDetached) { + const {frameId, reason} = frameDetached; + throw new TypeError(`removeFrameNode is not implemented`); + } + */ + + function getRootFrameURL(frameId) { + let frameNode = State.FrameNodes.get(frameId); + if ( ! frameNode ) { + DEBUG.verboseSlow && console.warn(new TypeError( + `Sanity check failed: frameId ${ + frameId + } is not in our FrameNodes data, which currently has ${ + State.FrameNodes.size + } entries.` + )); + return; + } + if ( frameNode.id !== frameId ) { + throw new TypeError( + `Sanity check failed: Child frameId ${ + frameNode.id + } was supposed to be ${ + frameId + }` + ); + } + while(frameNode.parent) { + frameNode = frameNode.parent; + } + return frameNode.url; + } + +// crawling + async function archiveAndIndexURL(url, { + crawl, + createIfMissing:createIfMissing = false, + timeout, + depth, + TargetId, + program, + } = {}) { + DEBUG.verboseSlow && console.log('ArchiveAndIndex', url, {crawl, createIfMissing, timeout, depth, TargetId, program}); + if ( Mode == 'serve' ) { + throw new TypeError(`archiveAndIndexURL can not be used in 'serve' mode.`); + } + if ( program ) { + State.program = program; + } + let targetId = TargetId; + let sessionId; + if ( ! dontCache({url}) ) { + const {send, on, close} = State.connection; + const {targetInfos:targs} = await send("Target.getTargets", {}); + const targets = targs.reduce((M,T) => { + M.set(T.url, T); + M.set(T.targetId, T); + return M; + }, new Map); + DEBUG.verboseSlow && console.log('Targets', targets); + if ( targets.has(url) || targets.has(targetId) ) { + DEBUG.verboseSlow && console.log('We have target', url, targetId); + const targetInfo = targets.get(url) || targets.get(targetId); + ({targetId} = targetInfo); + if ( crawl && ! State.CrawlData.has(targetId) ) { + State.CrawlIndexing.add(targetId) + State.CrawlData.set(targetId, {depth, links:[]}); + if ( State.visited.has(url) ) { + return []; + } else { + State.visited.add(url); + } + } + sessionId = State.Sessions.get(targetId); + DEBUG.verboseSlow && console.log( + "Reloading to archive and index in select (Bookmark) mode", + url + ); + if ( State.program && ! dontCache(targetInfo) ) { + const fs = Fs; + const path = Path; + try { + await sleep(500); + await eval(`(async () => { + try { + ${State.program} + } catch(e) { + console.warn('Error in program', e, State.program); + } + })();`); + await sleep(500); + } catch(e) { + console.warn(`Error evaluate program`, e); + } + } + + await untilTrue(async () => { + const {result:{value:loaded}} = await send("Runtime.evaluate", { + expression: `(function () { + return document.readyState === 'complete'; + }())`, + returnByValue: true + }, sessionId); + DEBUG.verboseSlow && console.log({loaded, targetInfo}); + return loaded; + }); + //send("Page.stopLoading", {}, sessionId); + send("Page.reload", {}, sessionId); + if ( crawl ) { + let resolve; + const pageLoaded = new Promise(res => resolve = res).then(() => sleep(1000)); + { + on("Page.loadEventFired", resolve); + //console.log(targets, targetId, targets.get(targetId)); + const {result:{value:loaded}} = await send("Runtime.evaluate", { + expression: `(function () { + return document.readyState === 'complete'; + }())`, + returnByValue: true + }, sessionId); + if ( loaded ) { + resolve(true); + } + } + let notifyStable; + const pageHTMLStabilized = new Promise(res => notifyStable = res); + setTimeout(async () => { + const timeout = MAX_TIME_PER_PAGE / 4; + const checkDurationMsecs = 1618; + const maxChecks = timeout / checkDurationMsecs; + let lastSize = 0; + let checkCounts = 1; + let countStableSizeIterations = 0; + const minStableSizeIterations = 3; + + while(checkCounts++ <= maxChecks) { + const flatDoc = await send("DOMSnapshot.captureSnapshot", { + computedStyles: [], + }, sessionId); + const pageText = processDoc(flatDoc).replace(STRIP_CHARS, ' '); + const currentSize = pageText.length; + + if(lastSize != 0 && currentSize == lastSize) + countStableSizeIterations++; + else + countStableSizeIterations = 0; //reset the counter + + if(countStableSizeIterations >= minStableSizeIterations) { + notifyStable(true); + } + + lastSize = currentSize; + await sleep(checkDurationMsecs); + } + + notifyStable(false); + }, 0); + + await pageLoaded; + + if ( State.program && ! dontCache(targetInfo) ) { + const fs = Fs; + const path = Path; + try { + await sleep(500); + await eval(`(async () => { + try { + ${State.program} + } catch(e) { + console.warn('Error in program', e, State.program); + } + })();`); + await sleep(500); + } catch(e) { + console.warn(`Error evaluate program`, e); + } + } + + await Promise.race([ + await Promise.all([ + pageHTMLStabilized, + untilTrue(() => !State.CrawlIndexing.has(targetId), timeout/5, timeout), + sleep(State.minPageCrawlTime || MIN_TIME_PER_PAGE) + ]), + sleep(State.maxPageCrawlTime || MAX_TIME_PER_PAGE) + ]); + + console.log(`Closing page ${url}, at target ${targetId}`); + + await send("Target.closeTarget", {targetId}); + State.CrawlTargets.delete(targetId); + } + } else if ( createIfMissing ) { + DEBUG.verboseSlow && console.log('We create target', url); + try { + targetId = null; + ({targetId} = await send("Target.createTarget", { + url: `${GO_SECURE ? 'https' : 'http'}://localhost:${args.server_port}/redirector.html?url=${ + encodeURIComponent(url) + }` + })); + } catch(e) { + console.warn("Error creating new tab for url", url, e); + return; + } + if ( crawl && ! State.CrawlData.has(targetId) ) { + State.CrawlTargets.add(targetId); + State.CrawlIndexing.add(targetId); + State.CrawlData.set(targetId, {depth, links:[]}); + } + return archiveAndIndexURL(url, { + crawl, timeout, depth, createIfMissing: false, /* prevent redirect loops */ + TargetId: targetId, + program, + }); + } + } else { + DEBUG.verboseSlow && console.warn( + `archiveAndIndexURL called in mode ${ + Mode + } for URL ${ + url + } but that URL is not in our Bookmarks list.` + ); + } + if ( crawl && State.CrawlData.has(targetId) ) { + const {links} = State.CrawlData.get(targetId); + console.log({targetId,links}); + State.CrawlData.delete(targetId); + return links; + } else { + return []; + } + } + + export async function startCrawl({ + urls, timeout, depth, saveToFile: saveToFile = false, + batchSize, + minPageCrawlTime, + maxPageCrawlTime, + program, + } = {}) { + if ( State.crawling ) { + console.log('Already crawling...'); + return; + } + if ( saveToFile ) { + logName = `crawl-${(new Date).toISOString()}.urls.txt`; + logStream = Fs.createWriteStream(Path.resolve(args.CONFIG_DIR, logName), {flags:'as+'}); + } + console.log('StartCrawl', {urls, timeout, depth, batchSize, saveToFile, minPageCrawlTime, maxPageCrawlTime, program}); + State.crawling = true; + State.crawlDepth = depth; + State.crawlTimeout = timeout; + State.visited = new Set(); + Object.assign(State,{ + batchSize, + minPageCrawlTime, + maxPageCrawlTime + }); + const batch_sz = State.batchSize || BATCH_SIZE; + let totalBytes = 0; + setTimeout(async () => { + try { + while(urls.length >= batch_sz) { + const jobs = []; + const batch = urls.splice(urls.length-batch_sz,batch_sz); + console.log({urls, batch}); + for( let i = 0; i < batch_sz; i++ ) { + const {depth,url} = batch.shift(); + if ( url.startsWith('https://news.ycombinator') ) { + await sleep(1618); + } + const pr = archiveAndIndexURL( + url, + {crawl: true, depth, timeout, createIfMissing:true, getLinks: depth >= 1, program} + ); + jobs.push(pr); + } + const links = (await Promise.all(jobs)).flat().filter(({url}) => !Q.has(url)); + if ( links.length ) { + urls.push(...links); + links.forEach(({url}) => Q.add(url)); + } + } + while(urls.length) { + const {depth,url} = urls.pop(); + if ( url.startsWith('https://news.ycombinator') ) { + await sleep(1618); + } + const links = (await archiveAndIndexURL( + url, + {crawl: true, depth, timeout, createIfMissing:true, getLinks: depth >= 1, program} + )).filter(({url}) => !Q.has(url)); + console.log(links, Q); + if ( links.length ) { + urls.push(...links); + links.forEach(({url}) => Q.add(url)); + } + } + } catch(e) { + console.warn(e); + throw new RichError({status:500, message: e.message}); + } finally { + await untilTrue(() => State.CrawlData.size === 0 && State.CrawlTargets.size === 0, -1) + State.crawling = false; + State.crawlDepth = false; + State.crawlTimeout = false; + State.visited = false; + if ( saveToFile ) { + logStream.close(); + totalBytes = logStream.bytesWritten; + console.log(`Wrote ${totalBytes} bytes of URLs to ${logName}`); + } + console.log(`Crawl finished.`); + } + }, 0); + } diff --git a/src/args.js b/src/args.js new file mode 100644 index 0000000..23639d1 --- /dev/null +++ b/src/args.js @@ -0,0 +1,157 @@ +import os from 'os'; +import path from 'path'; +import fs from 'fs'; + +const server_port = process.env.PORT || process.argv[2] || 22120; +const mode = process.argv[3] || 'save'; +const chrome_port = process.argv[4] || 9222; + +const Pref = {}; +export const CONFIG_DIR = path.resolve(os.homedir(), '.config', 'dosyago', 'DiskerNet'); +fs.mkdirSync(CONFIG_DIR, {recursive:true}); +const pref_file = path.resolve(CONFIG_DIR, 'config.json'); +const cacheId = Math.random(); + +loadPref(); + +let BasePath = Pref.BasePath; +export const archive_root = () => path.resolve(BasePath, '22120-arc'); +export const no_file = () => path.resolve(archive_root(), 'no.json'); +export const temp_browser_cache = () => path.resolve(archive_root(), 'temp-browser-cache' + cacheId); +export const library_path = () => path.resolve(archive_root(), 'public', 'library'); +export const cache_file = () => path.resolve(library_path(), 'cache.json'); +export const index_file = () => path.resolve(library_path(), 'index.json'); +export const fts_index_dir = () => path.resolve(library_path(), 'fts'); + +const flex_fts_index_dir = base => path.resolve(base || fts_index_dir(), 'flex'); +const ndx_fts_index_dir = base => path.resolve(base || fts_index_dir(), 'ndx'); +const fuzzy_fts_index_dir = base => path.resolve(base || fts_index_dir(), 'fuzzy'); + +const results_per_page = 10; + +console.log(`Args usage: `); + +updateBasePath(process.argv[5] || Pref.BasePath || CONFIG_DIR); + +const args = { + mode, + + server_port, + chrome_port, + + updateBasePath, + getBasePath, + + library_path, + no_file, + temp_browser_cache, + cache_file, + index_file, + fts_index_dir, + flex_fts_index_dir, + ndx_fts_index_dir, + fuzzy_fts_index_dir, + + results_per_page, + CONFIG_DIR +}; + +export default args; + +function updateBasePath(new_base_path, {force:force = false, before: before = []} = {}) { + new_base_path = path.resolve(new_base_path); + if ( !force && (BasePath == new_base_path) ) { + return false; + } + + console.log(`Updating base path from ${BasePath} to ${new_base_path}...`); + BasePath = new_base_path; + + if ( Array.isArray(before) ) { + for( const task of before ) { + try { task(); } catch(e) { + console.error(`before updateBasePath task failed. Task: ${task}`); + } + } + } else { + throw new TypeError(`If given, argument before to updateBasePath() must be an array of functions.`); + } + + if ( !fs.existsSync(library_path()) ) { + console.log(`Archive directory (${library_path()}) does not exist, creating...`); + fs.mkdirSync(library_path(), {recursive:true}); + console.log(`Created.`); + } + + if ( !fs.existsSync(cache_file()) ) { + console.log(`Cache file does not exist, creating...`); + fs.writeFileSync(cache_file(), JSON.stringify([])); + console.log(`Created!`); + } + + if ( !fs.existsSync(index_file()) ) { + //console.log(`INDEXLOG: Index file does not exist, creating...`); + fs.writeFileSync(index_file(), JSON.stringify([])); + console.log(`Created!`); + } + + if ( !fs.existsSync(flex_fts_index_dir()) ) { + console.log(`FTS Index directory does not exist, creating...`); + fs.mkdirSync(flex_fts_index_dir(), {recursive:true}); + console.log(`Created!`); + } + + if ( !fs.existsSync(ndx_fts_index_dir()) ) { + console.log(`NDX FTS Index directory does not exist, creating...`); + fs.mkdirSync(ndx_fts_index_dir(), {recursive:true}); + console.log(`Created!`); + } + + if ( !fs.existsSync(fuzzy_fts_index_dir()) ) { + console.log(`FUZZY FTS Index directory does not exist, creating...`); + fs.mkdirSync(fuzzy_fts_index_dir(), {recursive:true}); + fs.writeFileSync(path.resolve(fuzzy_fts_index_dir(), 'docs.fzz'), JSON.stringify([])); + console.log('Also creating FUZZY FTS Index docs file...'); + console.log(`Created all!`); + } + + + + console.log(`Base path updated to: ${BasePath}. Saving to preferences...`); + Pref.BasePath = BasePath; + savePref(); + console.log(`Saved!`); + + return true; +} + +function getBasePath() { + return BasePath; +} + +export function loadPref() { + if ( fs.existsSync(pref_file) ) { + try { + Object.assign(Pref, JSON.parse(fs.readFileSync(pref_file))); + } catch(e) { + console.warn("Error reading from preferences file", e); + } + } else { + console.log("Preferences file does not exist. Creating one..."); + savePref(); + } + return clone(Pref); +} + +function savePref() { + try { + fs.writeFileSync(pref_file, JSON.stringify(Pref,null,2)); + } catch(e) { + console.warn("Error writing preferences file", pref_file, Pref, e); + } +} + +function clone(o) { + return JSON.parse(JSON.stringify(o)); +} + diff --git a/src/blockedResponse.js b/src/blockedResponse.js new file mode 100644 index 0000000..f56aa7f --- /dev/null +++ b/src/blockedResponse.js @@ -0,0 +1,29 @@ +export const BLOCKED_CODE = 200; +export const BLOCKED_BODY = Buffer.from(` + +

Request blocked

+

This navigation was prevented by 22120 as a Chrome bug fix for some requests causing issues.

+`).toString("base64"); +export const BLOCKED_HEADERS = [ + {name: "X-Powered-By", value: "Dosyago-Corporation"}, + {name: "X-Blocked-Internally", value: "Custom 22120 Chrome bug fix"}, + {name: "Accept-Ranges", value: "bytes"}, + {name: "Cache-Control", value: "public, max-age=0"}, + {name: "Content-Type", value: "text/html; charset=UTF-8"}, + {name: "Content-Length", value: `${BLOCKED_BODY.length}`} +]; + +const BLOCKED_RESPONSE = ` +HTTP/1.1 ${BLOCKED_CODE} OK +X-Powered-By: Zanj-Dosyago-Corporation +X-Blocked-Internally: Custom ad blocking +Accept-Ranges: bytes +Cache-Control: public, max-age=0 +Content-Type: text/html; charset=UTF-8 +Content-Length: ${BLOCKED_BODY.length} + +${BLOCKED_BODY} +`; + +export default BLOCKED_RESPONSE; + diff --git a/src/bookmarker.js b/src/bookmarker.js new file mode 100644 index 0000000..246929c --- /dev/null +++ b/src/bookmarker.js @@ -0,0 +1,336 @@ +import os from 'os'; +import Path from 'path'; +import fs from 'fs'; + +import {DEBUG as debug} from './common.js'; + +const DEBUG = debug || false; +// Chrome user data directories by platform. + // Source 1: https://chromium.googlesource.com/chromium/src/+/HEAD/docs/user_data_dir.md + // Source 2: https://superuser.com/questions/329112/where-are-the-user-profile-directories-of-google-chrome-located-in + +const FS_WATCH_OPTS = { + persistent: false, +}; + +// Note: + // Not all the below are now used or supported by this code +const UDD_PATHS = { + 'win': '%LOCALAPPDATA%\\Google\\Chrome\\User Data', + 'winxp' : '%USERPROFILE%\\Local Settings\\Application Data\\Google\\Chrome\\User Data', + 'macos' : Path.resolve(os.homedir(), 'Library/Application Support/Google/Chrome'), + 'nix' : Path.resolve(os.homedir(), '.config/google-chrome'), + 'chromeos': '/home/chronos', /* no support */ + 'ios': 'Library/Application Support/Google/Chrome', /* no support */ +}; +const PLAT_TABLE = { + 'darwin': 'macos', + 'linux': 'nix' +}; +const PROFILE_DIR_NAME_REGEX = /^(Default|Profile \d+)$/i; +const isProfileDir = name => PROFILE_DIR_NAME_REGEX.test(name); +const BOOKMARK_FILE_NAME_REGEX = /^Bookmarks$/i; +const isBookmarkFile = name => BOOKMARK_FILE_NAME_REGEX.test(name); +const State = { + active: new Set(), /* active Bookmark files (we don't know these until file changes) */ + books: { + + } +}; + +export async function* bookmarkChanges() { + // try to get the profile directory + const rootDir = getProfileRootDir(); + + if ( !fs.existsSync(rootDir) ) { + throw new TypeError(`Sorry! The directory where we thought the Chrome profile directories may be found (${rootDir}), does not exist. We can't monitor changes to your bookmarks, so Bookmark Select Mode is not supported.`); + } + + // state constants and variables (including chokidar file glob observer) + const observers = []; + const ps = []; + let change = false; + let notifyChange = false; + let stopLooping = false; + let shuttingDown = false; + + // create sufficient observers + const dirs = fs.readdirSync(rootDir, {withFileTypes:true}).reduce((Files, dirent) => { + if ( dirent.isDirectory() && isProfileDir(dirent.name) ) { + const filePath = Path.resolve(rootDir, dirent.name); + + if ( fs.existsSync(filePath) ) { + Files.push(filePath); + } + } + return Files; + }, []); + for( const dirPath of dirs ) { + // first read it in + const filePath = Path.resolve(dirPath, 'Bookmarks'); + if ( fs.existsSync(filePath) ) { + const data = fs.readFileSync(filePath); + const jData = JSON.parse(data); + State.books[filePath] = flatten(jData, {toMap:true}); + } + + const observer = fs.watch(dirPath, FS_WATCH_OPTS); + console.log(`Observing ${dirPath}`); + // Note + // allow the parent process to exit + //even if observer is still active somehow + observer.unref(); + + // listen for all events from the observer + observer.on('change', (event, filename) => { + filename = filename || ''; + // listen to everything + const path = Path.resolve(dirPath, filename); + DEBUG.verboseSlow && console.log(event, path); + if ( isBookmarkFile(filename) ) { + if ( ! State.active.has(path) ) { + State.active.add(path); + } + // but only act if it is a bookmark file + DEBUG.verboseSlow && console.log(event, path, notifyChange); + // save the event type and file it happened to + change = {event, path}; + // drop the most recently pushed promise from our bookkeeping list + ps.pop(); + // resolve the promise in the wait loop to process the bookmark file and emit the changes + notifyChange && notifyChange(); + } + }); + observer.on('error', error => { + console.warn(`Bookmark file observer for ${dirPath} error`, error); + observers.slice(observers.indexOf(observer), 1); + if ( observers.length ) { + notifyChange && notifyChange(); + } else { + stopLooping && stopLooping(); + } + }); + observer.on('close', () => { + console.info(`Observer for ${dirPath} closed`); + observers.slice(observers.indexOf(observer), 1); + if ( observers.length ) { + notifyChange && notifyChange(); + } else { + stopLooping && stopLooping(); + } + }); + + observers.push(observer); + } + + // make sure we kill the watcher on process restart or shutdown + process.on('SIGTERM', shutdown); + process.on('SIGHUP', shutdown); + process.on('SIGINT', shutdown); + process.on('SIGBRK', shutdown); + + // the main wait loop that enables us to turn a traditional NodeJS eventemitter + // into an asychronous stream generator + waiting: while(true) { + // Note: code resilience + //the below two statements can come in any order in this loop, both work + + // get, process and publish changes + // only do if the change is there (first time it won't be because + // we haven't yielded out (async or yield) yet) + if ( change ) { + const {path} = change; + change = false; + + try { + const changes = flatten( + JSON.parse(fs.readFileSync(path)), + {toMap:true, map: State.books[path]} + ); + + for( const changeEvent of changes ) yield changeEvent; + } catch(e) { + console.warn(`Error publishing Bookmarks changes`, e); + } + } + + // wait for the next change + // always wait tho (to allow queueing of the next event to process) + try { + await new Promise((res, rej) => { + // save these + notifyChange = res; // so we can turn the next turn of the loop + stopLooping = rej; // so we can break out of the loop (on shutdown) + ps.push({res,rej}); // so we can clean up any left over promises + }); + } catch { + ps.pop(); + break waiting; + } + } + + shutdown(); + + return true; + + async function shutdown() { + if ( shuttingDown ) return; + shuttingDown = true; + console.log('Bookmark observer shutting down...'); + // clean up any outstanding waiting promises + while ( ps.length ) { + /* eslint-disable no-empty */ + try { ps.pop().rej(); } finally {} + /* eslint-enable no-empty */ + } + // stop the waiting loop + stopLooping && setTimeout(() => stopLooping('bookmark watching stopped'), 0); + // clean up any observers + while(observers.length) { + /* eslint-disable no-empty */ + try { observers.pop().close(); } finally {} + /* eslint-enable no-empty */ + } + console.log('Bookmark observer shut down cleanly.'); + } +} + +export function hasBookmark(url) { + return Object.keys(State.books).filter(key => { + if ( State.active.size == 0 ) return true; + return State.active.has(key); + }).map(key => State.books[key]) + .some(map => map.has(url)); +} + +function getProfileRootDir() { + const plat = os.platform(); + let name = PLAT_TABLE[plat]; + let rootDir; + + DEBUG.verboseSlow && console.log({plat, name}); + + if ( !name ) { + if ( plat === 'win32' ) { + // because Chrome profile dir location only changes in XP + // we only care if it's XP or not and so + // we try to resolve based on the version major and minor (given by release) + // source: https://docs.microsoft.com/en-us/windows/win32/sysinfo/operating-system-version?redirectedfrom=MSDN + const rel = os.release(); + const ver = parseFloat(rel); + if ( !Number.isNaN(ver) && ver <= 5.2 ) { + // this should be reliable + name = 'winxp'; + } else { + // this may not be reliable, but we just do it + name = 'win'; + } + } else { + throw new TypeError( + `Sorry! We don't know how to find the default Chrome profile on OS platform: ${plat}` + ); + } + } + + if ( UDD_PATHS[name] ) { + rootDir = Path.resolve(resolveEnvironmentVariablesToPathSegments(UDD_PATHS[name])); + } else { + throw new TypeError( + `Sorry! We don't know how to find the default Chrome profile on OS name: ${name}` + ); + } + + return rootDir; +} + +function flatten(bookmarkObj, {toMap: toMap = false, map} = {}) { + const nodes = [...Object.values(bookmarkObj.roots)]; + const urls = toMap? (map || new Map()) : []; + const urlSet = new Set(); + const changes = []; + + while(nodes.length) { + const next = nodes.pop(); + const {name, type, url} = next; + switch(type) { + case "url": + if ( toMap ) { + if ( map ) { + if ( urls.has(url) ) { + const {name:oldName} = urls.get(url); + if ( name !== oldName ) { + if ( !urlSet.has(url) ) { + changes.push({ + type: "Title updated", + url, + oldName, + name + }); + } + } + } else { + changes.push({ + type: "new", + name, url + }); + } + } + if ( !urlSet.has(url) ) { + urls.set(url, next); + } + urlSet.add(url); + } else { + urls.push(next); + } + break; + case "folder": + nodes.push(...next.children); + break; + default: + console.info("New type", type, next); + break; + + } + } + + if (map) { + [...map.keys()].forEach(url => { + if ( !urlSet.has(url) ) { + changes.push({ + type: "delete", + url + }); + map.delete(url); + } + }); + } + + return map ? changes : urls; +} + +// source: https://stackoverflow.com/a/33017068 +function resolveEnvironmentVariablesToPathSegments(path) { + return path.replace(/%([^%]+)%/g, function(_, key) { + return process.env[key]; + }); +} + +/* +test(); +async function test() { + for await ( const change of bookmarkChanges() ) { + console.log(change); + } +} +*/ + + +/* +function* profileDirectoryEnumerator(maxN = 9999) { + let index = 0; + while(index <= maxN) { + const profileDirName = index ? `Profile ${index}` : `Default`; + yield profileDirName; + } +} +*/ diff --git a/src/common.js b/src/common.js new file mode 100644 index 0000000..f680f4c --- /dev/null +++ b/src/common.js @@ -0,0 +1,115 @@ +import path from 'path'; +import {fileURLToPath} from 'url'; +import fs from 'fs'; +import os from 'os'; + +const DEEB = false; + +export const DEBUG = { + askFirst: true, + verboseSlow: process.env.VERBOSE_DEBUG_22120 || DEEB, + debug: process.env.DEBUG_22120 || DEEB, + checkPred: false +} +export const SHOW_FETCH = false; + +// server related +export const PUBLIC_SERVER = true; + +// crawl related +export const MIN_TIME_PER_PAGE = 10000; +export const MAX_TIME_PER_PAGE = 32000; +export const MIN_WAIT = 200; +export const MAX_WAITS = 300; +export const BATCH_SIZE = 5; // crawl batch size (how many concurrent tabs for crawling) +export const MAX_REAL_URL_LENGTH = 2**15 - 1; + +export const CHECK_INTERVAL = 400; +export const TEXT_NODE = 3; +export const MAX_HIGHLIGHTABLE_LENGTH = 0; /* 0 is no max length for highlight */ +export const MAX_TITLE_LENGTH = 140; +export const MAX_URL_LENGTH = 140; +export const MAX_HEAD = 140; + +export const GO_SECURE = fs.existsSync(path.resolve(os.homedir(), 'local-sslcerts', 'privkey.pem')); + +export class RichError extends Error { + constructor(msg) { + let textMessage; + try { + textMessage = JSON.stringify(msg); + } catch(e) { + console.warn(`Could not create RichError from argument ${msg.toString ? msg.toString() : msg} as JSON serialization failed. RichError argument MUST be JSON serializable. Failure error was:`, e); + return; + } + super(textMessage); + } +} + +/* text nodes inside these elements that are ignored */ +export const FORBIDDEN_TEXT_PARENT = new Set([ + 'STYLE', + 'SCRIPT', + 'NOSCRIPT', + /* we could remove these last two so as to index them as well */ + 'DATALIST', + 'OPTION' +]); +export const ERROR_CODE_SAFE_TO_IGNORE = new Set([ + -32000, /* message: + Can only get response body on requests captured after headers received. + * ignore because: + seems to only happen when new navigation aborts all + pending requests of the unloading page + */ + -32602, /* message: + Invalid InterceptionId. + * ignore because: + seems to only happen when new navigation aborts all + pending requests of the unloading page + */ +]); + +export const SNIP_CONTEXT = 31; + +export const NO_SANDBOX = (process.env.DEBUG_22120 && process.env.SET_22120_NO_SANDBOX) || false; + +//export const APP_ROOT = '.'; +export const APP_ROOT = path.dirname(process.argv[0]); +//export const APP_ROOT = path.dirname(fileURLToPath(import.meta.url)); + +export const sleep = ms => new Promise(res => setTimeout(res, ms)); + +export function say(o) { + console.log(JSON.stringify(o)); +} + +export function clone(o) { + return JSON.parse(JSON.stringify(o)); +} + +export async function untilTrue(pred, waitOverride = MIN_WAIT, maxWaits = MAX_WAITS) { + if ( waitOverride < 0 ) { + maxWaits = -1; + waitOverride = MIN_WAIT; + } + let waitCount = 0; + let resolve; + const pr = new Promise(res => resolve = res); + setTimeout(checkPred, 0); + return pr; + + async function checkPred() { + DEBUG.checkPred && console.log('Checking', pred.toString()); + if ( await pred() ) { + return resolve(true); + } else { + waitCount++; + if ( waitCount < maxWaits || maxWaits < 0 ) { + setTimeout(checkPred, waitOverride); + } else { + resolve(false); + } + } + } +} diff --git a/src/hello.js b/src/hello.js new file mode 100644 index 0000000..9ed889a --- /dev/null +++ b/src/hello.js @@ -0,0 +1 @@ +console.log(`hello...is it me you're looking for?`); diff --git a/src/highlighter.js b/src/highlighter.js new file mode 100644 index 0000000..557d0c6 --- /dev/null +++ b/src/highlighter.js @@ -0,0 +1,384 @@ +import ukkonen from 'ukkonen'; +import {DEBUG} from './common.js'; + +const MAX_ACCEPT_SCORE = 0.5; +const CHUNK_SIZE = 12; + +//testHighlighter(); + +function params(qLength, chunkSize = CHUNK_SIZE) { + const MaxDist = chunkSize; + const MinScore = Math.abs(qLength - chunkSize); + const MaxScore = Math.max(qLength, chunkSize) - MinScore; + return {MaxDist,MinScore,MaxScore}; +} + +export function highlight(query, doc, { + /* 0 is no maxLength */ + maxLength: maxLength = 0, + maxAcceptScore: maxAcceptScore = MAX_ACCEPT_SCORE, + chunkSize: chunkSize = CHUNK_SIZE +} = {}) { + if ( chunkSize % 2 ) { + throw new TypeError(`chunkSize must be even. Was: ${chunkSize} which is odd.`); + } + doc = Array.from(doc); + if ( maxLength ) { + doc = doc.slice(0, maxLength); + } + const highlights = []; + const extra = chunkSize; + // use array from then length rather than string length to + // give accurate length for all unicode + const qLength = Array.from(query).length; + const {MaxDist,MinScore,MaxScore} = params(qLength, chunkSize); + const doc2 = Array.from(doc); + // make doc length === 0 % chunkSize + doc.splice(doc.length, 0, ...(new Array((chunkSize - doc.length % chunkSize) % chunkSize)).join(' ').split('')); + const fragments = doc.reduce(getFragmenter(chunkSize), []); + //console.log(fragments); + // pad start of doc2 by half chunkSize + doc2.splice(0, 0, ...(new Array(chunkSize/2 + 1)).join(' ').split('')); + // make doc2 length === 0 % chunkSize + doc2.splice(doc2.length, 0, ...(new Array((chunkSize - doc2.length % chunkSize) % chunkSize)).join(' ').split('')); + const fragments2 = doc2.reduce(getFragmenter(chunkSize), []); + query.toLocaleLowerCase(); + DEBUG.verboseSlow && console.log(fragments); + + const scores = [...fragments, ...fragments2].map(fragment => { + const distance = ukkonen(query, fragment.text.toLocaleLowerCase(), MaxDist); + // the min score possible = the minimum number of edits between + const scaledScore = (distance - MinScore)/MaxScore; + return {score: scaledScore, fragment}; + }); + + // sort ascending (smallest scores win) + scores.sort(({score:a}, {score:b}) => a-b); + + for( const {score, fragment} of scores ) { + if ( score > maxAcceptScore ) { + break; + } + highlights.push({score,fragment}); + } + + let result; + + if ( highlights.length === 0 ) { + DEBUG.verboseSlow && console.log('Zero highlights, showing first score', scores[0]); + result = scores.slice(0,1); + } else { + let better = Array.from(highlights).slice(0, 10); + better = better.map(hl => { + const length = Array.from(hl.fragment.text).length; + let {offset, symbols} = hl.fragment; + const newText = symbols.slice(Math.max(0,offset - extra), offset).join('') + hl.fragment.text + symbols.slice(offset + length, offset + length + extra).join(''); + DEBUG.verboseSlow && console.log({newText, oldText:hl.fragment.text, p:[Math.max(0,offset-extra), offset, offset+length, offset+length+extra], trueText: symbols.slice(offset, offset+length).join('')}); + hl.fragment.text = newText; + const {MaxDist,MinScore,MaxScore} = params(Array.from(newText).length); + const distance = ukkonen(query, hl.fragment.text.toLocaleLowerCase(), MaxDist); + // the min score possible = the minimum number of edits between + const scaledScore = (distance - MinScore)/MaxScore; + hl.score = scaledScore; + return hl; + }); + better.sort(({score:a}, {score:b}) => a-b); + DEBUG.verboseSlow && console.log(JSON.stringify({better},null,2)); + result = better.slice(0,3); + } + + return result; +} + +// use overlapping trigrams to index +export function trilight(query, doc, { + /* 0 is no maxLength */ + maxLength: maxLength = 0, + ngramSize: ngramSize = 3, + /*minSegmentGap: minSegmentGap = 20,*/ + maxSegmentSize: maxSegmentSize = 140, +} = {}) { + query = Array.from(query); + const oDoc = Array.from(doc); + doc = Array.from(doc.toLocaleLowerCase()); + if ( maxLength ) { + doc = doc.slice(0, maxLength); + } + + const trigrams = doc.reduce(getFragmenter(ngramSize, {overlap:true}), []); + const index = trigrams.reduce((idx, frag) => { + let counts = idx.get(frag.text); + if ( ! counts ) { + counts = []; + idx.set(frag.text, counts); + } + counts.push(frag.offset); + return idx; + }, new Map); + const qtris = query.reduce(getFragmenter(ngramSize, {overlap:true}), []); + const entries = qtris.reduce((E, {text}, qi) => { + const counts = index.get(text); + if ( counts ) { + counts.forEach(di => { + const entry = {text, qi, di}; + E.push(entry); + }); + } + return E; + }, []); + entries.sort(({di:a}, {di:b}) => a-b); + let lastQi; + let lastDi; + let run; + const runs = entries.reduce((R, {text,qi,di}) => { + if ( ! run ) { + run = { + tris: [text], + qi, di + }; + } else { + const dQi = qi - lastQi; + const dDi = di - lastDi; + if ( dQi === 1 && dDi === 1 ) { + run.tris.push(text); + } else { + /* add two for the size 2 suffix of the final trigram */ + run.length = run.tris.length + (ngramSize - 1); + R.push(run); + run = { + qi, di, + tris: [text] + }; + } + } + lastQi = qi; + lastDi = di; + return R; + }, []); + let lastRun; + const gaps = runs.reduce((G, run) => { + if ( lastRun ) { + const gap = {runs: [lastRun, run], gap: run.di - (lastRun.di + lastRun.length)}; + G.push(gap); + } + lastRun = run; + return G; + }, []); + gaps.sort(({gap:a}, {gap:b}) => a-b); + const segments = []; + const runSegMap = {}; + while(gaps.length) { + const nextGap = gaps.shift(); + const {runs} = nextGap; + const leftSeg = runSegMap[runs[0].di]; + const rightSeg = runSegMap[runs[1].di]; + let newSegmentLength = 0; + let assigned = false; + if ( leftSeg ) { + newSegmentLength = runs[1].di + runs[1].length - leftSeg.start; + if ( newSegmentLength <= maxSegmentSize ) { + leftSeg.end = runs[1].di + runs[1].length; + leftSeg.score += runs[1].length; + runSegMap[runs[1].di] = leftSeg; + assigned = leftSeg; + } + } else if ( rightSeg ) { + newSegmentLength = rightSeg.end - runs[0].di; + if ( newSegmentLength <= maxSegmentSize ) { + rightSeg.start = runs[0].di; + rightSeg.score += runs[0].length; + runSegMap[runs[0].di] = rightSeg; + assigned = rightSeg; + } + } else { + const newSegment = { + start: runs[0].di, + end: runs[0].di + runs[0].length + nextGap.gap + runs[1].length, + score: runs[0].length + runs[1].length + }; + if ( newSegment.end - newSegment.start <= maxSegmentSize ) { + runSegMap[runs[0].di] = newSegment; + runSegMap[runs[1].di] = newSegment; + segments.push(newSegment); + assigned = newSegment; + newSegmentLength = newSegment.end - newSegment.start; + } + } + if ( assigned ) { + DEBUG.verboseSlow && console.log('Assigned ', nextGap, 'to segment', assigned, 'now having length', newSegmentLength); + } else { + DEBUG.verboseSlow && console.log('Gap ', nextGap, `could not be assigned as it would have made an existing + as it would have made an existing segment too long, or it was already too long itself.` + ); + } + } + segments.sort(({score:a}, {score:b}) => b-a); + const textSegments = segments.map(({start,end}) => oDoc.slice(start,end).join('')); + //console.log(JSON.stringify({gaps}, null, 2)); + DEBUG.verboseSlow && console.log(segments, textSegments); + + if ( textSegments.length === 0 ) { + DEBUG.verboseSlow && console.log({query, doc, maxLength, ngramSize, maxSegmentSize, + trigrams, + index, + entries, + runs, + gaps, + segments, + textSegments + }); + } + + return textSegments.slice(0,3); +} + +// returns a function that creates non-overlapping fragments +function getFragmenter(chunkSize, {overlap: overlap = false} = {}) { + if ( !Number.isInteger(chunkSize) || chunkSize < 1 ) { + throw new TypeError(`chunkSize needs to be a whole number greater than 0`); + } + + let currentLength; + + return function fragment(frags, nextSymbol, index, symbols) { + const pushBack = []; + let currentFrag; + // logic: + // if there are no running fragments OR + // adding the next symbol would exceed chunkSize + // then start a new fragment OTHERWISE + // keep adding to the currentFragment + if ( overlap || (frags.length && ((currentLength + 1) <= chunkSize)) ) { + let count = 1; + if ( overlap ) { + count = Math.min(index+1, chunkSize); + currentFrag = {text:'', offset:index, symbols}; + frags.push(currentFrag); + } + while(count--) { + currentFrag = frags.pop(); + //console.log({frags,nextSymbol,index,currentFrag}); + pushBack.push(currentFrag); + currentFrag.text += nextSymbol; + } + } else { + currentFrag = {text:nextSymbol, offset:index, symbols}; + currentLength = 0; + pushBack.push(currentFrag); + } + currentLength++; + while(pushBack.length) { + frags.push(pushBack.pop()); + } + return frags; + } +} + +// returns a function that creates overlapping fragments +// todo - try this one as well + + +// tests + /* + function testHighlighter() { + const query = 'metahead search'; + const doc = ` + Hacker News new | past | comments | ask | show | jobs | submit login + 1. + AWS appears to be down again + 417 points by riknox 2 hours ago | hide | 260 comments + 2. + FreeBSD Jails for Fun and Profit (topikettunen.com) + 42 points by kettunen 1 hour ago | hide | discuss + 3. + IMF, 10 countries simulate cyber attack on global financial system (nasdaq.com) + 33 points by pueblito 1 hour ago | hide | 18 comments + 4. + DNA seen through the eyes of a coder (berthub.eu) + 116 points by dunefox 3 hours ago | hide | 37 comments + 5. + Pure Bash lightweight web server (github.com/remileduc) + 74 points by turrini 2 hours ago | hide | 46 comments + 6. + Parser Combinators in Haskell (serokell.io) + 18 points by aroccoli 1 hour ago | hide | 3 comments + 7. + DeepMind’s New AI with a Memory Outperforms Algorithms 25 Times Its Size (singularityhub.com) + 233 points by darkscape 9 hours ago | hide | 88 comments + 8. + Tinder just permabanned me or the problem with big tech (paulefou.com) + 90 points by svalee 1 hour ago | hide | 106 comments + 9. + Rocky Mountain Basic (wikipedia.org) + 12 points by mattowen_uk 1 hour ago | hide | 5 comments + 10. + Teller Reveals His Secrets (2012) (smithsonianmag.com) + 56 points by Tomte 4 hours ago | hide | 26 comments + 11. + Heroku Is Currently Down (heroku.com) + 129 points by iamricks 2 hours ago | hide | 29 comments + 12. Convictional (YC W19) is hiring engineers to build the future of B2B trade-Remote (ashbyhq.com) + 2 hours ago | hide + 13. + Scientists find preserved dinosaur embryo preparing to hatch like a bird (theguardian.com) + 187 points by Petiver 9 hours ago | hide | 111 comments + 14. + I did a Mixergy interview so bad they didn't even release it (robfitz.com) + 15 points by robfitz 1 hour ago | hide | 7 comments + 15. + Now DuckDuckGo is building its own desktop browser (zdnet.com) + 132 points by waldekm 2 hours ago | hide | 64 comments + 16. + English has been my pain for 15 years (2013) (antirez.com) + 105 points by Tomte 1 hour ago | hide | 169 comments + 17. + Polish opposition duo hacked with NSO spyware (apnews.com) + 102 points by JumpCrisscross 2 hours ago | hide | 35 comments + 18. + Linux Has Grown into a Viable PC Gaming Platform and the Steam Stats Prove It (hothardware.com) + 119 points by rbanffy 3 hours ago | hide | 105 comments + 19. + LG’s new 16:18 monitor (theverge.com) + 50 points by tosh 1 hour ago | hide | 25 comments + 20. + Construction of radio equipment in a Japanese PoW camp (bournemouth.ac.uk) + 117 points by marcodiego 9 hours ago | hide | 16 comments + 21. + Everything I've seen on optimizing Postgres on ZFS (vadosware.io) + 27 points by EntICOnc 4 hours ago | hide | 2 comments + 22. + Microsoft Teams: 1 feature, 4 vulnerabilities (positive.security) + 269 points by kerm1t 4 hours ago | hide | 196 comments + 23. + Analog computers were the most powerful computers for thousands of years [video] (youtube.com) + 103 points by jdkee 9 hours ago | hide | 55 comments + 24. + Shipwrecks, Stolen Jewels, Skull-Blasting Are Some of This Year’s Best Mysteries (atlasobscura.com) + 8 points by CapitalistCartr 1 hour ago | hide | 1 comment + 25. + Isolating Xwayland in a VM (roscidus.com) + 94 points by pmarin 9 hours ago | hide | 32 comments + 26. + Show HN: Metaheads, a search engine for Facebook comments (metaheads.xyz) + 4 points by jawerty 1 hour ago | hide | 15 comments + 27. + Quantum theory based on real numbers can be experimentally falsified (nature.com) + 159 points by SquibblesRedux 14 hours ago | hide | 93 comments + 28. + Founder of Black Girls Code has been ousted as head of the nonprofit (businessinsider.com) + 29 points by healsdata 1 hour ago | hide | 7 comments + 29. + Waffle House Poet Laureate (2019) (atlantamagazine.com) + 5 points by brudgers 1 hour ago | hide | 4 comments + 30. + Earth’s magnetic field illuminates Biblical history (economist.com) + 46 points by helsinkiandrew 8 hours ago | hide | 17 comments + More + `; + + console.log(JSON.stringify(highlight( + query, doc + ).map(({fragment:{text,offset}}) => offset + ':' + text), null, 2)); + console.log(trilight('metahead search', doc.toLocaleLowerCase().replace(/\s+/g, ' '))); + } + */ diff --git a/src/index.js b/src/index.js new file mode 100644 index 0000000..536edee --- /dev/null +++ b/src/index.js @@ -0,0 +1,4 @@ +/* eslint-disable no-global-assign */ +require = require('esm')(module/*, options*/); +module.exports = require('./app.js'); +/* eslint-enable no-global-assign */ diff --git a/src/libraryServer.js b/src/libraryServer.js new file mode 100644 index 0000000..4a5fb8b --- /dev/null +++ b/src/libraryServer.js @@ -0,0 +1,417 @@ +import http from 'http'; +import https from 'https'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import express from 'express'; + +import args from './args.js'; +import { + GO_SECURE, + MAX_REAL_URL_LENGTH, + MAX_HEAD, MAX_HIGHLIGHTABLE_LENGTH, DEBUG, + say, sleep, APP_ROOT, + RichError +} from './common.js'; +import {startCrawl, Archivist} from './archivist.js'; +import {trilight, highlight} from './highlighter.js'; + +const SITE_PATH = path.resolve(APP_ROOT, '..', 'public'); + +const SearchCache = new Map(); + +const app = express(); + +let running = false; +let Server, upAt, port; + +const LibraryServer = { + start, stop +} + +const secure_options = {}; +const protocol = GO_SECURE ? https : http; + +export default LibraryServer; + +async function start({server_port}) { + if ( running ) { + DEBUG.verboseSlow && console.warn(`Attempting to start server when it is not closed. Exiting start()...`); + return; + } + running = true; + + try { + const sec = { + key: fs.readFileSync(path.resolve(os.homedir(), 'local-sslcerts', 'privkey.pem')), + cert: fs.readFileSync(path.resolve(os.homedir(), 'local-sslcerts', 'fullchain.pem')), + ca: fs.existsSync(path.resolve(os.homedir(), 'local-sslcerts', 'chain.pem')) ? + fs.readFileSync(path.resolve(os.homedir(), 'local-sslcerts', 'chain.pem')) + : + undefined + }; + console.log({sec}); + Object.assign(secure_options, sec); + } catch(e) { + console.warn(`No certs found so will use insecure no SSL.`); + } + + try { + port = server_port; + addHandlers(); + const secure = secure_options.cert && secure_options.key; + const server = protocol.createServer.apply(protocol, GO_SECURE && secure ? [secure_options, app] : [app]); + Server = server.listen(Number(port), err => { + if ( err ) { + running = false; + throw err; + } + upAt = new Date; + say({server_up:{upAt,port}}); + }); + } catch(e) { + running = false; + DEBUG.verboseSlow && console.error(`Error starting server`, e); + process.exit(1); + } +} + +function addHandlers() { + app.use(express.urlencoded({extended:true, limit: '50mb'})); + app.use(express.static(SITE_PATH)); + + if ( args.library_path() ) { + app.use("/library", express.static(args.library_path())) + } + + app.get('/search(.json)?', async (req, res) => { + await Archivist.isReady(); + let {query:oquery} = req.query; + if ( ! oquery ) { + return res.end(SearchResultView({results:[], query:'', HL:new Map, page:1})); + } + oquery = oquery.trim(); + if ( ! oquery ) { + return res.end(SearchResultView({results:[], query:'', HL:new Map, page:1})); + } + let {page} = req.query; + if ( ! page || ! Number.isInteger(parseInt(page)) ) { + page = 1; + } else { + page = parseInt(page); + } + let resultIds, query, HL; + if ( SearchCache.has(req.query.query) ) { + ({query, resultIds, HL} = SearchCache.get(oquery)); + } else { + ({query, results:resultIds, HL} = await Archivist.search(oquery)); + SearchCache.set(req.query.query, {query, resultIds, HL}); + } + const start = (page-1)*args.results_per_page; + const results = resultIds.slice(start,start+args.results_per_page).map(docId => Archivist.getDetails(docId)) + if ( req.path.endsWith('.json') ) { + res.end(JSON.stringify({ + results, query + }, null, 2)); + } else { + results.forEach(r => { + /* + r.snippet = '... ' + highlight(query, r.content, {maxLength:MAX_HIGHLIGHTABLE_LENGTH}) + .sort(({fragment:{offset:a}}, {fragment:{offset:b}}) => a-b) + .map(hl => Archivist.findOffsets(query, hl.fragment.text)) + .join(' ... '); + */ + r.snippet = '... ' + trilight(query, r.content, {maxLength:MAX_HIGHLIGHTABLE_LENGTH}) + .map(segment => Archivist.findOffsets(query, segment)) + .join(' ... '); + }); + res.end(SearchResultView({results, query, HL, page})); + } + }); + + app.get('/mode', async (req, res) => { + res.end(Archivist.getMode()); + }); + + app.get('/archive_index.html', async (req, res) => { + Archivist.saveIndex(); + const index = Archivist.getIndex(); + res.end(IndexView(index)); + }); + + app.get('/edit_index.html', async (req, res) => { + Archivist.saveIndex(); + const index = Archivist.getIndex(); + res.end(IndexView(index, {edit:true})); + }); + + app.post('/edit_index.html', async (req, res) => { + const {url_to_delete} = req.body; + await Archivist.deleteFromIndexAndSearch(url_to_delete); + res.redirect('/edit_index.html'); + }); + + app.post('/mode', async (req, res) => { + const {mode} = req.body; + Archivist.changeMode(mode); + //res.end(`Mode set to ${mode}`); + res.redirect('/'); + }); + + app.get('/base_path', async (req, res) => { + res.end(args.getBasePath()); + }); + + app.post('/base_path', async (req, res) => { + const {base_path} = req.body; + const change = args.updateBasePath(base_path, {before: [ + () => Archivist.beforePathChanged(base_path) + ]}); + + if ( change ) { + await Archivist.afterPathChanged(); + Server.close(async () => { + running = false; + console.log(`Server closed.`); + console.log(`Waiting 50ms...`); + await sleep(50); + start({server_port:port}); + console.log(`Server restarting.`); + }); + //res.end(`Base path set to ${base_path} and saved to preferences. See console for progress. Server restarting...`); + res.redirect('/#new_base_path'); + } else { + //res.end(`Base path did not change.`); + res.redirect('/'); + } + }); + + app.post('/crawl', async (req, res) => { + try { + let { + links, timeout, depth, saveToFile, + maxPageCrawlTime, minPageCrawlTime, batchSize, + program, + } = req.body; + const oTimeout = timeout; + timeout = Math.round(parseFloat(timeout)*1000); + depth = Math.round(parseInt(depth)); + batchSize = Math.round(parseInt(batchSize)); + saveToFile = !!saveToFile; + minPageCrawlTime = Math.round(parseInt(minPageCrawlTime)*1000); + maxPageCrawlTime = Math.round(parseInt(maxPageCrawlTime)*1000); + if ( Number.isNaN(timeout) || Number.isNaN(depth) || typeof links != 'string' ) { + console.warn({invalid:{timeout,depth,links}}); + throw new RichError({ + status: 400, + message: 'Invalid parameters: timeout, depth or links' + }); + } + const urls = links.split(/[\n\s\r]+/g).map(u => u.trim()).filter(u => { + const tooShort = u.length === 0; + if ( tooShort ) return false; + + const tooLong = u.length > MAX_REAL_URL_LENGTH; + if ( tooLong ) return false; + + let invalid = false; + try { + new URL(u); + } catch { + invalid = true; + }; + if ( invalid ) return false; + + return true; + }).map(url => ({url,depth:1})); + console.log(`Starting crawl from ${urls.length} URLs, waiting ${oTimeout} seconds for each to load, and continuing to a depth of ${depth} clicks...`); + await startCrawl({ + urls, timeout, depth, saveToFile, batchSize, minPageCrawlTime, maxPageCrawlTime, program, + }); + res.end(`Starting crawl from ${urls.length} URLs, waiting ${oTimeout} seconds for each to load, and continuing to a depth of ${depth} clicks...`); + } catch(e) { + if ( e instanceof RichError ) { + console.warn(e); + const {status, message} = JSON.parse(e.message); + res.status(status); + res.end(message); + } else { + console.warn(e); + res.sendStatus(500); + } + return; + } + }); +} + +async function stop() { + let resolve; + const pr = new Promise(res => resolve = res); + + console.log(`Closing library server...`); + + Server.close(() => { + console.log(`Library server closed.`); + resolve(); + }); + + return pr; +} + +function IndexView(urls, {edit:edit = false} = {}) { + return ` + + + + ${ edit ? 'Editing ' : ''} + Your HTML Library + + + ${ edit ? ` + + ` : ''} +
+

22120 — Archive Index

+
+
+
+ Search your archive + + +
+
+
+
+ + ${ edit ? ` + ` + : + '…' + } + +
+ +
+
+
+
    + ${ + urls.map(([url,{title, id}]) => ` +
  • + ${ DEBUG ? id + ':' : ''} + ${(title||url).slice(0, MAX_HEAD)} + ${ edit ? ` +
    + + +
    + ` : ''} +
    • + `).join('\n') + } +
+ ${ edit ? ` + + ` : ''} + ` +} + +function SearchResultView({results, query, HL, page}) { + return ` + + + ${query} - 22120 search results + +
+

22120 — Search Results

+
+

+ View your index, or +

+
+
+ Search again + + +
+
+

+ Showing results for ${query} +

+
    + ${ + results.map(({snippet, url,title,id}) => ` +
  1. + ${DEBUG ? id + ':' : ''} ${ + HL.get(id)?.title||(title||url||'').slice(0, MAX_HEAD) + } +
    + ${ + HL.get(id)?.url||(url||'').slice(0, MAX_HEAD) + } +

    ${snippet}

    +
    2. + `).join('\n') + } +
+

+ ${page > 1 ? ` + + < Page ${page-1} + |` : ''} + + Page ${page} + + | + + Page ${page+1} > + +

+ ` +} + diff --git a/src/protocol.js b/src/protocol.js new file mode 100644 index 0000000..a5b57d4 --- /dev/null +++ b/src/protocol.js @@ -0,0 +1,162 @@ +import Ws from 'ws'; +import Fetch from 'node-fetch'; +import {untilTrue, SHOW_FETCH, DEBUG, ERROR_CODE_SAFE_TO_IGNORE} from './common.js'; + +const ROOT_SESSION = "browser"; +const MESSAGES = new Map(); + +export async function connect({port:port = 9222} = {}) { + let webSocketDebuggerUrl, socket; + try { + await untilTrue(async () => { + let result = false; + try { + const {webSocketDebuggerUrl} = await Fetch(`http://127.0.0.1:${port}/json/version`).then(r => r.json()); + if ( webSocketDebuggerUrl ) { + result = true; + } + } finally { + return result; + } + }); + ({webSocketDebuggerUrl} = await Fetch(`http://127.0.0.1:${port}/json/version`).then(r => r.json())); + socket = new Ws(webSocketDebuggerUrl); + } catch(e) { + console.log("Error communicating with browser", e); + process.exit(1); + } + + const Resolvers = {}; + const Handlers = {}; + socket.on('message', handle); + let id = 0; + + let resolve; + const promise = new Promise(res => resolve = res); + + socket.on('open', () => resolve()); + + await promise; + + return { + send, + on, ons, ona, + close + }; + + async function send(method, params = {}, sessionId) { + const message = { + method, params, sessionId, + id: ++id + }; + if ( ! sessionId ) { + delete message[sessionId]; + } + const key = `${sessionId||ROOT_SESSION}:${message.id}`; + let resolve; + const promise = new Promise(res => resolve = res); + Resolvers[key] = resolve; + const outGoing = JSON.stringify(message); + MESSAGES.set(key, outGoing); + socket.send(outGoing); + DEBUG.verboseSlow && (SHOW_FETCH || !method.startsWith('Fetch')) && console.log("Sent", message); + return promise; + } + + async function handle(message) { + if ( typeof message !== "string" ) { + try { + message += ''; + } catch(e) { + message = message.toString(); + } + } + const stringMessage = message; + message = JSON.parse(message); + if ( message.error ) { + const showError = DEBUG.protocol || !ERROR_CODE_SAFE_TO_IGNORE.has(message.error.code); + if ( showError ) { + DEBUG.protocol && console.warn(message); + } + } + const {sessionId} = message; + const {method} = message; + const {id, result} = message; + + if ( id ) { + const key = `${sessionId||ROOT_SESSION}:${id}`; + const resolve = Resolvers[key]; + if ( ! resolve ) { + DEBUG.protocol && console.warn(`No resolver for key`, key, stringMessage.slice(0,140)); + } else { + Resolvers[key] = undefined; + try { + await resolve(result); + } catch(e) { + console.warn(`Resolver failed`, e, key, stringMessage.slice(0,140), resolve); + } + } + if ( DEBUG ) { + if ( message.error ) { + const showError = DEBUG || !ERROR_CODE_SAFE_TO_IGNORE.has(message.error.code); + if ( showError ) { + const originalMessage = MESSAGES.get(key); + DEBUG.protocol && console.warn({originalMessage}); + } + } + } + MESSAGES.delete(key); + } else if ( method ) { + const listeners = Handlers[method]; + if ( Array.isArray(listeners) ) { + for( const func of listeners ) { + try { + func({message, sessionId}); + } catch(e) { + console.warn(`Listener failed`, method, e, func.toString().slice(0,140), stringMessage.slice(0,140)); + } + } + } + } else { + console.warn(`Unknown message on socket`, message); + } + } + + function on(method, handler) { + let listeners = Handlers[method]; + if ( ! listeners ) { + Handlers[method] = listeners = []; + } + listeners.push(wrap(handler)); + } + + function ons(method, handler) { + let listeners = Handlers[method]; + if ( ! listeners ) { + Handlers[method] = listeners = []; + } + listeners.push(handler); + } + + function ona(method, handler, sessionId) { + let listeners = Handlers[method]; + if ( ! listeners ) { + Handlers[method] = listeners = []; + } + listeners.push(({message}) => { + if ( message.sessionId === sessionId ) { + handler(message.params); + } else { + console.log(`No such`, {method, handler, sessionId, message}); + } + }); + } + + function close() { + socket.close(); + } + + function wrap(fn) { + return ({message}) => fn(message.params) + } +}