Minor fixes and improvements with significants results

Lots of positive payoffs

package-lock.json

@@ -1796,6 +1796,18 @@
       "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==",
       "dev": true
     },
+    "node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/marky": {
       "version": "1.2.5",
       "resolved": "https://registry.npmjs.org/marky/-/marky-1.2.5.tgz",
@@ -1954,9 +1966,9 @@
       }
     },
     "node_modules/nodemon": {
-      "version": "2.0.20",
+      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.20.tgz",
+      "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.0.1.tgz",
-      "integrity": "sha512-Km2mWHKKY5GzRg6i1j5OxOHQtuvVsgskLfigG25yTtbyfRGn/GNvIbRyOf1PSCKJ2aT/58TiuUsuOU5UToVViw==",
+      "integrity": "sha512-g9AZ7HmkhQkqXkRc20w+ZfQ73cHLbE8hnPbtaFbFtCumZsjyMhKk9LajQ07U5Ux28lvFjZ5X7HvWR1xzU8jHVw==",
       "dev": true,
       "dependencies": {
         "chokidar": "^3.5.2",
@@ -1964,8 +1976,8 @@
         "ignore-by-default": "^1.0.1",
         "minimatch": "^3.1.2",
         "pstree.remy": "^1.1.8",
-        "semver": "^5.7.1",
+        "semver": "^7.5.3",
-        "simple-update-notifier": "^1.0.7",
+        "simple-update-notifier": "^2.0.0",
         "supports-color": "^5.5.0",
         "touch": "^3.1.0",
         "undefsafe": "^2.0.5"
@@ -1974,7 +1986,7 @@
         "nodemon": "bin/nodemon.js"
       },
       "engines": {
-        "node": ">=8.10.0"
+        "node": ">=10"
       },
       "funding": {
         "type": "opencollective",
@@ -2380,12 +2392,18 @@
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
     },
     "node_modules/semver": {
-      "version": "5.7.1",
+      "version": "7.5.4",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
-      "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
       "bin": {
-        "semver": "bin/semver"
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
       }
     },
     "node_modules/send": {
@@ -2483,24 +2501,15 @@
       }
     },
     "node_modules/simple-update-notifier": {
-      "version": "1.1.0",
+      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-1.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz",
-      "integrity": "sha512-VpsrsJSUcJEseSbMHkrsrAVSdvVS5I96Qo1QAQ4FxQ9wXFcB+pjj7FB7/us9+GcgfW4ziHtYMc1J0PLczb55mg==",
+      "integrity": "sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==",
       "dev": true,
       "dependencies": {
-        "semver": "~7.0.0"
+        "semver": "^7.5.3"
       },
       "engines": {
-        "node": ">=8.10.0"
+        "node": ">=10"
-      }
-    },
-    "node_modules/simple-update-notifier/node_modules/semver": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.0.0.tgz",
-      "integrity": "sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==",
-      "dev": true,
-      "bin": {
-        "semver": "bin/semver.js"
       }
     },
     "node_modules/statuses": {
@@ -2702,9 +2711,9 @@
       }
     },
     "node_modules/word-wrap": {
-      "version": "1.2.3",
+      "version": "1.2.5",
-      "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
+      "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz",
-      "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
+      "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==",
       "dev": true,
       "engines": {
         "node": ">=0.10.0"
@@ -2744,6 +2753,12 @@
         }
       }
     },
+    "node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
     "node_modules/yocto-queue": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",

public/index.html

@@ -1,12 +1,12 @@
 <!DOCTYPE html>
 <meta charset=utf-8>
-<title>Your HTML Library</title>
+<title>Your Personal Search Engine and Archive</title>
-<link rel=stylesheet href=/style.css>
+<link rel=stylesheet href=style.css>
 <header>
-  <h1><a href=/>22120</a> &mdash; Internet Offline Library</h1>
+  <h1><a href=/>DiskerNet</a> &mdash; Personal Web Search and Archive</h1>
 </header>
 <p>
-  View <a href=/archive_index.html>the index</a>
+  View <a href=/archive_index.html>your index</a>
 </p>
 <!--
 <form method=POST action=/crawl>
@@ -147,7 +147,7 @@
 <form method=GET action=/search>
   <fieldset class=search>
     <legend>Search your archive</legend>
-    <input class=search type=search name=query placeholder="search your library">
+    <input autofocus class=search type=search name=query placeholder="search your library">
     <button>Search</button>
   </fieldset>
 </form>
@@ -236,8 +236,11 @@
       <button disabled>Publish</button>
   </fieldset>
 </form>
+<p>
+  Notice a bug? <a href=https://github.com/dosyago/DiskerNet/issues>Open an issue!</a>
+</p>
 <footer>
   <cite>
-    <a rel=author href=https://github.com/crisdosyago/22120>22120 GitHub</a>
+    <a rel=author href=https://github.com/dosyago/DiskerNet>DiskerNet GitHub</a>
   </cite>
 </footer>

public/style.css

@@ -4,12 +4,12 @@
     background: lavenderblush;
   }
   body {
-    display: table;
     margin: 0 auto 0 min(10%, 5em);
     background: white;
     padding: 0.5em;
     border-bottom: 1px solid purple;
     max-width: min(777px, 80%);
+    overflow-x: hidden;
   }
   header {
     font-size: smaller;
@@ -80,7 +80,7 @@
 
   input[type="number"] {
     text-align: right;
-  ]
+  }
 
   input.search {
     flex-grow: 1;
@@ -99,3 +99,6 @@
   .grey {
     color: grey;
   }
+  a {
+    word-break: break-word;
+  }

run.cjs

@@ -0,0 +1,24 @@
+#!/usr/bin/env node
+
+const os = require('os');
+const { spawn } = require('child_process');
+
+// Checking if node_modules directory exists
+const fs = require('fs');
+const path = require('path');
+
+if (!fs.existsSync(path.join(process.cwd(), 'node_modules'))) {
+  spawn('npm', ['i'], { stdio: 'inherit' });
+}
+
+// Getting the total system memory
+const totalMemory = os.totalmem();
+
+// Allocating 90% of the total memory
+const memoryAllocation = Math.floor((totalMemory / (1024 * 1024)) * 0.8); // Converted bytes to MB and took 90% of it
+
+console.log(`Index can use up to: ${memoryAllocation}MB RAM`);
+
+// Running the application
+spawn('node', [`--max-old-space-size=${memoryAllocation}`, 'src/app.js'], { stdio: 'inherit' });
+

run.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-phys=$(free -t -m | grep -oP '\d+' | sed '10!d')
-alloc=$(echo "$phys * 90/100" | bc )
-echo $alloc
-node --max-old-space-size=$alloc src/app.js

src/app.js

@@ -143,6 +143,7 @@ async function killChrome(wait = true) {
 }
 
 async function cleanup(reason, err, {exit = false} = {}) {
+  console.log(`Shutting down everything because of an error!`);
   console.log(`Cleanup called on reason: ${reason}`, err);
 
   if ( quitting ) {

src/archivist.js

@@ -1222,7 +1222,8 @@
         obj.id = id;
         return obj;
       } catch(e) {
-        console.log({obj, index:State.Index, e, ndx, flex, fuzz});
+        DEBUG.verboseSlow && console.log({obj, index:State.Index, e, ndx, flex, fuzz});
+        console.error("Error", e);
         return obj;
       }
     });

src/common.js

@@ -7,6 +7,7 @@ import {APP_ROOT as __ROOT} from './root.js';
 const DEEB = false;
 
 export const DEBUG = {
+  debugSec: false,
   askFirst: true,
   verboseSlow: process.env.VERBOSE_DEBUG_22120 || DEEB,
   debug: process.env.DEBUG_22120 || DEEB,

src/libraryServer.js

@@ -9,8 +9,9 @@ import express from 'express';
 import args from './args.js';
 import {
   GO_SECURE,
+  DEBUG,
   MAX_REAL_URL_LENGTH,
-  MAX_HEAD, MAX_HIGHLIGHTABLE_LENGTH, DEBUG, 
+  MAX_HEAD, MAX_HIGHLIGHTABLE_LENGTH, 
   say, sleep, APP_ROOT,
   RichError
 } from './common.js';
@@ -51,7 +52,7 @@ async function start({server_port}) {
         :
           undefined
     };
-    console.log({sec});
+    DEBUG.debugSec && console.log({sec});
     Object.assign(secure_options, sec);
   } catch(e) {
     console.warn(`No certs found so will use insecure no SSL.`);
@@ -115,6 +116,7 @@ function addHandlers() {
     }
     const start = (page-1)*args.results_per_page;
     const results = resultIds.slice(start,start+args.results_per_page).map(docId => Archivist.getDetails(docId))
+    const hasMore = resultIds.length > start+args.results_per_page;
     if ( req.path.endsWith('.json') ) {
       res.end(JSON.stringify({
         results, query
@@ -131,7 +133,7 @@ function addHandlers() {
           .map(segment => Archivist.findOffsets(query, segment))
           .join(' ... ');
       });
-      res.end(SearchResultView({results, query, HL, page}));
+      res.end(SearchResultView({results, query, HL, page, hasMore}));
     }
   });
 
@@ -272,7 +274,7 @@ function IndexView(urls, {edit:edit = false} = {}) {
       ${ edit ? 'Editing ' : ''}
       Your HTML Library
     </title>
-    <link rel=stylesheet href=/style.css>
+    <link rel=stylesheet href=style.css>
     ${ edit ? `
     <script>
       const sleep = ms => new Promise(res => setTimeout(res, ms));
@@ -280,12 +282,12 @@ function IndexView(urls, {edit:edit = false} = {}) {
     </script>
     ` : ''}
     <header>
-      <h1><a href=/>22120</a> &mdash; Archive Index</h1>
+      <h1><a href=/>DiskerNet</a> &mdash; Search and Archive Index</h1>
     </header>
     <form method=GET action=/search style="margin-bottom: 1em;">
       <fieldset class=search>
         <legend>Search your archive</legend>
-        <input class=search type=search name=query placeholder="search your library">
+        <input autofocus class=search type=search name=query placeholder="search your library">
         <button>Search</button>
       </fieldset>
     </form>
@@ -356,7 +358,7 @@ function IndexView(urls, {edit:edit = false} = {}) {
         host = host.replace(/^www./i, '');
         await sleep(200);
         const reallyDelete = confirm(
-          \`\n are you sure you want to delete this \n\n  \${host} \n\n from the internet?\n\`
+          \`\n are you sure you want to delete this \n\n  \${host} \n\n from the ENTIRE internet?!?\n\`
         );
         if ( reallyDelete ) return form.submit();
         link.style.textDecoration = original;
@@ -366,14 +368,14 @@ function IndexView(urls, {edit:edit = false} = {}) {
   `
 }
 
-function SearchResultView({results, query, HL, page}) {
+function SearchResultView({results, query, HL, page, hasMore = false}) {
   return `
     <!DOCTYPE html>
     <meta charset=utf-8>
-    <title>${query} - 22120 search results</title>
+    <title>${query} - DiskerNet search results</title>
     <link rel=stylesheet href=/style.css>
     <header>
-      <h1><a href=/>22120</a> &mdash; Search Results</h1>
+      <h1><a href=/>DiskerNet</a> &mdash; Search Results</h1>
     </header>
     <p>
     View <a href=/archive_index.html>your index</a>, or
@@ -381,7 +383,7 @@ function SearchResultView({results, query, HL, page}) {
     <form method=GET action=/search>
       <fieldset class=search>
         <legend>Search again</legend>
-        <input class=search type=search name=query placeholder="search your library" value="${query}">
+        <input autofocus class=search type=search name=query placeholder="search your library" value="${query}">
         <button>Search</button>
       </fieldset>
     </form>
@@ -411,11 +413,11 @@ function SearchResultView({results, query, HL, page}) {
       </a> |` : ''}
       <span class=grey>
         Page ${page}
-      </span>
+      </span>${hasMore ? `
       |
       <a href=/search?query=${encodeURIComponent(query)}&page=${encodeURIComponent(page+1)}>
         Page ${page+1} &gt;
-      </a>
+      </a>` : ''}
     </p>
   `
 }