Minor fixes and improvements with significants results

Lots of positive payoffs
2024-10-02 13:17:13 +02:00 · 2023-07-23 09:48:28 +08:00 · 2023-07-23 09:48:28 +08:00 · 745a9a44a6
commit 745a9a44a6
parent cc1061483e
9 changed files with 99 additions and 55 deletions
--- a/package-lock.json
+++ b/package-lock.json
@ -1796,6 +1796,18 @@
      "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==",
      "dev": true
    },
+    "node_modules/lru-cache": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+      "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
+      "dependencies": {
+        "yallist": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
    "node_modules/marky": {
      "version": "1.2.5",
      "resolved": "https://registry.npmjs.org/marky/-/marky-1.2.5.tgz",
@ -1954,9 +1966,9 @@
      }
    },
    "node_modules/nodemon": {
-      "version": "2.0.20",
-      "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.20.tgz",
-      "integrity": "sha512-Km2mWHKKY5GzRg6i1j5OxOHQtuvVsgskLfigG25yTtbyfRGn/GNvIbRyOf1PSCKJ2aT/58TiuUsuOU5UToVViw==",
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.0.1.tgz",
+      "integrity": "sha512-g9AZ7HmkhQkqXkRc20w+ZfQ73cHLbE8hnPbtaFbFtCumZsjyMhKk9LajQ07U5Ux28lvFjZ5X7HvWR1xzU8jHVw==",
      "dev": true,
      "dependencies": {
        "chokidar": "^3.5.2",
@ -1964,8 +1976,8 @@
        "ignore-by-default": "^1.0.1",
        "minimatch": "^3.1.2",
        "pstree.remy": "^1.1.8",
-        "semver": "^5.7.1",
-        "simple-update-notifier": "^1.0.7",
+        "semver": "^7.5.3",
+        "simple-update-notifier": "^2.0.0",
        "supports-color": "^5.5.0",
        "touch": "^3.1.0",
        "undefsafe": "^2.0.5"
@ -1974,7 +1986,7 @@
        "nodemon": "bin/nodemon.js"
      },
      "engines": {
-        "node": ">=8.10.0"
+        "node": ">=10"
      },
      "funding": {
        "type": "opencollective",
@ -2380,12 +2392,18 @@
      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
    },
    "node_modules/semver": {
-      "version": "5.7.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
-      "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
      "dev": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
      "bin": {
-        "semver": "bin/semver"
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
      }
    },
    "node_modules/send": {
@ -2483,24 +2501,15 @@
      }
    },
    "node_modules/simple-update-notifier": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-1.1.0.tgz",
-      "integrity": "sha512-VpsrsJSUcJEseSbMHkrsrAVSdvVS5I96Qo1QAQ4FxQ9wXFcB+pjj7FB7/us9+GcgfW4ziHtYMc1J0PLczb55mg==",
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz",
+      "integrity": "sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==",
      "dev": true,
      "dependencies": {
-        "semver": "~7.0.0"
+        "semver": "^7.5.3"
      },
      "engines": {
-        "node": ">=8.10.0"
-      }
-    },
-    "node_modules/simple-update-notifier/node_modules/semver": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.0.0.tgz",
-      "integrity": "sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==",
-      "dev": true,
-      "bin": {
-        "semver": "bin/semver.js"
+        "node": ">=10"
      }
    },
    "node_modules/statuses": {
@ -2702,9 +2711,9 @@
      }
    },
    "node_modules/word-wrap": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
-      "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
+      "version": "1.2.5",
+      "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz",
+      "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==",
      "dev": true,
      "engines": {
        "node": ">=0.10.0"
@ -2744,6 +2753,12 @@
        }
      }
    },
+    "node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
+    },
    "node_modules/yocto-queue": {
      "version": "0.1.0",
      "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",
--- a/public/index.html
+++ b/public/index.html
@ -1,12 +1,12 @@
 <!DOCTYPE html>
 <meta charset=utf-8>
-<title>Your HTML Library</title>
-<link rel=stylesheet href=/style.css>
+<title>Your Personal Search Engine and Archive</title>
+<link rel=stylesheet href=style.css>
 <header>
-  <h1><a href=/>22120</a> &mdash; Internet Offline Library</h1>
+  <h1><a href=/>DiskerNet</a> &mdash; Personal Web Search and Archive</h1>
 </header>
 <p>
-  View <a href=/archive_index.html>the index</a>
+  View <a href=/archive_index.html>your index</a>
 </p>
 <!--
 <form method=POST action=/crawl>
@ -147,7 +147,7 @@
 <form method=GET action=/search>
  <fieldset class=search>
    <legend>Search your archive</legend>
-    <input class=search type=search name=query placeholder="search your library">
+    <input autofocus class=search type=search name=query placeholder="search your library">
    <button>Search</button>
  </fieldset>
 </form>
@ -236,8 +236,11 @@
      <button disabled>Publish</button>
  </fieldset>
 </form>
+<p>
+  Notice a bug? <a href=https://github.com/dosyago/DiskerNet/issues>Open an issue!</a>
+</p>
 <footer>
  <cite>
-    <a rel=author href=https://github.com/crisdosyago/22120>22120 GitHub</a>
+    <a rel=author href=https://github.com/dosyago/DiskerNet>DiskerNet GitHub</a>
  </cite>
 </footer>
--- a/public/style.css
+++ b/public/style.css
@ -4,12 +4,12 @@
    background: lavenderblush;
  }
  body {
-    display: table;
    margin: 0 auto 0 min(10%, 5em);
    background: white;
    padding: 0.5em;
    border-bottom: 1px solid purple;
    max-width: min(777px, 80%);
+    overflow-x: hidden;
  }
  header {
    font-size: smaller;
@ -80,7 +80,7 @@

  input[type="number"] {
    text-align: right;
-  ]
+  }

  input.search {
    flex-grow: 1;
@ -99,3 +99,6 @@
  .grey {
    color: grey;
  }
+  a {
+    word-break: break-word;
+  }
--- a/run.cjs
+++ b/run.cjs
@ -0,0 +1,24 @@
+#!/usr/bin/env node
+
+const os = require('os');
+const { spawn } = require('child_process');
+
+// Checking if node_modules directory exists
+const fs = require('fs');
+const path = require('path');
+
+if (!fs.existsSync(path.join(process.cwd(), 'node_modules'))) {
+  spawn('npm', ['i'], { stdio: 'inherit' });
+}
+
+// Getting the total system memory
+const totalMemory = os.totalmem();
+
+// Allocating 90% of the total memory
+const memoryAllocation = Math.floor((totalMemory / (1024 * 1024)) * 0.8); // Converted bytes to MB and took 90% of it
+
+console.log(`Index can use up to: ${memoryAllocation}MB RAM`);
+
+// Running the application
+spawn('node', [`--max-old-space-size=${memoryAllocation}`, 'src/app.js'], { stdio: 'inherit' });
+
--- a/run.sh
+++ b/run.sh
@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-phys=$(free -t -m | grep -oP '\d+' | sed '10!d')
-alloc=$(echo "$phys * 90/100" | bc )
-echo $alloc
-node --max-old-space-size=$alloc src/app.js
--- a/src/app.js
+++ b/src/app.js
@ -143,6 +143,7 @@ async function killChrome(wait = true) {
 }

 async function cleanup(reason, err, {exit = false} = {}) {
+  console.log(`Shutting down everything because of an error!`);
  console.log(`Cleanup called on reason: ${reason}`, err);

  if ( quitting ) {
--- a/src/archivist.js
+++ b/src/archivist.js
@ -1222,7 +1222,8 @@
        obj.id = id;
        return obj;
      } catch(e) {
-        console.log({obj, index:State.Index, e, ndx, flex, fuzz});
+        DEBUG.verboseSlow && console.log({obj, index:State.Index, e, ndx, flex, fuzz});
+        console.error("Error", e);
        return obj;
      }
    });
--- a/src/common.js
+++ b/src/common.js
@ -7,6 +7,7 @@ import {APP_ROOT as __ROOT} from './root.js';
 const DEEB = false;

 export const DEBUG = {
+  debugSec: false,
  askFirst: true,
  verboseSlow: process.env.VERBOSE_DEBUG_22120 || DEEB,
  debug: process.env.DEBUG_22120 || DEEB,
--- a/src/libraryServer.js
+++ b/src/libraryServer.js
@ -9,8 +9,9 @@ import express from 'express';
 import args from './args.js';
 import {
  GO_SECURE,
+  DEBUG,
  MAX_REAL_URL_LENGTH,
-  MAX_HEAD, MAX_HIGHLIGHTABLE_LENGTH, DEBUG, 
+  MAX_HEAD, MAX_HIGHLIGHTABLE_LENGTH, 
  say, sleep, APP_ROOT,
  RichError
 } from './common.js';
@ -51,7 +52,7 @@ async function start({server_port}) {
        :
          undefined
    };
-    console.log({sec});
+    DEBUG.debugSec && console.log({sec});
    Object.assign(secure_options, sec);
  } catch(e) {
    console.warn(`No certs found so will use insecure no SSL.`);
@ -115,6 +116,7 @@ function addHandlers() {
    }
    const start = (page-1)*args.results_per_page;
    const results = resultIds.slice(start,start+args.results_per_page).map(docId => Archivist.getDetails(docId))
+    const hasMore = resultIds.length > start+args.results_per_page;
    if ( req.path.endsWith('.json') ) {
      res.end(JSON.stringify({
        results, query
@ -131,7 +133,7 @@ function addHandlers() {
          .map(segment => Archivist.findOffsets(query, segment))
          .join(' ... ');
      });
-      res.end(SearchResultView({results, query, HL, page}));
+      res.end(SearchResultView({results, query, HL, page, hasMore}));
    }
  });

@ -272,7 +274,7 @@ function IndexView(urls, {edit:edit = false} = {}) {
      ${ edit ? 'Editing ' : ''}
      Your HTML Library
    </title>
-    <link rel=stylesheet href=/style.css>
+    <link rel=stylesheet href=style.css>
    ${ edit ? `
    <script>
      const sleep = ms => new Promise(res => setTimeout(res, ms));
@ -280,12 +282,12 @@ function IndexView(urls, {edit:edit = false} = {}) {
    </script>
    ` : ''}
    <header>
-      <h1><a href=/>22120</a> &mdash; Archive Index</h1>
+      <h1><a href=/>DiskerNet</a> &mdash; Search and Archive Index</h1>
    </header>
    <form method=GET action=/search style="margin-bottom: 1em;">
      <fieldset class=search>
        <legend>Search your archive</legend>
-        <input class=search type=search name=query placeholder="search your library">
+        <input autofocus class=search type=search name=query placeholder="search your library">
        <button>Search</button>
      </fieldset>
    </form>
@ -356,7 +358,7 @@ function IndexView(urls, {edit:edit = false} = {}) {
        host = host.replace(/^www./i, '');
        await sleep(200);
        const reallyDelete = confirm(
-          \`\n are you sure you want to delete this \n\n  \${host} \n\n from the internet?\n\`
+          \`\n are you sure you want to delete this \n\n  \${host} \n\n from the ENTIRE internet?!?\n\`
        );
        if ( reallyDelete ) return form.submit();
        link.style.textDecoration = original;
@ -366,14 +368,14 @@ function IndexView(urls, {edit:edit = false} = {}) {
  `
 }

-function SearchResultView({results, query, HL, page}) {
+function SearchResultView({results, query, HL, page, hasMore = false}) {
  return `
    <!DOCTYPE html>
    <meta charset=utf-8>
-    <title>${query} - 22120 search results</title>
+    <title>${query} - DiskerNet search results</title>
    <link rel=stylesheet href=/style.css>
    <header>
-      <h1><a href=/>22120</a> &mdash; Search Results</h1>
+      <h1><a href=/>DiskerNet</a> &mdash; Search Results</h1>
    </header>
    <p>
    View <a href=/archive_index.html>your index</a>, or
@ -381,7 +383,7 @@ function SearchResultView({results, query, HL, page}) {
    <form method=GET action=/search>
      <fieldset class=search>
        <legend>Search again</legend>
-        <input class=search type=search name=query placeholder="search your library" value="${query}">
+        <input autofocus class=search type=search name=query placeholder="search your library" value="${query}">
        <button>Search</button>
      </fieldset>
    </form>
@ -411,11 +413,11 @@ function SearchResultView({results, query, HL, page}) {
      </a> |` : ''}
      <span class=grey>
        Page ${page}
-      </span>
+      </span>${hasMore ? `
      |
      <a href=/search?query=${encodeURIComponent(query)}&page=${encodeURIComponent(page+1)}>
        Page ${page+1} &gt;
-      </a>
+      </a>` : ''}
    </p>
  `
 }