"Issues"

archivist.js

@@ -22,6 +22,7 @@ const {Index, registerCharset, registerLanguage} = FlexSearch;
 const FLEX_OPTS = {
   context: true,
 };
+const Targets = new Map();
 const Flex = new Index(FLEX_OPTS);
 const Cache = new Map();
 const Indexing = new Set();
@@ -110,11 +111,13 @@ async function collect({chrome_port:port, mode} = {}) {
   }
 
   on("Target.targetInfoChanged", indexURL);
+  on("Target.targetInfoChanged", updateTargetInfo);
   on("Target.targetInfoChanged", reloadIfNotLive);
   on("Target.targetInfoChanged", attachToTarget);
+  //on("Target.targetInfoChanged", displayTargetInfo);
   on("Target.attachedToTarget", installForSession);
   on("Fetch.requestPaused", cacheRequest);
-  on("Runtime.consoleAPICalled", confirmInstall);
+  on("Runtime.consoleAPICalled", handleMessage);
 
   await send("Target.setDiscoverTargets", {discover:true});
   await send("Target.setAutoAttach", {autoAttach:true, waitForDebuggerOnStart:false, flatten: true});
@@ -139,15 +142,25 @@ async function collect({chrome_port:port, mode} = {}) {
     };
   }
 
-  function confirmInstall(args) {
+  function handleMessage(args) {
     const {type, args:[{value:strVal}], context} = args;
     if ( type == 'info' ) {
       try {
         const val = JSON.parse(strVal);
-        const {installed:{sessionId}} = val;
-        if ( ! ConfirmedInstalls.has(sessionId) ) {
-          ConfirmedInstalls.add(sessionId);
-          console.log({confirmedInstall:val, context});
+        // possible messages
+        const {install, titleChange} = val;
+        switch(true) {
+          case !!install: {
+              confirmInstall({install});
+            }; break;
+          case !!titleChange: {
+              reindexOnTitleChange({titleChange});
+            }; break;
+          default: {
+              if ( DEBUG ) {
+                console.warn(`Unknown message`, strVal);
+              }
+            }; break;
         }
       } catch(e) {
         DEBUG && console.info('Not the message we expected to confirm install. This is OK.', {originalMessage:args});
@@ -155,13 +168,51 @@ async function collect({chrome_port:port, mode} = {}) {
     }
   }
 
+  function confirmInstall({install}) {
+    const {sessionId} = install;
+    if ( ! ConfirmedInstalls.has(sessionId) ) {
+      ConfirmedInstalls.add(sessionId);
+      console.log({confirmedInstall:val, context});
+    }
+  }
+
+  async function reindexOnTitleChange({titleChange}) {
+    const {currentTitle, url, sessionId} = titleChange;
+    const latestTargetInfo = await untilHas(Targets, sessionId);
+    latestTargetInfo.title = currentTitle;
+    Targets.set(sessionId, latestTargetInfo);
+    console.log(`Reindexing with`, latestTargetInfo);
+    indexURL({targetInfo:latestTargetInfo});
+  }
+
+  function displayTargetInfo({targetInfo}) {
+    if ( targetInfo.type === 'page' ) {
+      console.log("Title info", JSON.stringify(targetInfo, null, 2));
+    }
+  }
+
+  function updateTargetInfo({targetInfo}) {
+    if ( targetInfo.type === 'page' ) {
+      const sessionId = Sessions.get(targetInfo.targetId); 
+      if ( sessionId ) {
+        const existingTargetInfo = Targets.get(sessionId);
+        // if we have an existing target info for this URL and have saved an updated title
+        if ( existingTargetInfo && existingTargetInfo.url === targetInfo.url ) {
+          // keep that title (because targetInfo does not reflect the latest title)
+          targetInfo.title = existingTargetInfo.title;
+        }
+        Targets.set(sessionId, targetInfo);
+      }
+    }
+  }
+
   async function reloadIfNotLive({targetInfo}) {
     if ( Mode == 'serve' ) return; 
     const {attached, type} = targetInfo;
     if ( attached && type == 'page' ) {
       const {url, targetId} = targetInfo;
       const sessionId = Sessions.get(targetId);
-      if ( !!sessionId && !neverCache(url) && !ConfirmedInstalls.has(sessionId) ) {
+      if ( !!sessionId && !dontInstall({url}) && !ConfirmedInstalls.has(sessionId) ) {
         console.log({reloadingAsNotConfirmedInstalled:{url, sessionId}});
         send("Page.stopLoading", {}, sessionId);
         send("Page.reload", {}, sessionId);
@@ -170,24 +221,28 @@ async function collect({chrome_port:port, mode} = {}) {
   }
 
   function neverCache(url) {
-    return !url || url == "about:blank" || url.startsWith('chrome') || NEVER_CACHE.has(url);
+    return url == "about:blank" || url?.startsWith('chrome') || NEVER_CACHE.has(url);
   }
 
   async function installForSession({sessionId, targetInfo, waitingForDebugger}) {
-    console.log("installForSession called");
     if ( ! sessionId ) {
       throw new TypeError(`installForSession needs a sessionId`);
     }
 
     const {targetId, url} = targetInfo;
 
-    if ( Installations.has(sessionId) ) return;
+    if ( Mode == 'serve' ) return;
+
+    if ( dontInstall(targetInfo) ) return;
 
     if ( targetInfo.type != 'page' ) return;
 
-    if ( Mode == 'serve' ) return;
+    if ( Installations.has(sessionId) ) return;
+
+    console.log("installForSession running on " + targetId);
 
     Sessions.set(targetId, sessionId);
+    Targets.set(sessionId, targetInfo);
 
     if ( Mode == 'save' ) {
       send("Network.setCacheDisabled", {cacheDisabled:true}, sessionId);
@@ -333,7 +388,7 @@ async function collect({chrome_port:port, mode} = {}) {
   }
 
   async function attachToTarget(targetInfo) {
-    if ( dontCache(targetInfo) ) return;
+    if ( dontInstall(targetInfo) ) return;
     const {url} = targetInfo;
     if ( !!url && url != "about:blank" && !url.startsWith('chrome') ) {
 
@@ -358,7 +413,6 @@ async function collect({chrome_port:port, mode} = {}) {
     const isNavigationRequest = resourceType == "Document";
     const isFont = resourceType == "Font";
 
-
     if ( dontCache(request) ) {
       DEBUG && console.log("Not caching", request.url);
       return send("Fetch.continueRequest", {requestId});
@@ -434,12 +488,19 @@ async function collect({chrome_port:port, mode} = {}) {
     return resp;
   }
   
-  function dontCache(request) {
+  function dontInstall(request) {
     if ( ! request.url ) return false;
     const url = new URL(request.url);
     return NEVER_CACHE.has(url.origin) || (State.No && State.No.test(url.host));
   }
 
+  function dontCache(request) {
+    if ( ! request.url ) return true;
+    if ( neverCache(request.url) ) return true;
+    const url = new URL(request.url);
+    return NEVER_CACHE.has(url.origin) || (State.No && State.No.test(url.host));
+  }
+
   async function getResponseData(path) {
     try {
       return JSON.parse(await Fs.promises.readFile(path));

package.json

@@ -42,6 +42,7 @@
     "ws": "latest"
   },
   "devDependencies": {
+    "eslint": "^8.4.1",
     "nexe": "^1.1.6",
     "nodemon": "latest",
     "npx": "^3.0.0",

public/injection.js

@@ -1,7 +1,19 @@
 export function getInjection({sessionId}) {
+  // Notes:
+    // say() function
+      // why aliased? Resistant to page overwriting
+      // just a precaution as we are already in an isolated world here, but this makes
+      // this script more portable if it were introduced globally as well as robust 
+      // against API or behaviour changes of the browser or its remote debugging protocol
+      // in future
   return `
     {
       if ( top === self ) {
+        const ConsoleInfo = console.info.bind(console);
+        const JSONStringify = JSON.stringify.bind(JSON);
+        const TITLE_CHANGES = 10;
+        const INITIAL_CHECK_TIME = 500;
+        const TIME_MULTIPLIER = Math.E;
         const sessionId = "${sessionId}";
         const sleep = ms => new Promise(res => setTimeout(res, ms));
 
@@ -9,10 +21,44 @@ export function getInjection({sessionId}) {
 
         async function install() {
           console.log("Installing...");
-          console.info(JSON.stringify({installed: { sessionId, startUrl: location.href }}));
+          self.startUrl = location.href;
+          say({install: { sessionId, startUrl }});
           await sleep(500);
+          beginTitleChecks();
           console.log("Installed.");
         }
+
+        function beginTitleChecks() {
+          let lastTitle = null;
+          let checker;
+          let timeToNextCheck = INITIAL_CHECK_TIME;
+          let changesLogged = 0;
+
+          check();
+          console.log('Begun logging title changes.');
+
+          function check() {
+            clearTimeout(checker);
+            const currentTitle = document.title; 
+            if ( lastTitle !== currentTitle ) {
+              say({titleChange: {lastTitle, currentTitle, url: location.href, sessionId}});
+              lastTitle = currentTitle;
+              changesLogged++;
+            } else {
+              // increase check time if there's no change
+              timeToNextCheck *= TIME_MULTIPLIER;
+            }
+            if ( changesLogged < TITLE_CHANGES ) {
+              checker = setTimeout(check, timeToNextCheck);
+            } else {
+              console.log('Finished logging title changes.'); 
+            }
+          }
+        }
+
+        function say(thing) {
+          ConsoleInfo(JSONStringify(thing));
+        }
       }
     }
   `;

public/test-injection.html

@@ -0,0 +1 @@
+<script type=module src=injection.js></script>

todo

@@ -1,4 +1,5 @@
 - ensure we are correctly indexing / saving the page title when the page title updates.
+  - issues. Title still out of date. We are incorrectly connecting to newtab, etc.
 - ensure we are getting the page text to index once it is actually loaded (we should call again later, or add mutation observer and update on mutate)
 - add export and import of the FTS index
 - link up to search input and provide results page