"Big progress on integrating search."

archivist.js

@@ -111,15 +111,13 @@ async function collect({chrome_port:port, mode} = {}) {
 
   on("Target.targetInfoChanged", indexURL);
   on("Target.targetInfoChanged", reloadIfNotLive);
-
   on("Target.targetInfoChanged", attachToTarget);
-
   on("Target.attachedToTarget", installForSession);
-
   on("Fetch.requestPaused", cacheRequest);
-
   on("Runtime.consoleAPICalled", confirmInstall);
 
+  await send("Target.setDiscoverTargets", {discover:true});
+  await send("Target.setAutoAttach", {autoAttach:true, waitForDebuggerOnStart:false, flatten: true});
   await send("Fetch.enable", {
     patterns: [
       {
@@ -129,9 +127,6 @@ async function collect({chrome_port:port, mode} = {}) {
     ], 
   });
 
-  await send("Target.setDiscoverTargets", {discover:true});
-  await send("Target.setAutoAttach", {autoAttach:true, waitForDebuggerOnStart:false, flatten: true});
-
   const {targetInfos:targets} = await send("Target.getTargets", {});
   const pageTargets = targets.filter(({type}) => type == 'page');
   pageTargets.forEach(attachToTarget);
@@ -164,7 +159,7 @@ async function collect({chrome_port:port, mode} = {}) {
     if ( attached && type == 'page' ) {
       const {url, targetId} = targetInfo;
       const sessionId = Sessions.get(targetId);
-      if ( !!sessionId && !!url && url != "about:blank" && !url.startsWith('chrome') && !ConfirmedInstalls.has(sessionId) ) {
+      if ( !!sessionId && !neverCache(url) && !ConfirmedInstalls.has(sessionId) ) {
         console.log({reloadingAsNotConfirmedInstalled:{url, sessionId}});
         send("Page.stopLoading", {}, sessionId);
         send("Page.reload", {}, sessionId);
@@ -172,6 +167,10 @@ async function collect({chrome_port:port, mode} = {}) {
     }
   }
 
+  function neverCache(url) {
+    return !url || url == "about:blank" || url.startsWith('chrome') || NEVER_CACHE.has(url);
+  }
+
   async function installForSession({sessionId, targetInfo, waitingForDebugger}) {
     console.log("installForSession called");
     if ( ! sessionId ) {
@@ -230,8 +229,6 @@ async function collect({chrome_port:port, mode} = {}) {
       await untilHas(Installations, sessionId);
     }
 
-    console.log('hi', sessionId);
-
     send("DOMSnapshot.enable", {}, sessionId);
 
     await sleep(500);
@@ -242,12 +239,13 @@ async function collect({chrome_port:port, mode} = {}) {
     const pageText = processDoc(flatDoc);
     //Flex.updateAsync(info.url, pageText).then(r => console.log('Search index update done'));
     //Flex.addAsync(info.url, pageText).then(r => console.log('Search index update done'));
-    const res = Flex.add(info.url, pageText);
+    const res = Flex.update(info.url, pageText);
     DEBUG && console.log('Flex Index Result>>>', res);
 
     State.Indexing.delete(info.targetId);
 
-    console.log(`Indexed ${info.url} to ${info.title}`);
+    const {title, url} = info;
+    console.log({title, url, indexed: true, searchable: true, indexType: 'full text and full content'});
   }
 
   async function untilHas(thing, key) {

common.js

@@ -30,6 +30,20 @@ export const FORBIDDEN_TEXT_PARENT = new Set([
   'SCRIPT',
   'NOSCRIPT'
 ]);
+export const ERROR_CODE_SAFE_TO_IGNORE = new Set([
+  -32000, /* message:
+            Can only get response body on requests captured after headers received.
+           * ignore because: 
+              seems to only happen when new navigation aborts all 
+              pending requests of the unloading page 
+           */
+  -32602, /* message:
+            Invalid InterceptionId.
+           * ignore because: 
+              seems to only happen when new navigation aborts all 
+              pending requests of the unloading page 
+           */
+]);
 
 export const NO_SANDBOX = process.env.DEBUG_22120 || false;
 

protocol.js

@@ -1,9 +1,10 @@
-import {DEBUG, context} from './common.js';
+import {DEBUG, context, ERROR_CODE_SAFE_TO_IGNORE} from './common.js';
 
 const ROOT_SESSION = "browser";
 // actually we use 'tot' but in chrome.debugger.attach 'tot' is 
 // not a supported version string
 const VERSION = "1.3"; 
+const MESSAGES = new Map();
 
 function promisify(context, name, err) {
   return async function(...args) {
@@ -77,7 +78,9 @@ export async function connect({port:port = 9222} = {}) {
       let resolve;
       const promise = new Promise(res => resolve = res);
       Resolvers[key] = resolve;
-      socket.send(JSON.stringify(message));
+      const outGoing = JSON.stringify(message);
+      MESSAGES.set(key, outGoing);
+      socket.send(outGoing);
       DEBUG && console.log("Sent", message);
       return promise;
     }
@@ -86,7 +89,10 @@ export async function connect({port:port = 9222} = {}) {
       const stringMessage = message;
       message = JSON.parse(message);
       if ( message.error ) {
-        console.warn(message);
+        const showError = DEBUG || !ERROR_CODE_SAFE_TO_IGNORE.has(message.error.code);
+        if ( showError ) {
+          console.warn(message);
+        }
       }
       const {sessionId} = message;
       const {method, params} = message;
@@ -105,6 +111,16 @@ export async function connect({port:port = 9222} = {}) {
             console.warn(`Resolver failed`, e, key, stringMessage.slice(0,140), resolve);
           }
         }
+        if ( DEBUG ) {
+          if ( message.error ) {
+            const showError = DEBUG || !ERROR_CODE_SAFE_TO_IGNORE.has(message.error.code);
+            if ( showError ) {
+              const originalMessage = MESSAGES.get(key);
+              console.warn({originalMessage});
+            }
+          }
+        }
+        MESSAGES.delete(key);
       } else if ( method ) {
         const listeners = Handlers[method];
         if ( Array.isArray(listeners) ) {

todo

@@ -1,3 +1 @@
-- fix error on capture response body regarding headers
-- ensure fonts are capturing correctly (the vf fix)
 - continue making 22120 new