split server file

2024-11-08 20:22:44 +01:00 · 2021-04-13 17:26:11 +02:00 · 2021-04-13 17:26:11 +02:00 · 19b199a3cb
commit 19b199a3cb
parent 7716034c20
14 changed files with 3314 additions and 540 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,2 +1,3 @@
 /node_modules
 /dist
+/data
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@ -0,0 +1,2 @@
+{
+}
--- a/create_db.sh
+++ b/create_db.sh
@ -0,0 +1,10 @@
+set -eu
+cd "$(dirname "$0")"
+rm -rf dist/data
+mkdir -p dist/data
+cat create_db.sql | sqlite3 -cmd '.echo on' dist/data/db.sqlite3
+bytes="$(stat --printf="%s" dist/data/db.sqlite3)"
+serverChunkSize=$((50 * 1024 * 1024))
+suffixLength=3
+split data/db.sqlite3 --bytes=$serverChunkSize dist/data/db.sqlite3. --suffix-length=$suffixLength --numeric-suffixes 
+echo '{"requestChunkSize": 4096, "databaseLengthBytes": '$bytes', "serverChunkSize": '$serverChunkSize', "urlPrefix": "db.sqlite3.", "suffixLength": '$suffixLength'}' > dist/data/config.json
--- a/create_db.sql
+++ b/create_db.sql
@ -6,7 +6,7 @@ attach database 'youtube-metadata-pg4096.sqlite3' as ytm;

 CREATE TABLE authors (id integer primary key autoincrement, name text not null unique);

-insert into authors (name) select distinct author from ytm.videoData;
+insert into authors (name) select author from ytm.videoData group by author having count(*) >= 3; -- authors with at least 3 vids in database

 create table videoData as select * from ytm.videoData order by author; -- important to sort here so it can be fetched quickly by author;

@ -17,6 +17,7 @@ CREATE TABLE "sponsorTimes" (
        "startTime"     REAL NOT NULL,
        "endTime"       REAL NOT NULL,
        "votes" INTEGER NOT NULL,
+        "category" TEXT NOT NULL,
        "shadowHidden"  INTEGER NOT NULL,
        "authorID"      INTEGER);

@ -24,7 +25,7 @@ CREATE INDEX "sponsorTimes_videoID" ON "sponsorTimes" ("videoID");
 CREATE INDEX sptauth on sponsorTimes(authorID);

 insert into sponsorTimes
-select s.videoID, s.startTime, s.endTime, s.votes, s.shadowHidden, authors.id
+select s.videoID, s.startTime, s.endTime, s.votes, s.category, s.shadowHidden, authors.id
 from ytm.sponsorTimes s
 join ytm.videoData v on v.videoID = s.videoID
 join authors on authors.name = v.author
--- a/index.html
+++ b/index.html
@ -2,7 +2,7 @@

 <meta charset="utf-8">

-<title>Sponsorblock stat by uploader</title>
+<title>Sponsorship stat by uploader</title>

 <meta http-equiv="X-UA-Compatible" content="IE=edge">
 <meta name="viewport" content="width=device-width, initial-scale=1">
--- a/package.json
+++ b/package.json
@ -3,17 +3,20 @@
    "dev": "webpack serve"
  },
  "dependencies": {
-    "@types/better-sqlite3": "^5.4.1",
+    "@types/debounce-promise": "^3.1.3",
    "@types/react": "^17.0.3",
    "@types/react-dom": "^17.0.3",
+    "@types/react-plotly.js": "^2.2.4",
    "@types/react-select": "^4.0.14",
    "@types/sql.js": "^1.4.2",
-    "better-sqlite3": "^7.1.4",
    "comlink": "^4.3.0",
+    "debounce-promise": "^3.1.2",
    "mobx": "^6.2.0",
    "mobx-react": "^7.1.0",
+    "plotly.js": "^1.58.4",
    "react": "^17.0.2",
    "react-dom": "^17.0.2",
+    "react-plotly.js": "^2.5.1",
    "react-select": "^4.3.0",
    "ts-loader": "^8.1.0",
    "ts-node": "^9.1.1",
--- a/sql.js/Makefile
+++ b/sql.js/Makefile
@ -21,8 +21,7 @@ CFLAGS = \
 	-O2 \
 	-DSQLITE_OMIT_LOAD_EXTENSION \
 	-DSQLITE_DISABLE_LFS \
-	-DSQLITE_ENABLE_FTS3 \
-	-DSQLITE_ENABLE_FTS3_PARENTHESIS \
+	-DSQLITE_ENABLE_FTS5 \
 	-DSQLITE_ENABLE_JSON1 \
 	-DSQLITE_THREADSAFE=0 \
 	-DSQLITE_ENABLE_NORMALIZE
@ -54,8 +53,7 @@ EMFLAGS_WASM = \
 EMFLAGS_OPTIMIZED= \
 	-s INLINING_LIMIT=50 \
 	-O3 \
-	-flto \
-	--closure 1
+	-flto

 EMFLAGS_DEBUG = \
 	-s INLINING_LIMIT=10 \
--- a/sql.js/src/api.js
+++ b/sql.js/src/api.js
@ -816,10 +816,10 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
        // (created by create_function call)
        this.functions = {};
    }
-    function UrlDatabase(url, chunksize = 4096) {
-        this.filename = url.replace("/", "_");
-        const {createLazyFile} = require('../../src/lazyFile');
-        this.lazyFile = createLazyFile(FS, "/", this.filename, url, true, true, chunksize);
+
+    function UrlDatabase(filename, lazyFile) {
+        this.filename = filename;
+        this.lazyFile = lazyFile;
        const ret = sqlite3_open(this.filename, apiTemp, 1, null);
        this.db = getValue(apiTemp, "i32");
        this.handleError(ret);
@ -1231,9 +1231,9 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {

    // export Database to Module
    Module.Database = Database;
-    Module.VfsDatabase = VfsDatabase;
-    Module.UrlDatabase = UrlDatabase;
-    Module.FS = FS;
+    Module["VfsDatabase"] = VfsDatabase;
+    Module["UrlDatabase"] = UrlDatabase;
+    Module["FS"] = FS;
    VfsDatabase.prototype = Object.create(Database.prototype);
    UrlDatabase.prototype = Object.create(Database.prototype);
 };
--- a/src/UI.tsx
+++ b/src/UI.tsx
@ -1,57 +1,228 @@
 import { observer, useLocalObservable } from "mobx-react";
 import * as React from "react";
 import { Database } from "sql.js";
-import { authorsSearch, createDbWorker, getForAuthor } from "./db";
-import { makeAutoObservable, makeObservable, observable } from "mobx";
-import AsyncSelect from 'react-select/async';
+import {
+  authorsSearch,
+  createDbWorker,
+  getForAuthor,
+  SponsorInfo,
+  SqliteWorker,
+  VideoMeta,
+} from "./db";
+import { action, makeAutoObservable, makeObservable, observable } from "mobx";
+import AsyncSelect from "react-select/async";
+import debounce from "debounce-promise";
+import Plot from "react-plotly.js";
+function formatDuration(sec_num: number) {
+  const hours = Math.floor(sec_num / 3600);
+  const minutes = Math.floor((sec_num - hours * 3600) / 60);
+  const seconds = Math.round(sec_num - hours * 3600 - minutes * 60);
+
+  return (
+    String(hours).padStart(2, "0") +
+    ":" +
+    String(minutes).padStart(2, "0") +
+    ":" +
+    String(seconds).padStart(2, "0")
+  );
+}
+const SponsorPlot: React.FC<{
+  data: SponsorInfo[];
+  onHover: (m: SponsorInfo) => void;
+}> = observer((p) => {
+  console.log("RERENTDERING PLOT");
+  return (
+    <Plot
+      style={{ width: "100%", maxWidth: "1200px", margin: "0 auto" }}
+      onClick={(e) => {
+        console.log("hover", e);
+        const element = p.data[e.points[0].pointIndex];
+        if (element) p.onHover(element);
+      }}
+      data={[
+        {
+          x: p.data.map((e) => new Date(e.meta.published * 1000)),
+          y: p.data.map((e) => e.percentSponsor / 100),
+
+          text: p.data.map(
+            (e) =>
+              `<b>${e.meta.title}</b><br>
+              published ${new Date(
+                e.meta.published * 1000
+              ).toLocaleDateString()}<br>
+              Length: ${formatDuration(e.meta.lengthSeconds)}<br>
+              Sponsor duration: ${formatDuration(
+                e.durationSponsor
+              )} (<b>${e.percentSponsor.toFixed(0)}%</b>)`
+          ),
+          hovertemplate: "%{text}",
+          type: "scatter",
+          mode: "markers",
+        },
+      ]}
+      layout={{
+        autosize: true,
+        yaxis: { tickformat: ",.0%", title: "Part that is Sponsorship" },
+        xaxis: { title: "Upload date" },
+      }}
+    />
+  );
+});
+
+type SqliteStats = {
+  filename: string;
+  totalBytes: number;
+  totalFetchedBytes: number;
+  totalRequests: number;
+};
+function formatBytes(b: number) {
+  if (b > 1e6) {
+    return (b / 1e6).toFixed(2) + "MB";
+  }
+  if (b > 1e3) {
+    return (b / 1e3).toFixed(2) + "KB";
+  }
+  return b + "B";
+}
+
+const SqliteStats: React.FC<{ stats: SqliteStats }> = observer(({ stats }) => {
+  return (
+    <>
+      Sqlite stats: fetched {formatBytes(stats.totalFetchedBytes)} in{" "}
+      {stats.totalRequests} requests (DB size: {formatBytes(stats.totalBytes)})
+    </>
+  );
+});
+
+const VideoMetaDisplay: React.FC<{ video: SponsorInfo }> = observer(
+  ({ video }) => {
+    return (
+      <div>
+        <a href={`https://youtube.com/watch?v=${video.meta.videoID}`}>
+          <img
+            src={video.meta.maxresdefault_thumbnail}
+            width={200}
+            style={{ float: "left" }}
+          ></img>
+          <h4>{video.meta.title}</h4>
+        </a>
+        {video.meta.viewCount} views
+        <br />
+        published {new Date(video.meta.published * 1000).toLocaleDateString()}
+        <br />
+        Length: {formatDuration(video.meta.lengthSeconds)}
+        <br />
+        Sponsor duration: {formatDuration(video.durationSponsor)} (
+        <b>{video.percentSponsor.toFixed(0)}%</b>)
+      </div>
+    );
+  }
+);

@observer
 export class UI extends React.Component {
+  worker: SqliteWorker | null = null;
  db: Database | null = null;
-  @observable authorSearch = "";
  @observable
-  suggestions = {
-    error: "type more",
-    results: [] as string[],
+  data:
+    | { state: "noinput" }
+    | { state: "loading"; author: string }
+    | { state: "loaded"; author: string; segs: SponsorInfo[] } = {
+    state: "noinput",
  };
  @observable
-  data: { author: string; segs: any[] } | null = null;
+  stats: SqliteStats | null = null;
+  @observable
+  focussedVideo: SponsorInfo | null = null;
+  @observable searchInput: string = "";

  constructor(p: {}) {
    super(p);
    this.init();
    makeObservable(this);
  }
-  async init() {
-    this.db = await createDbWorker();
+  interval: number = 0;
+  componentDidMount() {
+    this.interval = setInterval(async () => {
+      this.stats = (await this.worker?.getStats()) || null;
+    }, 1000);
  }
-  async setAuthor(t: string) {
-    this.authorSearch = t;
+  componentWillUnmount() {
+    clearInterval(this.interval);
+  }
+  async init() {
+    console.log("INIT");
+    const res = await createDbWorker();
+    this.db = res.db;
+    this.worker = res.worker;
+    const initialAuthor = new URLSearchParams(location.search).get("uploader");
+    if (initialAuthor) this.setAuthor(initialAuthor);
+  }
+  async setAuthor(search: string) {
+    this.searchInput = search;
+    this.focussedVideo = null;
    if (this.db) {
-      const search = this.authorSearch;
      this.data = {
+        state: "loading",
+        author: search,
+      };
+      this.data = {
+        state: "loaded",
        author: search,
        segs: await getForAuthor(this.db, search),
      };
      console.log("data", this.data);
+      {
+        const searchParams = new URLSearchParams(location.search);
+        searchParams.set("uploader", search);
+        window.history.replaceState(null, document.title, "?" + searchParams);
+      }
    }
  }
  async authorsSearch(search: string) {
-    return (await authorsSearch(this.db!, search))
+    if (this.db) {
+      return await authorsSearch(this.db, search);
+    }
+    return [];
  }
+  authorsSearchDebounce = debounce(this.authorsSearch.bind(this), 250, {
+    leading: true,
+  });
+  @action
+  setFocussed = (e: SponsorInfo) => (this.focussedVideo = e);

  render() {
    return (
      <div>
        <div>
-          Search for author:{" "}
-          <AsyncSelect<{name: string}> cacheOptions defaultOptions loadOptions={this.authorsSearch.bind(this)} />
+          Search for YouTuber:{" "}
+          <AsyncSelect<{ name: string }>
+            cacheOptions
+            inputValue={this.searchInput}
+            onInputChange={(e) => (this.searchInput = e)}
+            loadOptions={this.authorsSearchDebounce}
+            getOptionLabel={(e) => e.name}
+            getOptionValue={(e) => e.name}
+            onChange={(e) => this.setAuthor(e.name)}
+          />
        </div>
-        {this.data && (
+        {this.data.state === "noinput" ? (
+          <></>
+        ) : this.data.state === "loading" ? (
+          <div>Loading videos for author "{this.data.author}"</div>
+        ) : (
          <div>
-            Found {this.data.segs.length} videos for author "{this.data.author}"
+            <p>
+              Found {this.data.segs.length} videos with sponsorships from{" "}
+              {this.data.author}
+            </p>{" "}
+            <SponsorPlot data={this.data.segs} onHover={this.setFocussed} />
          </div>
        )}
+        {this.focussedVideo && <VideoMetaDisplay video={this.focussedVideo} />}
+        <footer style={{ marginTop: "5em", color: "gray" }}>
+          {this.stats ? <SqliteStats stats={this.stats} /> : ""}{" "}
+        </footer>
      </div>
    );
  }
--- a/src/db.ts
+++ b/src/db.ts
@ -1,7 +1,5 @@
-import initSqlJs from "../sql.js/dist/sql-wasm-debug.js";
-import wasmfile from "../sql.js/dist/sql-wasm-debug.wasm";
 import * as Comlink from "comlink";
-import SqliteWorker from "./sqlite.worker";
+import SqliteWorker, { SplitFileConfig } from "./sqlite.worker";

 import { chooseSegments, DBSegment } from "./util";
 import { SqliteMod } from "./sqlite.worker.js";
@ -19,11 +17,17 @@ Comlink.transferHandlers.set("WORKERSQLPROXIES", {
    return Comlink.wrap(port);
  },
 });
+export type SqliteWorker = Comlink.Remote<SqliteMod>;
 export async function createDbWorker() {
  const sqlite = Comlink.wrap<SqliteMod>(new SqliteWorker());

  const chunkSize = 4096;
-  const db = await sqlite.UrlDatabase("db.sqlite3", chunkSize);
+  const configUrl = new URL("dist/data/config.json", location.href);
+  const config: SplitFileConfig = await fetch(configUrl.toString()).then(e => e.json());
+  const db = await sqlite.SplitFileHttpDatabase({
+    ...config,
+    urlPrefix: new URL(config.urlPrefix, configUrl).toString(),
+  });
  const pageSizeResp = await db.exec("pragma page_size");
  const pageSize = pageSizeResp[0].values[0][0];
  if (pageSize !== chunkSize)
@ -31,8 +35,7 @@ export async function createDbWorker() {
      `Chunk size does not match page size: pragma page_size = ${pageSize} but chunkSize = ${chunkSize}`
    );

-
-  return db;
+  return { worker: sqlite, db };
 }

 async function testLoop(metaDb: Database) {
@ -57,7 +60,7 @@ async function go() {

 function toObjects<T>(res: QueryExecResult[]): T[] {
  const r = res[0];
-  if(!r) return [];
+  if (!r) return [];
  return r.values.map((v) => {
    const o: any = {};
    for (let i = 0; i < r.columns.length; i++) {
@ -66,26 +69,46 @@ function toObjects<T>(res: QueryExecResult[]): T[] {
    return o as T;
  });
 }
-type VideoMeta = {
+export type VideoMeta = {
  videoID: string;
-  published: number;
-  lengthSeconds: number;
  title: string;
+  maxresdefault_thumbnail: string;
+  published: number;
+  publishedText: string;
+  viewCount: number;
+  likeCount: number;
+  author: string;
+  authorURL: string;
+  channelThumbnail: string;
+  lengthSeconds: number;
+  category: string;
 };
 export async function authorsSearch(db: Database, author: string) {
  try {
-  const query_inner = author.split(" ").map(n => n.replace(/"/g, "")).map(e => `"${e}"`).join(" ");
-  const query = `NEAR(${query_inner})`;
-  const sql_query = `select name from authors_search where name match ? order by rank limit 20`;
-  console.log("executing search query", query, sql_query);
-  const ret = toObjects<{name: string}>(await db.exec(sql_query, [query]));
-  return ret;
-  } catch(e) {
+    const query_inner = author
+      .split(" ")
+      .map((n) => n.replace(/"/g, ""))
+      .map((e) => `"${e}"*`)
+      .join(" ");
+    const query = `NEAR(${query_inner})`;
+    const sql_query = `select name from authors_search where name match ? limit 20`;
+    console.log("executing search query", query, sql_query);
+    const ret = toObjects<{ name: string }>(await db.exec(sql_query, [query]));
+    return ret;
+  } catch (e) {
    console.error("authorsSearch", e);
    throw e;
  }
 }
-export async function getForAuthor(db: Database, author: string) {
+export type SponsorInfo = {
+  meta: VideoMeta;
+  durationSponsor: number;
+  percentSponsor: number;
+};
+export async function getForAuthor(
+  db: Database,
+  author: string
+): Promise<SponsorInfo[]> {
  /*await db.exec(`select s.rowid from sponsorTimes s
  join videoData v on s.videoid = v.videoid
  
@ -93,7 +116,7 @@ export async function getForAuthor(db: Database, author: string) {

  const videos = toObjects<VideoMeta>(
    await db.exec(
-      "select videoData.videoID, published, lengthSeconds, title from videoData where author = ?",
+      "select * from videoData where author = ? order by published asc",
      [author]
    )
  );
@ -102,7 +125,7 @@ export async function getForAuthor(db: Database, author: string) {
    await db.exec(
      // "select videoData.videoID, published, lengthSeconds, title from videoData join sponsorTimes on sponsorTimes.videoID = videoData.videoID where author = ? order by sponsorTimes.rowid asc",
      // [author]
-      "select * from sponsorTimes where authorID = (select id from authors where name = ?) order by videoID, startTime",
+      "select * from sponsorTimes where authorID = (select id from authors where name = ?) and not shadowHidden and category = 'sponsor' order by videoID, startTime",
      [author]
    )
  ); // select sponsorTimes.rowid, sponsorTimes.videoID from videoData join sponsorTimes on sponsorTimes.videoID = videoData.videoID where author = 'Adam Ragusea';
@ -130,17 +153,14 @@ export async function getForAuthor(db: Database, author: string) {
    const segments = chooseSegments(sponsorTimes.filter((s) => s.votes > -1));
    const duration = segments
      .map((m) => m.endTime - m.startTime)
-      .reduce((a, b) => a + b);
+      .reduce((a, b) => a + b, 0);
    const total = video.meta.lengthSeconds;
-    const percent = (duration / total) * 100;
+    const percentSponsor = (duration / total) * 100;
    out.push({
+      meta: video.meta,
      durationSponsor: duration,
-      durationTotal: total,
-      percent,
-      videoTitle: video.meta.title,
-      videoID: video.meta.videoID,
-      published: new Date(video.meta.published * 1000)
-    })
+      percentSponsor,
+    });
  }
  return out;
 }
--- a/src/lazyFile.ts
+++ b/src/lazyFile.ts
@ -4,6 +4,18 @@
 // XHR, which is not possible in browsers except in a web worker! Use preloading,
 // either --preload-file in emcc or FS.createPreloadedFile

+export type RangeMapper = (
+  fromByte: number,
+  toByte: number
+) => { url: string; fromByte: number; toByte: number };
+
+export type LazyFileConfig = {
+  rangeMapper: RangeMapper;
+  /** must be known beforehand if there's multiple server chunks (i.e. rangeMapper returns different urls) */
+  fileLength?: number;
+  requestChunkSize: number;
+};
+
 // Lazy chunked Uint8Array (implements get and length from Uint8Array). Actual getting is abstracted away for eventual reuse.
 class LazyUint8Array {
  lengthKnown = false;
@ -14,7 +26,17 @@ class LazyUint8Array {

  lastEnd = 0;
  speed = 1;
-  constructor(private _chunkSize: number, private url: string) {}
+  _chunkSize: number;
+  rangeMapper: RangeMapper;
+
+  constructor(config: LazyFileConfig) {
+    this._chunkSize = config.requestChunkSize;
+    this.rangeMapper = config.rangeMapper;
+    if(config.fileLength) {
+      this._length = config.fileLength;
+      this.lengthKnown = true;
+    }
+  }
  get(idx: number) {
    if (idx > this.length - 1 || idx < 0) {
      return undefined;
@ -25,22 +47,32 @@ class LazyUint8Array {
  }
  getter(chunkNum: number) {
    const start = chunkNum * this.chunkSize;
-   
+
    if (typeof this.chunks[chunkNum] === "undefined") {
-      if(this.lastEnd === start - 1) {
+      if (this.lastEnd === start - 1) {
        this.speed = Math.min(64, this.speed * 2);
      } else {
        this.speed = 1;
      }
      const chunksToFetch = this.speed;
-      let end = (chunkNum + chunksToFetch) * this.chunkSize - 1; // including this byte
-      end = Math.min(end, this.length - 1); // if datalength-1 is selected, this is the last block
+      let endByte = (chunkNum + chunksToFetch) * this.chunkSize - 1; // including this byte
+      endByte = Math.min(endByte, this.length - 1); // if datalength-1 is selected, this is the last block

-      this.lastEnd = end;
-      const buf = this.doXHR(start, end);
-      for(let i = 0; i < chunksToFetch; i++) {
+      this.lastEnd = endByte;
+      const buf = this.doXHR(start, endByte);
+      for (let i = 0; i < chunksToFetch; i++) {
        const curChunk = chunkNum + i;
-        this.chunks[curChunk] = new Uint8Array(buf, i * this.chunkSize, this.chunkSize);
+        if (i * this.chunkSize >= buf.byteLength) break; // past end of file
+        const curSize =
+          (i + i) * this.chunkSize > buf.byteLength
+            ? buf.byteLength - i * this.chunkSize
+            : this.chunkSize;
+        // console.log("constructing chunk", buf.byteLength, i * this.chunkSize, curSize);
+        this.chunks[curChunk] = new Uint8Array(
+          buf,
+          i * this.chunkSize,
+          curSize
+        );
      }
    }
    if (typeof this.chunks[chunkNum] === "undefined")
@ -50,10 +82,11 @@ class LazyUint8Array {
  cacheLength() {
    // Find length
    var xhr = new XMLHttpRequest();
-    xhr.open("HEAD", this.url, false);
+    const url = this.rangeMapper(0, 0).url;
+    xhr.open("HEAD", url, false);
    xhr.send(null);
    if (!((xhr.status >= 200 && xhr.status < 300) || xhr.status === 304))
-      throw new Error("Couldn't load " + this.url + ". Status: " + xhr.status);
+      throw new Error("Couldn't load " + url + ". Status: " + xhr.status);
    var datalength = Number(xhr.getResponseHeader("Content-length"));

    var header;
@ -82,22 +115,32 @@ class LazyUint8Array {
    }
    return this._chunkSize!;
  }
-  private doXHR(from: number, to: number) {
-    console.log(`- [xhr of size ${(to + 1-from)/1024} KiB]`);
-    this.totalFetchedBytes += to - from;
+  private doXHR(absoluteFrom: number, absoluteTo: number) {
+    console.log(
+      `- [xhr of size ${(absoluteTo + 1 - absoluteFrom) / 1024} KiB]`
+    );
+    this.totalFetchedBytes += absoluteTo - absoluteFrom;
    this.totalRequests++;
-    if (from > to)
+    if (absoluteFrom > absoluteTo)
      throw new Error(
-        "invalid range (" + from + ", " + to + ") or no bytes requested!"
+        "invalid range (" +
+          absoluteFrom +
+          ", " +
+          absoluteTo +
+          ") or no bytes requested!"
      );
-    if (to > this.length - 1)
+    if (absoluteTo > this.length - 1)
      throw new Error(
        "only " + this.length + " bytes available! programmer error!"
      );
+    const { fromByte: from, toByte: to, url } = this.rangeMapper(
+      absoluteFrom,
+      absoluteTo
+    );

    // TODO: Use mozResponseArrayBuffer, responseStream, etc. if available.
    var xhr = new XMLHttpRequest();
-    xhr.open("GET", this.url, false);
+    xhr.open("GET", url, false);
    if (this.length !== this.chunkSize)
      xhr.setRequestHeader("Range", "bytes=" + from + "-" + to);

@ -109,7 +152,7 @@ class LazyUint8Array {

    xhr.send(null);
    if (!((xhr.status >= 200 && xhr.status < 300) || xhr.status === 304))
-      throw new Error("Couldn't load " + this.url + ". Status: " + xhr.status);
+      throw new Error("Couldn't load " + url + ". Status: " + xhr.status);
    if (xhr.response !== undefined) {
      return xhr.response as ArrayBuffer;
    } else {
@ -122,24 +165,15 @@ export function createLazyFile(
  FS: any,
  parent: string,
  name: string,
-  url: string,
  canRead: boolean,
  canWrite: boolean,
-  chunkSize: number = 4096
+  lazyFileConfig: LazyFileConfig
 ) {
-  var lazyArray = new LazyUint8Array(chunkSize, url);
+  var lazyArray = new LazyUint8Array(lazyFileConfig);
  var properties = { isDevice: false, contents: lazyArray };

  var node = FS.createFile(parent, name, properties, canRead, canWrite);
-  // This is a total hack, but I want to get this lazy file code out of the
-  // core of MEMFS. If we want to keep this lazy file concept I feel it should
-  // be its own thin LAZYFS proxying calls to MEMFS.
-  if (properties.contents) {
-    node.contents = properties.contents;
-  } else if (properties.url) {
-    node.contents = null;
-    node.url = properties.url;
-  }
+  node.contents = lazyArray;
  // Add a function that defers querying the file size until it is asked the first time.
  Object.defineProperties(node, {
    usedBytes: {
--- a/src/sqlite.worker.ts
+++ b/src/sqlite.worker.ts
@ -1,13 +1,14 @@
 import * as Comlink from "comlink";
 import wasmfile from "../sql.js/dist/sql-wasm-debug.wasm";
 import initSqlJs from "../sql.js/dist/sql-wasm-debug.js";
-
-type Database = import("sql.js").Database;
+import { createLazyFile, RangeMapper } from "./lazyFile";
+import { getSyntheticTrailingComments } from "typescript";
+import { Database } from "sql.js";

 // https://gist.github.com/frankier/4bbc85f65ad3311ca5134fbc744db711
 function initTransferHandlers(sql: typeof import("sql.js")) {
  Comlink.transferHandlers.set("WORKERSQLPROXIES", {
-    canHandle: (obj) => {
+    canHandle: (obj): obj is unknown => {
      let isDB = obj instanceof sql.Database;
      let hasDB = obj.db && obj.db instanceof sql.Database; // prepared statements
      return isDB || hasDB;
@ -24,15 +25,6 @@ function initTransferHandlers(sql: typeof import("sql.js")) {
  });
 }

-function stats(db: Database) {
-  console.log(
-    db.filename,
-    "total bytes fetched:",
-    db.lazyFile.contents.totalFetchedBytes,
-    "total requests:",
-    db.lazyFile.contents.totalRequests
-  );
-}
 async function init() {
  const sql = await initSqlJs({
    locateFile: (_file: string) => wasmfile,
@ -41,13 +33,49 @@ async function init() {
  return sql;
 }
 const sqljs = init();
+export type SplitFileConfig = {
+  urlPrefix: string;
+  serverChunkSize: number;
+  databaseLengthBytes: number;
+  requestChunkSize: number;
+};
 const mod = {
-  async UrlDatabase(url: string, chunkSize: number): Promise<Database> {
+  db: null as null | Database,
+  async SplitFileHttpDatabase(p: SplitFileConfig): Promise<Database> {
    const sql = await sqljs;
-    const db = new sql.UrlDatabase(url, chunkSize);
+    console.log("constructing url database");
+    const rangeMapper: RangeMapper = (from: number, to: number) => {
+      const serverChunkId = (from / p.serverChunkSize) | 0;
+      const serverFrom = from % p.serverChunkSize;
+      const serverTo = serverFrom + (to - from);
+      return {
+        url: p.urlPrefix + String(serverChunkId).padStart(3, "0"),
+        fromByte: serverFrom,
+        toByte: serverTo,
+      };
+    };

-    setInterval(() => stats(db), 10000);
-    return db;
+    const filename = p.urlPrefix.replace(/\//g, "_");
+    console.log("filename", filename);
+    const lazyFile = createLazyFile(sql.FS, "/", filename, true, true, {
+      rangeMapper,
+      requestChunkSize: p.requestChunkSize,
+      fileLength: p.databaseLengthBytes,
+    });
+
+    this.db = new sql.UrlDatabase(filename, lazyFile);
+
+    return this.db!;
+  },
+  async getStats() {
+    const db = this.db;
+    if (!db) return null;
+    return {
+      filename: db.filename,
+      totalBytes: db.lazyFile.contents.length,
+      totalFetchedBytes: db.lazyFile.contents.totalFetchedBytes,
+      totalRequests: db.lazyFile.contents.totalRequests,
+    };
  },
 };
 export type SqliteMod = typeof mod;
--- a/webpack.config.js
+++ b/webpack.config.js
@ -1,39 +1,44 @@
 const path = require("path");

 const ts = {
-    loader: "ts-loader",
-    options: { transpileOnly: true },
+  loader: "ts-loader",
+  options: { transpileOnly: true },
 };
 module.exports = {
-    entry: "./src",
-    mode: "development",
-    module: {
-        rules: [
-            {
-                test: /\.tsx?$/,
-                use: ts,
-                exclude: /node_modules/,
-            },
-            {
-                test: /\.worker.ts$/,
-                use: [{loader: "worker-loader"}, ts],
-            },
-            { test: /\.wasm$/, type: "asset/resource" },
-        ],
+  entry: "./src",
+  mode: "development",
+  devtool: "source-map",
+  module: {
+    rules: [
+      {
+        test: /\.tsx?$/,
+        use: ts,
+        exclude: /node_modules/,
+      },
+      {
+        test: /\.worker.ts$/,
+        use: [{ loader: "worker-loader" }, ts],
+      },
+      { test: /\.wasm$/, type: "asset/resource" },
+    ],
+  },
+  resolve: {
+    extensions: [".tsx", ".ts", ".js"],
+    fallback: {
+      fs: false,
+      crypto: false,
+      path: false,
    },
-    resolve: {
-        extensions: [".tsx", ".ts", ".js"],
-        fallback: {
-            fs: false,
-            crypto: false,
-            path: false,
-        },
-    },
-    output: {
-        filename: "bundle.js",
-        path: path.resolve(__dirname, "dist"),
-    },
-    stats: {
-        children: true
-    }
+  },
+  output: {
+    filename: "bundle.js",
+    path: path.resolve(__dirname, "dist"),
+  },
+  stats: {
+    children: true,
+  },
+  devServer: {
+    hot: false,
+    liveReload: false,
+  },
 };
--- a/yarn.lock
+++ b/yarn.lock