generated at
scrapbox-link-database@0.2.0
scrapbox projectごとのlink情報をIndexed DBに格納したもの
複数のUserScriptから参照できるようにする
APIのcacheとして用いる

使い方
js
import {get, clear} from '/api/code/programming-notes/scrapbox-link-database@0.2.0/script.js'; // projectのリンク情報を取得する const [{id, project, pages}] = await get(['57ba889cc59c3e0f00979915' /*shokai*/]); // projectのリンク情報を削除する await clear(['57ba889cc59c3e0f00979915' /*shokai*/]);

scrapbox-link-databaseが動かなくなったので作り変える
classは止める
bundleしづらくなる
コードが複雑になる

仕組み
databaseの操作にはidbを使う
cache戦略

data structure
cache-links
api/pages/:projectname/search/titlesの情報を格納したもの
project: string
scrapbox project name
これをkeyにする
fetched: Date
データを取得した日時
index化しておく
pages: Page[]
link情報
ts
type Page = { title: string; // ページタイトル hasIcon: boolean; // アイコンが有るかどうか links: string[] // 内部リンクのリスト }
api/pages/:projectname/search/titlesにアイコンデータも含まれるようになったので、database を2つ作る必要がなくなった
一度に情報を取得できる

実装したいこと
done補完ソースの生成処理をworkerに移譲する
リンクデータのfetchにかなり時間がかかるので、UI threadをblockしないようにしたい
update() をworkerに移す
codeはESModulesで書き、data URLに変換して読み込む

2021-06-27
12:05:57 リンクデータのfetchをweb workerに移譲した
何故かData URIで読み込むと、indexedDB null になってしまう
text/javascript application/javascript もだめ
JS fileから読み込んだときは問題ない
iifeをそのままworkerに読ませたらDataCloneError: The object could not be cloned.がでた
12:05:13 await を付け忘れてPromiseを直接postMessage()に渡してしまっただけだった
Promise を返そうとしてもこのエラーが出るっぽいな
11:26:58 設定変数を別ファイルに切り出した
web workerからも使いたかったので
2021-06-19
22:10:33 get のintefaceを変える
更新されたかどうかを表すフラグを返り値に追加する
15:59:12 存在しないprojectのidは除外する
↓の原因はこれだった
14:40:12 どうやら data があるものとないものとがあるらしい
ないやつは、データ取得に失敗している?
どのprojectだろう?
データのないやつを一覧してみる
15:29:14 参加しているprojectsの情報が取得されていなかった
こうしたら取得されるようになった
演算子の優先順位が変だったみたい
diff
- ({name, id, updated}) => - updated > projectUpdateds.find(data => data.id === id)?.prevFetched ?? 0 ? - [{project: name, id}] : - [] - ); + ({name, id, updated}) => { + const prevFetched = (projectUpdateds.find(data => data.id === id)?.prevFetched ?? 0); + console.log({name, updated, prevFetched}); + return updated > prevFetched ? [{project: name, id}] : []; + });
でも何故かdatabaseには保存されない……
databaseを見てみると、cacheされているprojectとされていないprojectとがある
get() で返されるprojectのなかで、 pages undefined になるのはやはりcacheされていないprojectだけだ
22 projectsが該当する
それなのにconsoleで64 projectsと表示されるのは、scrapboxのproject情報を一括して取得するUserScriptにて参加しているprojectの情報も返されてしまうから
ここから参加しているprojectを除いて取得できるようにコードを変えよう
15:47:09 変えた
それでもどの参加しているprojectもcacheから読み取れない……
まさか容量超過?
でもこの程度のデータ容量で超過するわけないのだが……
14:31:24 await store.get(id) でデータは取得できている
dev toolでも確認できる
なのにそこからdataを更に取り出そうとすると undefined に変わってしまう……
14:03:46 APIを叩きすぎてtimeoutしてしまったので、先にそっちを調節する
13:51:24 keyを project から projectId に変更した
scrapboxのproject情報を一括して取得するUserScriptでproject情報を一括して取得できるようにした
……のだが、何故かdatabaseからのデータ取得に失敗する
scrapbox-link-database@0.2.0#60cd63f61280f0000080a658 undefined has no properties になる
原因を調べる
13:27:02 とりあえず完成
dataのfetchはもう少し工夫したほうがいいかも
serverに負担がかからないように少しずらす

dependencies
script.js
import { openDB } from '../idb/with-async-ittr.js'; import {src} from './workerSrc.js'; import {DBName, Version, StoreName} from './settings.js'; const worker = new Worker(src); // databaseを初期化する function initialize() { return openDB(DBName, Version, { // 更新処理 upgrade(db) { // Object Storeをすべて消す [...db.objectStoreNames].forEach(storeName => db.deleteObjectStore(storeName) ); // object storeを作り直す const store = db.createObjectStore(StoreName, { keyPath: 'id', }); store.createIndex('fetched', 'fetched'); }, }); } // 初期化 const getDB = initialize(); export async function get(projectIds, options) { // 予め重複を消しておく projectIds = [...new Set(projectIds)]; if (projectIds.length === 0) return {}; // 先に更新する const hasUpdate = await update(projectIds, options); // databaseから取ってくる const db = await getDB; const tx = db.transaction(StoreName, 'readwrite'); const store = tx.objectStore(StoreName); const result = await Promise.all(projectIds.map(async id => { const data = await store.get(id); return data ? {id, project: data.project, pages: data.pages} : undefined; })); await tx.done; return {data: result.filter(data => data), hasUpdate}; } // cacheを消す export async function clear(projectIds) { // cacheを削除する const db = await getDB; const tx = db.transaction(StoreName, 'readwrite'); const store = tx.objectStore(StoreName); await Promise.all(projectIds.map(id => store.delete(id))); await tx.done; } // dataを更新する // reloadをつけるとprojects全てをnetworkから取得し直すが、更新が見つからなければ何もしない function update(projectIds, options) { return new Promise(resolve => { const handleMessage = ({data}) => { worker.removeEventListener('message', handleMessage); resolve(data); }; worker.addEventListener('message', handleMessage); worker.postMessage({projectIds, options}); }); }

settings.js
export const StoreName = 'cache-links'; export const DBName = 'UserScript'; export const Version = 7;

workerSrc.js
export const src = '/api/code/programming-notes/scrapbox-link-database@0.2.0/worker_min.js';
worker_min.js
(()=>{var L=(t,r)=>r.some(n=>t instanceof n),me,le;function Fe(){return me||(me=[IDBDatabase,IDBObjectStore,IDBIndex,IDBCursor,IDBTransaction])}function Le(){return le||(le=[IDBCursor.prototype.advance,IDBCursor.prototype.continue,IDBCursor.prototype.continuePrimaryKey])}var ce=new WeakMap,V=new WeakMap,pe=new WeakMap,G=new WeakMap,W=new WeakMap;function Pe(t){let r=new Promise((n,i)=>{let d=()=>{t.removeEventListener("success",s),t.removeEventListener("error",u)},s=()=>{n(v(t.result)),d()},u=()=>{i(t.error),d()};t.addEventListener("success",s),t.addEventListener("error",u)});return r.then(n=>{n instanceof IDBCursor&&ce.set(n,t)}).catch(()=>{}),W.set(r,t),r}function He(t){if(V.has(t))return;let r=new Promise((n,i)=>{let d=()=>{t.removeEventListener("complete",s),t.removeEventListener("error",u),t.removeEventListener("abort",u)},s=()=>{n(),d()},u=()=>{i(t.error||new DOMException("AbortError","AbortError")),d()};t.addEventListener("complete",s),t.addEventListener("error",u),t.addEventListener("abort",u)});V.set(t,r)}var Z={get(t,r,n){if(t instanceof IDBTransaction){if(r==="done")return V.get(t);if(r==="objectStoreNames")return t.objectStoreNames||pe.get(t);if(r==="store")return n.objectStoreNames[1]?void 0:n.objectStore(n.objectStoreNames[0])}return v(t[r])},set(t,r,n){return t[r]=n,!0},has(t,r){return t instanceof IDBTransaction&&(r==="done"||r==="store")?!0:r in t}};function P(t){Z=t(Z)}function Re(t){return t===IDBDatabase.prototype.transaction&&!("objectStoreNames"in IDBTransaction.prototype)?function(r,...n){let i=t.call(k(this),r,...n);return pe.set(i,r.sort?r.sort():[r]),v(i)}:Le().includes(t)?function(...r){return t.apply(k(this),r),v(ce.get(this))}:function(...r){return v(t.apply(k(this),r))}}function Qe(t){return typeof t=="function"?Re(t):(t instanceof IDBTransaction&&He(t),L(t,Fe())?new Proxy(t,Z):t)}function v(t){if(t instanceof IDBRequest)return Pe(t);if(G.has(t))return G.get(t);let r=Qe(t);return r!==t&&(G.set(t,r),W.set(r,t)),r}var k=t=>W.get(t);function ge(t,r,{blocked:n,upgrade:i,blocking:d,terminated:s}={}){let u=indexedDB.open(t,r),m=v(u);return i&&u.addEventListener("upgradeneeded",f=>{i(v(u.result),f.oldVersion,f.newVersion,v(u.transaction))}),n&&u.addEventListener("blocked",()=>n()),m.then(f=>{s&&f.addEventListener("close",()=>s()),d&&f.addEventListener("versionchange",()=>d())}).catch(()=>{}),m}var $e=["get","getKey","getAll","getAllKeys","count"],Be=["put","add","delete","clear"],K=new Map;function xe(t,r){if(!(t instanceof IDBDatabase&&!(r in t)&&typeof r=="string"))return;if(K.get(r))return K.get(r);let n=r.replace(/FromIndex$/,""),i=r!==n,d=Be.includes(n);if(!(n in(i?IDBIndex:IDBObjectStore).prototype)||!(d||$e.includes(n)))return;let s=async function(u,...m){let f=this.transaction(u,d?"readwrite":"readonly"),p=f.store;return i&&(p=p.index(m.shift())),(await Promise.all([p[n](...m),d&&f.done]))[0]};return K.set(r,s),s}P(t=>({...t,get:(r,n,i)=>xe(r,n)||t.get(r,n,i),has:(r,n)=>!!xe(r,n)||t.has(r,n)}));var ze=["continue","continuePrimaryKey","advance"],he={},J=new WeakMap,ve=new WeakMap,Xe={get(t,r){if(!ze.includes(r))return t[r];let n=he[r];return n||(n=he[r]=function(...i){J.set(this,ve.get(this)[r](...i))}),n}};async function*Ve(...t){let r=this;if(r instanceof IDBCursor||(r=await r.openCursor(...t)),!r)return;r=r;let n=new Proxy(r,Xe);for(ve.set(n,r),W.set(n,k(r));r;)yield n,r=await(J.get(n)||r.continue()),J.delete(n)}function De(t,r){return r===Symbol.asyncIterator&&L(t,[IDBIndex,IDBObjectStore,IDBCursor])||r==="iterate"&&L(t,[IDBIndex,IDBObjectStore])}P(t=>({...t,get(r,n,i){return De(r,n)?Ve:t.get(r,n,i)},has(r,n){return De(r,n)||t.has(r,n)}}));async function je({project:t}){let r=null,n=[],i=[];do{let d=await fetch(r?`/api/pages/${t}/search/titles?followingId=${r}`:`/api/pages/${t}/search/titles`);r=d.headers.get("X-Following-Id"),i.push(d.json().then(s=>n.push(...s)))}while(r);return await Promise.all(i),n}function o(t){if(t===null||t===!0||t===!1)return NaN;var r=Number(t);return isNaN(r)?r:r<0?Math.ceil(r):Math.floor(r)}function e(t,r){if(r.length<t)throw new TypeError(t+" argument"+(t>1?"s":"")+" required, but only "+r.length+" present")}function a(t){e(1,arguments);let r=Object.prototype.toString.call(t);return t instanceof Date||typeof t=="object"&&r==="[object Date]"?new Date(t.getTime()):typeof t=="number"||r==="[object Number]"?new Date(t):((typeof t=="string"||r==="[object String]")&&typeof console!="undefined"&&(console.warn("Starting with v2.0.0-beta.1 date-fns doesn't accept strings as arguments. Please use `parseISO` to parse strings. See: https://git.io/fjule"),console.warn(new Error().stack)),new Date(NaN))}function D(t,r){e(2,arguments);var n=a(t),i=o(r);return isNaN(i)?new Date(NaN):(i&&n.setDate(n.getDate()+i),n)}function O(t,r){e(2,arguments);var n=a(t),i=o(r);if(isNaN(i))return new Date(NaN);if(!i)return n;var d=n.getDate(),s=new Date(n.getTime());s.setMonth(n.getMonth()+i+1,0);var u=s.getDate();return d>=u?s:(n.setFullYear(s.getFullYear(),s.getMonth(),d),n)}function C(t){return function(r){var n=r||{},i=n.width?String(n.width):t.defaultWidth,d=t.formats[i]||t.formats[t.defaultWidth];return d}}var st={full:"EEEE, MMMM do, y",long:"MMMM do, y",medium:"MMM d, y",short:"MM/dd/yyyy"},dt={full:"h:mm:ss a zzzz",long:"h:mm:ss a z",medium:"h:mm:ss a",short:"h:mm a"},ut={full:"{{date}} 'at' {{time}}",long:"{{date}} 'at' {{time}}",medium:"{{date}}, {{time}}",short:"{{date}}, {{time}}"},Qs={date:C({formats:st,defaultWidth:"full"}),time:C({formats:dt,defaultWidth:"full"}),dateTime:C({formats:ut,defaultWidth:"full"})};function I(t){return function(r,n){var i=n||{},d=i.context?String(i.context):"standalone",s;if(d==="formatting"&&t.formattingValues){let m=t.defaultFormattingWidth||t.defaultWidth,f=i.width?String(i.width):m;s=t.formattingValues[f]||t.formattingValues[m]}else{let m=t.defaultWidth,f=i.width?String(i.width):t.defaultWidth;s=t.values[f]||t.values[m]}var u=t.argumentCallback?t.argumentCallback(r):r;return s[u]}}var ft={narrow:["B","A"],abbreviated:["BC","AD"],wide:["Before Christ","Anno Domini"]},mt={narrow:["1","2","3","4"],abbreviated:["Q1","Q2","Q3","Q4"],wide:["1st quarter","2nd quarter","3rd quarter","4th quarter"]},lt={narrow:["J","F","M","A","M","J","J","A","S","O","N","D"],abbreviated:["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"],wide:["January","February","March","April","May","June","July","August","September","October","November","December"]},ct={narrow:["S","M","T","W","T","F","S"],short:["Su","Mo","Tu","We","Th","Fr","Sa"],abbreviated:["Sun","Mon","Tue","Wed","Thu","Fri","Sat"],wide:["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"]},pt={narrow:{am:"a",pm:"p",midnight:"mi",noon:"n",morning:"morning",afternoon:"afternoon",evening:"evening",night:"night"},abbreviated:{am:"AM",pm:"PM",midnight:"midnight",noon:"noon",morning:"morning",afternoon:"afternoon",evening:"evening",night:"night"},wide:{am:"a.m.",pm:"p.m.",midnight:"midnight",noon:"noon",morning:"morning",afternoon:"afternoon",evening:"evening",night:"night"}},gt={narrow:{am:"a",pm:"p",midnight:"mi",noon:"n",morning:"in the morning",afternoon:"in the afternoon",evening:"in the evening",night:"at night"},abbreviated:{am:"AM",pm:"PM",midnight:"midnight",noon:"noon",morning:"in the morning",afternoon:"in the afternoon",evening:"in the evening",night:"at night"},wide:{am:"a.m.",pm:"p.m.",midnight:"midnight",noon:"noon",morning:"in the morning",afternoon:"in the afternoon",evening:"in the evening",night:"at night"}};function xt(t,r){var n=Number(t),i=n%100;if(i>20||i<10)switch(i%10){case 1:return n+"st";case 2:return n+"nd";case 3:return n+"rd"}return n+"th"}var Vs={ordinalNumber:xt,era:I({values:ft,defaultWidth:"wide"}),quarter:I({values:mt,defaultWidth:"wide",argumentCallback:function(t){return Number(t)-1}}),month:I({values:lt,defaultWidth:"wide"}),day:I({values:ct,defaultWidth:"wide"}),dayPeriod:I({values:pt,defaultWidth:"wide",formattingValues:gt,defaultFormattingWidth:"wide"})};function ie(t){return function(r,n){var i=String(r),d=n||{},s=i.match(t.matchPattern);if(!s)return null;var u=s[0],m=i.match(t.parsePattern);if(!m)return null;var f=t.valueCallback?t.valueCallback(m[0]):m[0];return f=d.valueCallback?d.valueCallback(f):f,{value:f,rest:i.slice(u.length)}}}function S(t){return function(r,n){var i=String(r),d=n||{},s=d.width,u=s&&t.matchPatterns[s]||t.matchPatterns[t.defaultMatchWidth],m=i.match(u);if(!m)return null;var f=m[0],p=s&&t.parsePatterns[s]||t.parsePatterns[t.defaultParseWidth],g;return Object.prototype.toString.call(p)==="[object Array]"?g=vt(p,function(l){return l.test(f)}):g=ht(p,function(l){return l.test(f)}),g=t.valueCallback?t.valueCallback(g):g,g=d.valueCallback?d.valueCallback(g):g,{value:g,rest:i.slice(f.length)}}}function ht(t,r){for(var n in t)if(t.hasOwnProperty(n)&&r(t[n]))return n}function vt(t,r){for(var n=0;n<t.length;n++)if(r(t[n]))return n}var Dt=/^(\d+)(th|st|nd|rd)?/i,jt=/\d+/i,wt={narrow:/^(b|a)/i,abbreviated:/^(b\.?\s?c\.?|b\.?\s?c\.?\s?e\.?|a\.?\s?d\.?|c\.?\s?e\.?)/i,wide:/^(before christ|before common era|anno domini|common era)/i},bt={any:[/^b/i,/^(a|c)/i]},yt={narrow:/^[1234]/i,abbreviated:/^q[1234]/i,wide:/^[1234](th|st|nd|rd)? quarter/i},Ot={any:[/1/i,/2/i,/3/i,/4/i]},Tt={narrow:/^[jfmasond]/i,abbreviated:/^(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)/i,wide:/^(january|february|march|april|may|june|july|august|september|october|november|december)/i},It={narrow:[/^j/i,/^f/i,/^m/i,/^a/i,/^m/i,/^j/i,/^j/i,/^a/i,/^s/i,/^o/i,/^n/i,/^d/i],any:[/^ja/i,/^f/i,/^mar/i,/^ap/i,/^may/i,/^jun/i,/^jul/i,/^au/i,/^s/i,/^o/i,/^n/i,/^d/i]},St={narrow:/^[smtwf]/i,short:/^(su|mo|tu|we|th|fr|sa)/i,abbreviated:/^(sun|mon|tue|wed|thu|fri|sat)/i,wide:/^(sunday|monday|tuesday|wednesday|thursday|friday|saturday)/i},Mt={narrow:[/^s/i,/^m/i,/^t/i,/^w/i,/^t/i,/^f/i,/^s/i],any:[/^su/i,/^m/i,/^tu/i,/^w/i,/^th/i,/^f/i,/^sa/i]},kt={narrow:/^(a|p|mi|n|(in the|at) (morning|afternoon|evening|night))/i,any:/^([ap]\.?\s?m\.?|midnight|noon|(in the|at) (morning|afternoon|evening|night))/i},_t={any:{am:/^a/i,pm:/^p/i,midnight:/^mi/i,noon:/^no/i,morning:/morning/i,afternoon:/afternoon/i,evening:/evening/i,night:/night/i}},td={ordinalNumber:ie({matchPattern:Dt,parsePattern:jt,valueCallback:function(t){return parseInt(t,10)}}),era:S({matchPatterns:wt,defaultMatchWidth:"wide",parsePatterns:bt,defaultParseWidth:"any"}),quarter:S({matchPatterns:yt,defaultMatchWidth:"wide",parsePatterns:Ot,defaultParseWidth:"any",valueCallback:function(t){return t+1}}),month:S({matchPatterns:Tt,defaultMatchWidth:"wide",parsePatterns:It,defaultParseWidth:"any"}),day:S({matchPatterns:St,defaultMatchWidth:"wide",parsePatterns:Mt,defaultParseWidth:"any"}),dayPeriod:S({matchPatterns:kt,defaultMatchWidth:"any",parsePatterns:_t,defaultParseWidth:"any"})};var Am=24*60*60*1e3;function B(t){e(1,arguments);var r=a(t),n=r.getTime();return n}function z(t){return e(1,arguments),Math.floor(B(t)/1e3)}function U(t,r){e(2,arguments);var n=o(r);return D(t,-n)}function X(t,r){e(2,arguments);var n=o(r);return O(t,-n)}function F(t,r){if(e(2,arguments),!r||typeof r!="object")return new Date(NaN);let n="years"in r?o(r.years):0,i="months"in r?o(r.months):0,d="weeks"in r?o(r.weeks):0,s="days"in r?o(r.days):0,u="hours"in r?o(r.hours):0,m="minutes"in r?o(r.minutes):0,f="seconds"in r?o(r.seconds):0,p=X(a(t),i+n*12),g=U(p,s+d*7),l=m+u*60,h=(f+l*60)*1e3;return new Date(g.getTime()-h)}var Dr=Math.pow(10,8)*24*60*60*1e3,qv=-Dr;async function Ye(t,{includeJoined:r=!0}={}){let n=Math.floor(t.length/100)+1,d=(await Promise.all([...Array(n).keys()].map(async s=>{let u=new URLSearchParams;t.slice(s*100,100+s*100).forEach(p=>u.append("ids",p));let m=await fetch(`/api/projects?${u.toString()}`),{projects:f}=await m.json();return f}))).flat();return r?[...new Set(d.map(({id:u})=>u))].map(u=>d.find(m=>m.id===u)):[...new Set(d.map(({id:s})=>s))].filter(s=>t.some(u=>u===s)).map(s=>d.find(u=>u.id===s))}async function Ce(t,{maxInProgress:r=void 0}={}){if(!r||r<0||t.length<=r)return await Promise.all(t.map(d=>d()));let n=t.map(d=>!1),i=[];return await Promise.all([...Array(r).keys()].map(async d=>{do n[d]=!0,i[d]=await t[d](),d=n.findIndex(s=>!s);while(d!==-1)})),i}var M="cache-links",Ne="UserScript",Ee=7;var jr=ge(Ne,Ee);self.addEventListener("message",async({data:{projectIds:t,options:r}})=>{self.postMessage(await wr(await jr,t,r))});async function wr(t,r,n){let{reload:i=!1,expired:d=3600}=n??{},s=[],u=new Date;{let l=t.transaction(M,"readonly"),c=l.objectStore(M),h=await c.getAllKeys();s.push(...r.flatMap(j=>h.includes(j)?[]:[{id:j,prevFetched:0}]));let x=c.index("fetched"),y=i?x.iterate():x.iterate(IDBKeyRange.upperBound(F(u,{seconds:d}),!0));for await(let j of y){let{id:fe,fetched:Ue}=j.value;!r.includes(fe)||s.push({id:fe,prevFetched:z(Ue)})}await l.done}if(s.length===0)return!1;{let l=t.transaction(M,"readwrite"),c=l.objectStore(M);await Promise.all(s.flatMap(async({id:h,prevFetched:x})=>{if(x===0)return[];let{fetched:y,...j}=await c.get(h);return[await c.put({fetched:u,...j})]})),await l.done}let f=(await Ye(s.map(({id:l})=>l),{includeJoined:!1})).flatMap(({name:l,id:c,updated:h})=>{let x=s.find(y=>y.id===c)?.prevFetched??0;return h>x?[{project:l,id:c}]:[]});if(f.length===0)return!1;let p=0,g=await Ce(f.map(({project:l,id:c})=>async()=>{let h=p++;console.log(`[${h}/${f.length}] Fetching pages from /${l}...`);let x=await je({project:l});return console.log(`[${h}/${f.length}] Fetched pages from /${l}.`),{project:l,id:c,pages:x}}),{maxInProgress:10});{let l=t.transaction(M,"readwrite"),c=l.objectStore(M),h=await c.getAllKeys();await Promise.all(g.map(({id:x,project:y,pages:j})=>h.includes(x)?c.put({id:x,project:y,pages:j,fetched:u}):c.add({id:x,project:y,pages:j,fetched:u}))),await l.done}return!0}})();
sh
deno run --allow-net --allow-read --allow-write --allow-run --allow-env --unstable https://scrapbox.io/api/code/programming-notes/scrapbox-link-database@0.2.0/build.ts | xsel
build.ts
import {run} from '/api/code/takker/UserScriptをbundleするDeno_script/script.ts'; import {BlobToURI} from '/api/code/takker/BlobをData_URIに変換する/script.js'; const {outputFiles} = await run( `https://scrapbox.io/api/code/programming-notes/scrapbox-link-database@0.2.0/worker.js`, { '../date-fns.min.js/script.js': 'https://deno.land/x/date_fns@v2.15.0/index.js', }, { charset: 'utf8', bundle: true, minify: true, write: false, // 標準出力やfileにbundleしたコードを出力しない format: 'iife', } ); console.log(outputFiles?.[0]?.text ?? '');
worker.js
import { openDB } from '../idb/with-async-ittr.js'; import { fetchLinks, getProjectUpdated, } from '/api/code/takker/scrapbox-api-helper/scrapboxAPI.js'; import {sub, getUnixTime} from '../date-fns.min.js/script.js'; import {getProjectInfo} from '../scrapboxのproject情報を一括して取得するUserScript/script.js'; import {parallel} from '../promise-parallel-throttle/script.js'; import {DBName, Version, StoreName} from './settings.js'; const getDB = openDB(DBName, Version); self.addEventListener('message', async ({data: {projectIds, options}}) => { self.postMessage(await update(await getDB, projectIds, options)); }); // dataを更新する // reloadをつけるとprojects全てをnetworkから取得し直すが、更新が見つからなければ何もしない async function update(db, projectIds, options) { const {reload = false, expired = 3600,} = options ?? {}; // 更新対象のproject listを作る // projectsに含まれるもののみを更新対象とする const projectUpdateds = []; const now = new Date(); { const tx = db.transaction(StoreName, 'readonly'); const store = tx.objectStore(StoreName); const cachedProjects = await store.getAllKeys(); // cacheのないprojectを先に入れておく projectUpdateds.push(...projectIds.flatMap(id => { return cachedProjects.includes(id) ? [] : [{id, prevFetched: 0}]; })); const index = store.index('fetched'); const iterator = reload ? // 再読み込みする場合はすべてのprojectを更新対象にする index.iterate() : // fetched + expired < 現在時刻であるもののみ更新対象とする index.iterate( IDBKeyRange.upperBound(sub(now, {seconds: expired}), true) ); for await (const cursor of iterator) { const {id, fetched} = cursor.value; if (!projectIds.includes(id)) continue; projectUpdateds.push({ id, prevFetched: getUnixTime(fetched), // 秒単位のtimestampに変換しておく }); } await tx.done; } if (projectUpdateds.length === 0) return false; // 先に更新日時を書き込んでおく { const tx = db.transaction(StoreName, 'readwrite'); const store = tx.objectStore(StoreName); await Promise.all(projectUpdateds.flatMap(async ({id, prevFetched}) => { if (prevFetched === 0) return []; const {fetched, ...rest} = await store.get(id); return [await store.put({fetched: now, ...rest})]; })); await tx.done; } // 実際に更新する必要のあるproject listを作る const projectInfos = await getProjectInfo(projectUpdateds.map(({id}) => id), {includeJoined: false}); const targetProjects = projectInfos .flatMap( ({name, id, updated}) => { const prevFetched = (projectUpdateds.find(data => data.id === id)?.prevFetched ?? 0); return updated > prevFetched ? [{project: name, id}] : []; }); if (targetProjects.length === 0) return false; // networkからdataを取得する let counter = 0; const data = await parallel(targetProjects.map(({project, id}) => async () => { const index = counter++; console.log(`[${index}/${targetProjects.length}] Fetching pages from /${project}...`); const pages = await fetchLinks({project}); console.log(`[${index}/${targetProjects.length}] Fetched pages from /${project}.`); return {project, id, pages}; } ), {maxInProgress: 10}); // dataを格納する { const tx = db.transaction(StoreName, 'readwrite'); const store = tx.objectStore(StoreName); const keys = await store.getAllKeys(); await Promise.all( data.map(({id, project, pages}) => keys.includes(id) ? store.put({id, project, pages, fetched: now}) : store.add({id, project, pages, fetched: now}) ) ); await tx.done; } return true; }

test code
js
import('/api/code/programming-notes/scrapbox-link-database@0.2.0/test.js');
test.js
import {get, clear} from './script.js'; const watchListIds = Object.keys(JSON.parse(localStorage.getItem('projectsLastAccessed'))); window.getLinks = async (options) => await get(watchListIds, options); window.clearLinks = async (options) => await clear(watchListIds);

JavaScript
Deno