/takker/scrapbox-cache-fetch

generated at 11/17/2024, 8:13:59 AM
scrapbox-cache-fetch
Scrapbox REST APIの取得結果をlocal storageにcacheしておくscript

用途
awesome-suggestionの高速化
 fetch の数を減らして待ち時間を減らす
scrapbox.ioへのサーバー負荷を減らす

挙動
初回 or cacheが存在しない
APIを叩く
cacheの取得日時から maxAge 秒過ぎている
対象projectの最終更新日時がcacheの取得日時よりあとの場合のみAPIを叩く
特に更新がなければcacheを返す
otherwise
cacheを返す

interface
 getAllLinks(projects, {maxAge, verbose}) 
 projecs のすべてのリンクを取得する
/scrapboxlab/api/pages/:projectname/search/titlesを叩く
 maxAge  (optional)
cacheの有効期間
単位は秒
default:  3600 * 24 
 verbose  (optional)
ログを出力する
default:  false 
 reload  (optional)
networkから取得し直す
default:  false 
返り値の型
tstype LinksResult = {
  project: string;
  results: {
    title: string;
    links: string[];
  }[];
  type: 'network' : 'cache';
};
 getAllIcons(project, {maxAge}) 
 project のすべてのアイコン入りページを取得する
/scrapboxlab/api/pages/:projectname/を叩く
parametersは getAllLinks と同じ
返り値の型
tstype IconsResult = {
  project: string;
  results: string[];
  type: 'network' : 'cache';
};
 getAllPageCards(projects, {maxAge}) 
 projects のすべてのscrapboxのページカードを取得する
/scrapboxlab/api/pages/:projectname/を叩く
parametersは getAllLinks と同じ
返り値の型
tstype CardsResult = {
  project: string;
  results: {
    title: string;
    image: string;
    descriptions: string[];
  }[];
  type: 'network' | 'cache';
};
更新通知event
 scrapbox-user-script-cache-update-links 
 e.detail: (LinksResult & {project: stirng})[] 
 scrapbox-user-script-cache-update-icons 
 e.detail: (IconsResult & {project: stirng})[] 
 scrapbox-user-script-cache-update-cards 
 e.detail: (CardsResult & {project: stirng})[] 
 window.addEventListener() で受け取れる

既知の問題
local storageの容量をすぐ越えてしまう
対策：Indexed DBに切り替える

実装
/takker/external-completionにcache機能を導入するを参考にした

script.jsexport async function getAllLinks(project, {maxAge = 24 * 3600, verbose = false, reload = false} = {}) {
  return await getData(
    'scrapbox-user-script-cache-update-links',
    async () => (await fetchAllLinks(project, verbose))
      .map(({title, links}) => {return {title, links}}),
    project,
    {maxAge, verbose, reload},);
}
export async function getAllIcons(project, {maxAge = 24 * 3600, verbose = false, reload = false} = {}) {
  return await getData(
    'scrapbox-user-script-cache-update-icons',
    () => fetchAllIcons(project, verbose),
    project,
    {maxAge, verbose, reload},);
}

共通する処理
script.jsasync function getData(eventName, callback, project, {maxAge, verbose, reload}) {
  const lastUpdated = getTimestamp(eventName, project);
  const expired = lastUpdated + maxAge;
  const now = (new Date()).getTime() / 1000;
  
  if (reload) return await getFromNetwork(eventName, callback, project, verbose);
  if (expired < now) {
    const updated = await getProjectUpdated(project, verbose);
    _log('getData', verbose, {lastUpdated, expired, now, updated});
    if (lastUpdated < updated) { 
      return await getFromNetwork(eventName, callback, project, verbose);
    }
  }
  
  const cache = getFromCache(eventName, project, verbose);
  if (cache.results) return cache;
  return await getFromNetwork(eventName, callback, project, verbose);
}

Local Storageと通信を行う
script.jsfunction getTimestamp(eventName, project) {
  return parseInt(localStorage.getItem(`${eventName}-${project}-timestamp`) ?? 0);
}

function getFromCache(eventName, project, verbose) {
  _log('getFromCache', verbose, `Get data of /${project} from cache.`);
  const dataString = localStorage.getItem(`${eventName}-${project}`);
  return {
    project,
    results: dataString !== null ? JSON.parse(dataString) : undefined,
    state: 'cache',
  };
}
async function getFromNetwork(eventName, callback, project, verbose) {
  _log('getFromNetwork', verbose, `Get data of /${project} from network.`);
  const results = await callback();
  UpdateCache(eventName, project, results, verbose);
  return {project, results, state: 'network',};
}

function UpdateCache(eventName, project, results, verbose) {
  localStorage.setItem(`${eventName}-${project}`, JSON.stringify(results));
  localStorage.setItem(`${eventName}-${project}-timestamp`, Math.round((new Date()).getTime() / 1000));

  // 同じタブのscriptに更新通知を出す
  dispatchUpdateEvent(eventName, project, {state: 'cache', verbose});
}

function dispatchUpdateEvent(eventName, project, {state, verbose}) {
  const {results} = getFromCache(eventName, project, verbose);
  window.dispatchEvent(new CustomEvent(eventName, {bubbles: true, detail: {
    project, results, state,
  }}));
}

実際にAPIを叩く関数
script.jsasync function fetchAllLinks(project, verbose) {
  let followingId = null;
  const linksData = [];
  const promises = []; //処理待ち用
  _log('fetchAllLinks', verbose, `Start loading links from /${project}...`);
  do {
    _log('fetchAllLinks', verbose, `Loading links from /${project}: followingId = ${followingId}`);
    const response = await (!followingId ? 
      fetch(`/api/pages/${project}/search/titles`) :
      fetch(`/api/pages/${project}/search/titles?followingId=${followingId}`)
      )
    followingId = response.headers.get('X-Following-Id');
    promises.push(response.json().then(links => linksData.push(...links)));
  } while(followingId);
  // 全てのリンク情報を配列に格納し終わったら返す
  await Promise.all(promises);
  _log('fetchAllLinks', verbose, `Loaded all links from /${project}: `, linksData);
  return linksData;
}

script.jsasync function fetchAllIcons(project, verbose) {
  const cards = await fetchAllCards(project, verbose);
  return cards.flatMap(({title, image}) => image !== null ? [title] : []);
}

script.jsasync function fetchAllCards(project, verbose) {
  const pageNum = await getPageCount(project); //取得するページ数
  const maxIndex = pageNum > 1000 ? Math.floor(pageNum / 1000) + 1 : 1; // APIを叩く回数
  const pages = []; // ページ情報を格納する配列
  
  // 一気にAPIを叩いてページ情報を取得する
  const promises = [...Array(maxIndex)]
    .map(async (_, index) => {
      const res = await fetch(`/api/pages/${project}/?limit=1000&skip=${index*1000}`);
      const json = await res.json();
      return json.pages;
    });
  
  const rawPages = (await Promise.all(promises)).flat();
  return rawPages.map(({title, image, descriptions}) => {return {title, image, descriptions};});
}

script.jsasync function getProjectUpdated(project, verbose) {
  try {
    const response = await fetch(`/api/projects/${project}`);
    if(!response.ok) throw Error(`Invalid response. status code: ${response.status}`);
    const {updated} = await response.json();
    return updated;
  } catch(e) {
   throw e;
  }
}

script.jsasync function getPageCount(project, verbose) {
  try {
    const response = await fetch(`/api/pages/${project}/?limit=1`);
    if(!response.ok) throw Error(`Invalid response. status code: ${response.status}`);
    const json = await response.json();
    return parseInt(json.count);
  } catch(e) {
   throw e;
  }
}

ログ出力
script.jsfunction _log(functionName, verbose, ...messages) {
  if (!verbose) return;
  console.log(`[${functionName}@scrapbox-cache-fetch]`, ...messages);
}

テストコード
js(async () => {
  const {execute} = await import('/api/code/takker/scrapbox-cache-fetch/test1.js');
  await execute();
})();
test1.jsimport {getAllLinks} from '/api/code/takker/scrapbox-cache-fetch/script.js';
export const projects = [
  'hub',
  'shokai',
  'nishio',
  'masui',
  'motoso',
  'villagepump',
  'rashitamemo',
  'rakusai',
  'yuiseki',
  'june29',
  'ucdktr2016',
  'thinkandcreateteck',
  'customize',
  'scrapboxlab',
  'scrasobox',
  'foldrr',
  'scrapbox-drinkup',
  'public-mrsekut',
  'mrsekut-p',
  'marshmallow-rm',
  'wkpmm',
  'sushitecture',
  'nwtgck',
  'dojineko',
  'kadoyau',
  'inteltank',
  'sta',
  'kn1cht',
  'miyamonz',
  'rmaruon',
  'MISONLN41',
  'yuta0801',
  'choiyakiBox',
  'choiyaki-hondana',
  'spud-oimo',
  'keroxp',
  'aioilight',
];
export const execute = async ({maxAge = 60, verbose=  true} = {}) => {
  console.log(await getAllLinks(projects, {maxAge, verbose}));
};

#2021-02-20 02:57:59 