getAllPages()
scrapbox project内の全てのページを読み込む函数
project内の全てのページ情報を取得するscriptとは違って、本文も取得して返す
実装
owner or admin
others
api/pages/:projectname/search/titlesでタイトルを取得し、api/pages/:projectname/:pagetitleを順に叩いて本文を得る
一度に大量のfetchを叩かないよう、throttleしてある
実装はtakker99/ScrapBubbleから拝借した
page数は事前にapi/projects/:projectnameから取得しておく
scrapbox-userscript-stdに入れるかは悩んでいる
入れるなら、もっと汎用化したいところ
とりあえずこのページに置いておこう
$ deno check --remote -r=https://scrapbox.io "https://scrapbox.io/api/code/takker/getAllPages()/mod.ts"mod.tsimport {
BaseOptions,
Fetch,
exportPages,
listPages,
getPage,
readLinksBulk,
InvalidFollowingIdError,
Result,
TooLongURIError,
} from "../scrapbox-userscript-std/rest.ts";
import {
Page,
ExportedPage,
NotFoundError,
NotLoggedInError,
NotMemberError,
} from "../scrapbox-jp%2Ftypes/rest.ts";
import { makeThrottle } from "https://raw.githubusercontent.com/takker99/Scrapbubble/0.7.3/throttle.ts";
export interface PageLoader {
count: number;
pages: ExportedPage<true>[] | AsyncGenerator<
Result<Page, NotFoundError | NotLoggedInError | NotMemberError | TooLongURIError>,
void,
unknown
>;
}
export const getAllPages = async (project: string, options?: BaseOptions): Promise<Result<PageLoader, NotFoundError | NotLoggedInError | NotMemberError | InvalidFollowingIdError>> => {
{
const result = await exportPages(project, { metadata: true, ...options });
if (result.ok) return {
ok: true,
value: {
count: result.value.pages.length,
pages: result.value.pages,
},
};
const error = result.value;
if (error.name === "NotFoundError") return { ok: false, value: error };
}
const result = await listPages(project, { ...options, limit: 1 });
if (!result.ok) return result;
const reader = await readLinksBulk(project, options);
if ("name" in reader) return { ok: false, value: reader };
return {
ok: true,
value: {
count: result.value.count,
pages: (async function* () {
const promises: Promise<
Result<Page, NotFoundError | NotLoggedInError | NotMemberError | TooLongURIError>
>[] = [];
const throttle = makeThrottle<Response>(3);
const { fetch: fetch_ = globalThis.fetch, ...rest } = options ?? {};
const fetch: Fetch = (args) => throttle(() => fetch_(args));
for await (const links of reader) {
promises.push(...links.map(
(link) => getPage(project, link.title, { ...rest, fetch })
));
}
for await (const promise of promises) {
yield await promise;
}
})(),
},
};
};
jsawait (async() => {
const res = await getAllPages("customize");
if(!res.ok){
console.error(res.value);
return;
}
let i = 0;
for await (const result of res.value.pages){
i++;
if(!result.ok){
console.error(result.value);
continue;
}
console.log(`%c${i}/${res.value.count}`, "color:gray", result.value.title);
}
})();
#2023-02-20 09:45:36
#2023-02-17 05:57:49